pypi.rb 9.39 KB
Newer Older
1
# typed: false
2
3
# frozen_string_literal: true

4
# Helper functions for updating PyPI resources.
Markus Reiter's avatar
Markus Reiter committed
5
6
#
# @api private
7
module PyPI
8
9
  extend T::Sig

10
11
12
  module_function

  PYTHONHOSTED_URL_PREFIX = "https://files.pythonhosted.org/packages/"
Markus Reiter's avatar
Markus Reiter committed
13
  private_constant :PYTHONHOSTED_URL_PREFIX
14
15
16

  @pipgrip_installed = nil

17
18
19
20
21
  # PyPI Package
  #
  # @api private
  class Package
    extend T::Sig
22

23
24
25
    attr_accessor :name
    attr_accessor :extras
    attr_accessor :version
26

27
28
29
    sig { params(package_string: String, is_url: T::Boolean).void }
    def initialize(package_string, is_url: false)
      @pypi_info = nil
30

31
32
33
34
35
      if is_url
        unless package_string.start_with?(PYTHONHOSTED_URL_PREFIX) &&
               match = File.basename(package_string).match(/^(.+)-([a-z\d.]+?)(?:.tar.gz|.zip)$/)
          raise ArgumentError, "package should be a valid PyPI url"
        end
36

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
        @name = match[1]
        @version = match[2]
        return
      end

      @name = package_string
      @name, @version = @name.split("==") if @name.include? "=="

      return unless match = @name.match(/^(.*?)\[(.+)\]$/)

      @name = match[1]
      @extras = match[2].split ","
    end

    # Get name, URL, SHA-256 checksum, and latest version for a given PyPI package.
    sig { params(version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) }
    def pypi_info(version: nil)
      return @pypi_info if @pypi_info.present? && version.blank?

      version ||= @version
      metadata_url = if version.present?
        "https://pypi.org/pypi/#{@name}/#{version}/json"
      else
        "https://pypi.org/pypi/#{@name}/json"
      end
      out, _, status = curl_output metadata_url, "--location"

      return unless status.success?

      begin
        json = JSON.parse out
      rescue JSON::ParserError
        return
      end

      sdist = json["urls"].find { |url| url["packagetype"] == "sdist" }
      return json["info"]["name"] if sdist.nil?

      @pypi_info = [json["info"]["name"], sdist["url"], sdist["digests"]["sha256"], json["info"]["version"]]
    end

    sig { returns(T::Boolean) }
    def valid_pypi_package?
      info = pypi_info
      info.present? && info.is_a?(Array)
    end

    sig { returns(String) }
    def to_s
      out = @name
      out += "[#{@extras.join(",")}]" if @extras.present?
      out += "==#{@version}" if @version.present?
      out
90
    end
91

92
93
94
95
    sig { params(other: Package).returns(T::Boolean) }
    def same_package?(other)
      @name.tr("_", "-") == other.name.tr("_", "-")
    end
96

97
98
99
100
    # Compare only names so we can use .include? on a Package array
    sig { params(other: Package).returns(T::Boolean) }
    def ==(other)
      same_package?(other)
101
102
    end

103
104
105
106
107
    sig { params(other: Package).returns(T.nilable(Integer)) }
    def <=>(other)
      @name <=> other.name
    end
  end
108

109
110
111
112
113
114
  sig { params(url: String, version: T.any(String, Version)).returns(T.nilable(String)) }
  def update_pypi_url(url, version)
    package = Package.new url, is_url: true

    _, url = package.pypi_info(version: version)
    url
115
116
  end

117
  # Return true if resources were checked (even if no change).
118
119
120
121
122
123
124
125
126
127
128
129
  sig do
    params(
      formula:                  Formula,
      version:                  T.nilable(String),
      package_name:             T.nilable(String),
      extra_packages:           T.nilable(T::Array[String]),
      exclude_packages:         T.nilable(T::Array[String]),
      print_only:               T::Boolean,
      silent:                   T::Boolean,
      ignore_non_pypi_packages: T::Boolean,
    ).returns(T.nilable(T::Boolean))
  end
130
131
132
  def update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil, exclude_packages: nil,
                               print_only: false, silent: false, ignore_non_pypi_packages: false)

133
    auto_update_list = formula.tap&.pypi_formula_mappings
134
135
    if auto_update_list.present? && auto_update_list.key?(formula.full_name) &&
       package_name.blank? && extra_packages.blank? && exclude_packages.blank?
136
137
138
139

      list_entry = auto_update_list[formula.full_name]
      case list_entry
      when false
140
141
142
        unless print_only
          odie "The resources for \"#{formula.name}\" need special attention. Please update them manually."
        end
143
144
145
146
147
148
149
      when String
        package_name = list_entry
      when Hash
        package_name = list_entry["package_name"]
        extra_packages = list_entry["extra_packages"]
        exclude_packages = list_entry["exclude_packages"]
      end
150
151
    end

152
153
154
155
156
157
158
159
160
    main_package = if package_name.present?
      Package.new(package_name)
    else
      begin
        Package.new(formula.stable.url, is_url: true)
      rescue ArgumentError
        nil
      end
    end
161

162
    if main_package.blank?
163
164
165
166
167
168
      return if ignore_non_pypi_packages

      odie <<~EOS
        Could not infer PyPI package name from URL:
          #{Formatter.url(formula.stable.url)}
      EOS
169
170
    end

171
172
    unless main_package.valid_pypi_package?
      return if ignore_non_pypi_packages
173

174
      odie "\"#{main_package}\" is not available on PyPI."
175
176
    end

177
    main_package.version = version if version.present?
178

179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
    extra_packages = (extra_packages || []).map { |p| Package.new p }
    exclude_packages = (exclude_packages || []).map { |p| Package.new p }

    input_packages = [main_package]
    extra_packages.each do |extra_package|
      if !extra_package.valid_pypi_package? && !ignore_non_pypi_packages
        odie "\"#{extra_package}\" is not available on PyPI."
      end

      input_packages.each do |existing_package|
        if existing_package.same_package?(extra_package) && existing_package.version != extra_package.version
          odie "Conflicting versions specified for the `#{extra_package.name}` package: "\
                "#{existing_package.version}, #{extra_package.version}"
        end
      end
194

195
      input_packages << extra_package unless input_packages.include? extra_package
196
197
    end

198
199
200
201
    formula.resources.each do |resource|
      if !print_only && !resource.url.start_with?(PYTHONHOSTED_URL_PREFIX)
        odie "\"#{formula.name}\" contains non-PyPI resources. Please update the resources manually."
      end
202
203
    end

204
205
206
    @pipgrip_installed ||= Formula["pipgrip"].any_version_installed?
    odie '"pipgrip" must be installed (`brew install pipgrip`)' unless @pipgrip_installed

207
208
209
210
    found_packages = []
    input_packages.each do |package|
      ohai "Retrieving PyPI dependencies for \"#{package}\"..." if !print_only && !silent
      pipgrip_output = Utils.popen_read Formula["pipgrip"].bin/"pipgrip", "--json", "--no-cache-dir", package.to_s
211
212
      unless $CHILD_STATUS.success?
        odie <<~EOS
213
214
          Unable to determine dependencies for \"#{package}\" because of a failure when running
          `pipgrip --json --no-cache-dir #{package}`.
215
216
217
          Please update the resources for \"#{formula.name}\" manually.
        EOS
      end
218

219
220
221
222
223
224
225
226
227
      JSON.parse(pipgrip_output).to_h.each do |new_name, new_version|
        new_package = Package.new("#{new_name}==#{new_version}")

        found_packages.each do |existing_package|
          if existing_package.same_package?(new_package) && existing_package.version != new_package.version
            odie "Conflicting versions found for the `#{new_package.name}` resource: "\
                 "#{existing_package.version}, #{new_package.version}"
          end
        end
228

229
        found_packages << new_package unless found_packages.include? new_package
230
      end
231
232
    end

233
    # Remove extra packages that may be included in pipgrip output
234
    exclude_list = %W[#{main_package.name.downcase} argparse pip setuptools wheel wsgiref].map { |p| Package.new p }
235
236
    found_packages.delete_if { |package| exclude_list.include? package }

237
    new_resource_blocks = ""
238
    found_packages.sort.each do |package|
239
      if exclude_packages.include? package
240
        ohai "Excluding \"#{package}\"" if !print_only && !silent
241
242
243
        next
      end

244
245
      ohai "Getting PyPI info for \"#{package}\"" if !print_only && !silent
      name, url, checksum = package.pypi_info
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
      # Fail if unable to find name, url or checksum for any resource
      if name.blank?
        odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."
      elsif url.blank? || checksum.blank?
        odie <<~EOS
          Unable to find the URL and/or sha256 for the \"#{name}\" resource.
          Please update the resources for \"#{formula.name}\" manually.
        EOS
      end

      # Append indented resource block
      new_resource_blocks += <<-EOS
  resource "#{name}" do
    url "#{url}"
    sha256 "#{checksum}"
  end

      EOS
    end

    if print_only
      puts new_resource_blocks.chomp
      return
    end

271
272
    # Check whether resources already exist (excluding virtualenv dependencies)
    if formula.resources.all? { |resource| resource.name.start_with?("homebrew-") }
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
      # Place resources above install method
      inreplace_regex = /  def install/
      new_resource_blocks += "  def install"
    else
      # Replace existing resource blocks with new resource blocks
      inreplace_regex = /  (resource .* do\s+url .*\s+sha256 .*\s+ end\s*)+/
      new_resource_blocks += "  "
    end

    ohai "Updating resource blocks" unless silent
    Utils::Inreplace.inreplace formula.path do |s|
      if s.inreplace_string.scan(inreplace_regex).length > 1
        odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually."
      end
      s.sub! inreplace_regex, new_resource_blocks
    end
289
290

    true
291
292
  end
end