Merge pull request #8059 from Rylan12/pypi-poet-migration

dev-cmd: add update-python-resources command

Library/Homebrew/dev-cmd/bump-formula-pr.rb

@@ -2,6 +2,7 @@
 
 require "formula"
 require "cli/parser"
+require "utils/pypi"
 
 module Homebrew
   module_function
@@ -329,6 +330,11 @@ module Homebrew
       alias_rename.map! { |a| formula.tap.alias_dir/a }
     end
 
+    ohai "brew update-python-resources #{formula.name}"
+    if !args.dry_run? || (args.dry_run? && args.write?)
+      PyPI.update_python_resources! formula, new_formula_version, silent: true, ignore_non_pypi_packages: true
+    end
+
     run_audit(formula, alias_rename, old_contents)
 
     formula.path.parent.cd do

Library/Homebrew/dev-cmd/create.rb

@@ -4,6 +4,7 @@ require "formula"
 require "formula_creator"
 require "missing_formula"
 require "cli/parser"
+require "utils/pypi"
 
 module Homebrew
   module_function
@@ -137,6 +138,8 @@ module Homebrew
 
     fc.generate!
 
+    PyPI.update_python_resources! Formula[fc.name], ignore_non_pypi_packages: true if args.python?
+
     puts "Please run `brew audit --new-formula #{fc.name}` before submitting, thanks."
     exec_editor fc.path
   end

Library/Homebrew/dev-cmd/update-python-resources.rb

+# frozen_string_literal: true
+
+require "cli/parser"
+require "utils/pypi"
+
+module Homebrew
+  module_function
+
+  def update_python_resources_args
+    Homebrew::CLI::Parser.new do
+      usage_banner <<~EOS
+        `update-python-resources` [<options>] <formula>
+
+        Update versions for PyPI resource blocks in <formula>.
+      EOS
+      switch "-p", "--print-only",
+             description: "Print the updated resource blocks instead of changing <formula>."
+      switch "-s", "--silent",
+             description: "Suppress any output."
+      switch "--ignore-non-pypi-packages",
+             description: "Don't fail if <formula> is not a PyPI package."
+      flag "--version=",
+           description: "Use the specified <version> when finding resources for <formula>. "\
+                        "If no version is specified, the current version for <formula> will be used."
+      min_named :formula
+    end
+  end
+
+  def update_python_resources
+    args = update_python_resources_args.parse
+
+    args.formulae.each do |formula|
+      PyPI.update_python_resources! formula, args.version, print_only: args.print_only?, silent: args.silent?,
+                                    ignore_non_pypi_packages: args.ignore_non_pypi_packages?
+    end
+  end
+end

Library/Homebrew/formula_creator.rb

@@ -134,7 +134,7 @@ module Homebrew
           # depends_on "cmake" => :build
         <% end %>
 
-        <% if mode == :perl || mode == :python %>
+        <% if mode == :perl %>
           # Additional dependency
           # resource "" do
           #   url ""

Library/Homebrew/utils/pypi.rb

+# frozen_string_literal: true
+
+module PyPI
+  module_function
+
+  PYTHONHOSTED_URL_PREFIX = "https://files.pythonhosted.org/packages/"
+
+  @pipgrip_installed = nil
+
+  # Get name, url, and version for a given pypi package
+  def get_pypi_info(package, version)
+    metadata_url = "https://pypi.org/pypi/#{package}/#{version}/json"
+    out, _, status = curl_output metadata_url, "--location"
+
+    return unless status.success?
+
+    begin
+      json = JSON.parse out
+    rescue JSON::ParserError
+      return
+    end
+
+    sdist = json["urls"].find { |url| url["packagetype"] == "sdist" }
+    [json["info"]["name"], sdist["url"], sdist["digests"]["sha256"]]
+  end
+
+  def update_python_resources!(formula, version = nil, print_only: false, silent: false,
+                               ignore_non_pypi_packages: false)
+
+    @pipgrip_installed ||= Formula["pipgrip"].any_version_installed?
+    odie '"pipgrip" must be installed (`brew install pipgrip`)' unless @pipgrip_installed
+
+    # PyPI package name isn't always the same as the formula name. Try to infer from the URL.
+    pypi_name = if formula.stable.url.start_with?(PYTHONHOSTED_URL_PREFIX)
+      File.basename(formula.stable.url).match(/^(.+)-[a-z\d.]+$/)[1]
+    else
+      formula.name
+    end
+
+    version ||= formula.version
+
+    if get_pypi_info(pypi_name, version).blank?
+      odie "\"#{pypi_name}\" at version #{version} is not available on PyPI." unless ignore_non_pypi_packages
+      return
+    end
+
+    non_pypi_resources = formula.resources.reject do |resource|
+      resource.url.start_with? PYTHONHOSTED_URL_PREFIX
+    end
+
+    if non_pypi_resources.present? && !print_only
+      odie "\"#{formula.name}\" contains non-PyPI resources. Please update the resources manually."
+    end
+
+    ohai "Retrieving PyPI dependencies for \"#{pypi_name}==#{version}\"" if !print_only && !silent
+    pipgrip_output = Utils.popen_read Formula["pipgrip"].bin/"pipgrip", "--json", "#{pypi_name}==#{version}"
+    unless $CHILD_STATUS.success?
+      odie <<~EOS
+        Unable to determine dependencies for \"#{pypi_name}\" because of a failure when running
+        `pipgrip --json #{pypi_name}==#{version}`. Please update the resources for \"#{formula.name}\" manually.
+      EOS
+    end
+
+    packages = JSON.parse(pipgrip_output).sort.to_h
+
+    # Remove extra packages that may be included in pipgrip output
+    exclude_list = %W[#{pypi_name} argparse pip setuptools wheel wsgiref]
+    packages.delete_if do |package|
+      exclude_list.include? package
+    end
+
+    new_resource_blocks = ""
+    packages.each do |package, package_version|
+      name, url, checksum = get_pypi_info package, package_version
+      # Fail if unable to find name, url or checksum for any resource
+      if name.blank?
+        odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."
+      elsif url.blank? || checksum.blank?
+        odie <<~EOS
+          Unable to find the URL and/or sha256 for the \"#{name}\" resource.
+          Please update the resources for \"#{formula.name}\" manually.
+        EOS
+      end
+
+      # Append indented resource block
+      new_resource_blocks += <<-EOS
+  resource "#{name}" do
+    url "#{url}"
+    sha256 "#{checksum}"
+  end
+
+      EOS
+    end
+
+    if print_only
+      puts new_resource_blocks.chomp
+      return
+    end
+
+    # Check whether resources already exist (excluding homebrew-virtualenv)
+    if formula.resources.blank? ||
+       (formula.resources.length == 1 && formula.resources.first.name == "homebrew-virtualenv")
+      # Place resources above install method
+      inreplace_regex = /  def install/
+      new_resource_blocks += "  def install"
+    else
+      # Replace existing resource blocks with new resource blocks
+      inreplace_regex = /  (resource .* do\s+url .*\s+sha256 .*\s+ end\s*)+/
+      new_resource_blocks += "  "
+    end
+
+    ohai "Updating resource blocks" unless silent
+    Utils::Inreplace.inreplace formula.path do |s|
+      if s.inreplace_string.scan(inreplace_regex).length > 1
+        odie "Unable to update resource blocks for \"#{formula.name}\" automatically. Please update them manually."
+      end
+      s.sub! inreplace_regex, new_resource_blocks
+    end
+  end
+end

completions/internal_commands_list.txt

@@ -88,6 +88,7 @@ untap
 up
 update
 update-license-data
+update-python-resources
 update-report
 update-reset
 update-test

docs/Manpage.md

@@ -1081,6 +1081,19 @@ directory.
 * `--commit`:
   Commit changes to the SPDX license data.
 
+### `update-python-resources` [*`options`*] *`formula`*
+
+Update versions for PyPI resource blocks in *`formula`*.
+
+* `-p`, `--print-only`:
+  Print the updated resource blocks instead of changing *`formula`*.
+* `-s`, `--silent`:
+  Suppress any output.
+* `--ignore-non-pypi-packages`:
+  Don't fail if *`formula`* is not a PyPI package.
+* `--version`:
+  Use the specified *`version`* when finding resources for *`formula`*. If no version is specified, the current version for *`formula`* will be used.
+
 ### `update-test` [*`options`*]
 
 Run a test of `brew update` with a new repository clone. If no options are

manpages/brew.1

@@ -1416,6 +1416,25 @@ Return a failing status code if current license data\'s version is the same as t
 \fB\-\-commit\fR
 Commit changes to the SPDX license data\.
 .
+.SS "\fBupdate\-python\-resources\fR [\fIoptions\fR] \fIformula\fR"
+Update versions for PyPI resource blocks in \fIformula\fR\.
+.
+.TP
+\fB\-p\fR, \fB\-\-print\-only\fR
+Print the updated resource blocks instead of changing \fIformula\fR\.
+.
+.TP
+\fB\-s\fR, \fB\-\-silent\fR
+Suppress any output\.
+.
+.TP
+\fB\-\-ignore\-non\-pypi\-packages\fR
+Don\'t fail if \fIformula\fR is not a PyPI package\.
+.
+.TP
+\fB\-\-version\fR
+Use the specified \fIversion\fR when finding resources for \fIformula\fR\. If no version is specified, the current version for \fIformula\fR will be used\.
+.
 .SS "\fBupdate\-test\fR [\fIoptions\fR]"
 Run a test of \fBbrew update\fR with a new repository clone\. If no options are passed, use \fBorigin/master\fR as the start commit\.
 .