livecheck migration: create Homebrew::Livecheck

Co-authored-by: Sam Ford <1584702+samford@users.noreply.github.com>
Co-authored-by: Thierry Moisan <thierry.moisan@gmail.com>
Co-authored-by: Dawid Dziurla <dawidd0811@gmail.com>
Co-authored-by: Maxim Belkin <maxim.belkin@gmail.com>
Co-authored-by: Issy Long <me@issyl0.co.uk>
Co-authored-by: Mike McQuaid <mike@mikemcquaid.com>
Co-authored-by: Seeker <meaningseeking@protonmail.com>

Library/Homebrew/livecheck/livecheck.rb

+# frozen_string_literal: true
+
+module Homebrew
+  module Livecheck
+    module_function
+
+    GITHUB_SPECIAL_CASES = %w[
+      api.github.com
+      /latest
+      mednafen
+      camlp5
+      kotlin
+      osrm-backend
+      prometheus
+      pyenv-virtualenv
+      sysdig
+      shairport-sync
+      yuicompressor
+    ].freeze
+
+    UNSTABLE_VERSION_KEYWORDS = %w[
+      alpha
+      beta
+      bpo
+      dev
+      experimental
+      prerelease
+      preview
+      rc
+    ].freeze
+
+    def livecheck_formulae(formulae_to_check, args)
+      # Identify any non-homebrew/core taps in use for current formulae
+      non_core_taps = {}
+      formulae_to_check.each do |f|
+        next if f.tap.blank?
+        next if f.tap.name == CoreTap.instance.name
+        next if non_core_taps[f.tap.name]
+
+        non_core_taps[f.tap.name] = f.tap
+      end
+      non_core_taps = non_core_taps.sort.to_h
+
+      # Load additional Strategy files from taps
+      non_core_taps.each_value do |tap|
+        tap_strategy_path = "#{tap.path}/livecheck/strategy"
+        Dir["#{tap_strategy_path}/*.rb"].sort.each(&method(:require)) if Dir.exist?(tap_strategy_path)
+      end
+
+      # Cache demodulized strategy names, to avoid repeating this work
+      @livecheck_strategy_names = {}
+      Strategy.constants.sort.each do |strategy_symbol|
+        strategy = Strategy.const_get(strategy_symbol)
+        @livecheck_strategy_names[strategy] = strategy.name.demodulize
+      end
+      @livecheck_strategy_names.freeze
+
+      has_a_newer_upstream_version = false
+      formulae_checked = formulae_to_check.sort.map.with_index do |formula, i|
+        if args.debug? && i.positive?
+          puts <<~EOS
+
+            ----------
+
+          EOS
+        end
+
+        skip_result = skip_conditions(formula, args: args)
+        next skip_result if skip_result != false
+
+        formula.head.downloader.shutup! if formula.head?
+
+        current = formula.head? ? formula.installed_version.version.commit : formula.version
+
+        latest = if formula.stable?
+          version_info = latest_version(formula, args: args)
+          version_info[:latest] if version_info.present?
+        else
+          formula.head.downloader.fetch_last_commit
+        end
+
+        if latest.blank?
+          no_versions_msg = "Unable to get versions"
+          raise TypeError, no_versions_msg unless args.json?
+
+          next version_info if version_info.is_a?(Hash) && version_info[:status] && version_info[:messages]
+
+          next status_hash(formula, "error", [no_versions_msg], args: args)
+        end
+
+        if (m = latest.to_s.match(/(.*)-release$/)) && !current.to_s.match(/.*-release$/)
+          latest = Version.new(m[1])
+        end
+
+        # A HEAD-only formula is outdated when the latest commit hash and installed commit hash differ
+        is_outdated = if formula.head?
+          (current != latest)
+        else
+          (current < latest)
+        end
+
+        is_newer_than_upstream = formula.stable? && (current > latest)
+
+        info = {
+          formula: formula_name(formula, args: args),
+          version: {
+            current:             current.to_s,
+            latest:              latest.to_s,
+            outdated:            is_outdated,
+            newer_than_upstream: is_newer_than_upstream,
+          },
+          meta:    {
+            livecheckable: formula.livecheckable?,
+          },
+        }
+        info[:meta][:head_only] = true if formula.head?
+        info[:meta].merge!(version_info[:meta]) if version_info.present? && version_info.key?(:meta)
+
+        next if args.newer_only? && !info[:version][:outdated]
+
+        has_a_newer_upstream_version ||= true
+
+        if args.json?
+          info.except!(:meta) unless args.verbose?
+          next info
+        end
+
+        print_latest_version(info, args: args)
+        nil
+      rescue => e
+        Homebrew.failed = true
+
+        if args.json?
+          status_hash(formula, "error", [e.to_s], args: args)
+        elsif !args.quiet?
+          onoe "#{Tty.blue}#{formula_name(formula, args: args)}#{Tty.reset}: #{e}"
+          nil
+        end
+      end
+
+      if [args.newer_only?, !has_a_newer_upstream_version, !args.json?, !args.debug?].all?
+        puts "No newer upstream versions."
+      end
+
+      puts JSON.generate(formulae_checked.compact) if args.json?
+    end
+
+    def formula_name(formula, args:)
+      args.full_name? ? formula.full_name : formula.name
+    end
+
+    def status_hash(formula, status_str, messages = nil, args:)
+      status_hash = {
+        formula: formula_name(formula, args: args),
+        status:  status_str,
+      }
+      status_hash[:messages] = messages.presence
+
+      if args.verbose?
+        status_hash[:meta] = {
+          livecheckable: formula.livecheckable?,
+        }
+        status_hash[:meta][:head_only] = true if formula.head?
+      end
+
+      status_hash
+    end
+
+    def skip_conditions(formula, args:)
+      if formula.deprecated? && !formula.livecheckable?
+        return status_hash(formula, "deprecated", args: args) if args.json?
+
+        unless args.quiet?
+          puts "#{Tty.red}#{formula_name(formula, args: args)}#{Tty.reset} : deprecated"
+          return
+        end
+      end
+
+      if formula.versioned_formula? && !formula.livecheckable?
+        return status_hash(formula, "versioned", args: args) if args.json?
+
+        unless args.quiet?
+          puts "#{Tty.red}#{formula_name(formula, args: args)}#{Tty.reset} : versioned"
+          return
+        end
+      end
+
+      if formula.head? && !formula.any_version_installed?
+        head_only_msg = "HEAD only formula must be installed to be livecheckable"
+        return status_hash(formula, "error", [head_only_msg], args: args) if args.json?
+
+        unless args.quiet?
+          puts "#{Tty.red}#{formula_name(formula, args: args)}#{Tty.reset} : #{head_only_msg}"
+          return
+        end
+      end
+
+      is_gist = formula.stable&.url&.include?("gist.github.com")
+      if formula.livecheck.skip? || is_gist
+        skip_msg = if formula.livecheck.skip_msg.is_a?(String) &&
+                      formula.livecheck.skip_msg.present?
+          formula.livecheck.skip_msg.to_s
+        elsif is_gist
+          "Stable URL is a GitHub Gist"
+        else
+          ""
+        end
+
+        return status_hash(formula, "skipped", (skip_msg.blank? ? nil : [skip_msg]), args: args) if args.json?
+
+        unless args.quiet?
+          puts "#{Tty.red}#{formula_name(formula, args: args)}#{Tty.reset} : skipped" \
+              "#{" - #{skip_msg}" if skip_msg.present?}"
+          return
+        end
+      end
+
+      false
+    end
+
+    def print_latest_version(info, args:)
+      formula_s = "#{Tty.blue}#{info[:formula]}#{Tty.reset}"
+      formula_s += " (guessed)" if !info[:meta][:livecheckable] && args.verbose?
+
+      current_s = if info[:version][:newer_than_upstream]
+        "#{Tty.red}#{info[:version][:current]}#{Tty.reset}"
+      else
+        info[:version][:current]
+      end
+
+      latest_s = if info[:version][:outdated]
+        "#{Tty.green}#{info[:version][:latest]}#{Tty.reset}"
+      else
+        info[:version][:latest]
+      end
+
+      puts "#{formula_s} : #{current_s} ==> #{latest_s}"
+    end
+
+    def checkable_urls(formula)
+      urls = []
+      urls << formula.head.url if formula.head
+      if formula.stable
+        urls << formula.stable.url
+        urls.concat(formula.stable.mirrors)
+      end
+      urls << formula.homepage if formula.homepage
+
+      urls.compact
+    end
+
+    def preprocess_url(url)
+      # Check for GitHub repos on github.com, not AWS
+      url.sub!("github.s3.amazonaws.com", "github.com") if url.include?("github")
+
+      # Use repo from GitHub or GitLab inferred from download URL
+      if url.include?("github.com") && GITHUB_SPECIAL_CASES.none? { |sc| url.include? sc }
+        if url.include? "archive"
+          url = url.sub(%r{/archive/.*}, ".git") if url.include? "github"
+        elsif url.include? "releases"
+          url = url.sub(%r{/releases/.*}, ".git")
+        elsif url.include? "downloads"
+          url = Pathname.new(url.sub(%r{/downloads(.*)}, "\\1")).dirname.to_s+".git"
+        elsif !url.end_with?(".git")
+          # Truncate the URL at the user/repo part, if possible
+          %r{(?<github_repo_url>(?:[a-z]+://)?github.com/[^/]+/[^/#]+)} =~ url
+          url = github_repo_url if github_repo_url.present?
+
+          url.delete_suffix!("/") if url.end_with?("/")
+          url += ".git"
+        end
+      elsif url.include?("/-/archive/")
+        url = url.sub(%r{/-/archive/.*$}i, ".git")
+      end
+
+      url
+    end
+
+    def latest_version(formula, args:)
+      has_livecheckable = formula.livecheckable?
+      livecheck = formula.livecheck
+      livecheck_regex = livecheck.regex
+      livecheck_strategy = livecheck.strategy
+      livecheck_url = livecheck.url
+
+      urls = [livecheck_url] if livecheck_url.present?
+      urls ||= checkable_urls(formula)
+
+      if args.debug?
+        puts
+        puts "Formula:          #{formula_name(formula, args: args)}"
+        puts "Head only?:       true" if formula.head?
+        puts "Livecheckable?:   #{has_livecheckable ? "Yes" : "No"}"
+      end
+
+      urls.each_with_index do |original_url, i|
+        if args.debug?
+          puts
+          puts "URL:              #{original_url}"
+        end
+
+        # Skip Gists until/unless we create a method of identifying revisions
+        if original_url.include?("gist.github.com")
+          odebug "Skipping: GitHub Gists are not supported"
+          next
+        end
+
+        # Do not preprocess the URL when livecheck.strategy is set to :page_match
+        url = if livecheck_strategy == :page_match
+          original_url
+        else
+          preprocess_url(original_url)
+        end
+
+        strategies = Strategy.from_url(url, livecheck_regex.present?)
+        strategy = Strategy.from_symbol(livecheck_strategy)
+        strategy ||= strategies.first
+        strategy_name = @livecheck_strategy_names[strategy]
+
+        if args.debug?
+          puts "URL (processed):  #{url}" if url != original_url
+          if strategies.present? && args.verbose?
+            puts "Strategies:       #{strategies.map { |s| @livecheck_strategy_names[s] }.join(", ")}"
+          end
+          puts "Strategy:         #{strategy.blank? ? "None" : strategy_name}"
+          puts "Regex:            #{livecheck_regex.inspect}" if livecheck_regex.present?
+        end
+
+        if livecheck_strategy == :page_match && livecheck_regex.blank?
+          odebug "#{strategy_name} strategy requires a regex"
+          next
+        end
+
+        if livecheck_strategy.present? && !strategies.include?(strategy)
+          odebug "#{strategy_name} strategy does not apply to this URL"
+          next
+        end
+
+        next if strategy.blank?
+
+        strategy_data = strategy.find_versions(url, livecheck_regex)
+        match_version_map = strategy_data[:matches]
+        regex = strategy_data[:regex]
+
+        if strategy_data[:messages].is_a?(Array) && match_version_map.blank?
+          puts strategy_data[:messages] unless args.json?
+          next if i + 1 < urls.length
+
+          return status_hash(formula, "error", strategy_data[:messages], args: args)
+        end
+
+        if args.debug?
+          puts "URL (strategy):   #{strategy_data[:url]}" if strategy_data[:url] != url
+          puts "Regex (strategy): #{strategy_data[:regex].inspect}" if strategy_data[:regex] != livecheck_regex
+        end
+
+        match_version_map.delete_if do |_match, version|
+          next true if version.blank?
+          next false if has_livecheckable
+
+          UNSTABLE_VERSION_KEYWORDS.any? do |rejection|
+            version.to_s.include?(rejection)
+          end
+        end
+
+        if args.debug? && match_version_map.present?
+          puts
+          puts "Matched Versions:"
+
+          if args.verbose?
+            match_version_map.each do |match, version|
+              puts "#{match} => #{version.inspect}"
+            end
+          else
+            puts match_version_map.values.join(", ")
+          end
+        end
+
+        next if match_version_map.blank?
+
+        version_info = {
+          latest: Version.new(match_version_map.values.max),
+        }
+
+        if args.json? && args.verbose?
+          version_info[:meta] = {
+            url:      {
+              original: original_url,
+            },
+            strategy: strategy.blank? ? nil : strategy_name,
+          }
+          version_info[:meta][:url][:processed] = url if url != original_url
+          version_info[:meta][:url][:strategy] = strategy_data[:url] if strategy_data[:url] != url
+          if strategies.present?
+            version_info[:meta][:strategies] = strategies.map { |s| @livecheck_strategy_names[s] }
+          end
+          version_info[:meta][:regex] = regex.inspect if regex.present?
+        end
+
+        return version_info
+      end
+
+      nil
+    end
+  end
+end

Library/Homebrew/test/.rubocop_todo.yml

@@ -200,6 +200,7 @@ RSpec/VerifiedDoubles:
     - 'formula_spec.rb'
     - 'language/python_spec.rb'
     - 'linkage_cache_store_spec.rb'
+    - 'livecheck/livecheck_spec.rb'
     - 'resource_spec.rb'
     - 'software_spec_spec.rb'
     - 'support/helper/formula.rb'

Library/Homebrew/test/livecheck/livecheck_spec.rb

+# Frozen_string_literal: true
+
+require "livecheck/livecheck"
+
+describe Homebrew::Livecheck do
+  subject(:livecheck) { described_class }
+
+  let(:f) do
+    formula("test") do
+      desc "Test formula"
+      homepage "https://brew.sh"
+      url "https://brew.sh/test-0.0.1.tgz"
+      head "https://github.com/Homebrew/brew.git"
+
+      livecheck do
+        url "https://github.s3.amazonaws.com/Homebrew/brew/releases/latest"
+        regex(%r{href=.*?/tag/v?(\d+(?:\.\d+)+)["' >]}i)
+      end
+    end
+  end
+
+  let(:f_deprecated) do
+    formula("test_deprecated") do
+      desc "Deprecated test formula"
+      homepage "https://brew.sh"
+      url "https://brew.sh/test-0.0.1.tgz"
+      deprecate!
+    end
+  end
+
+  let(:f_gist) do
+    formula("test_gist") do
+      desc "Gist test formula"
+      homepage "https://brew.sh"
+      url "https://gist.github.com/Homebrew/0000000000"
+    end
+  end
+
+  let(:f_head_only) do
+    formula("test_head_only") do
+      desc "HEAD-only test formula"
+      homepage "https://brew.sh"
+      head "https://github.com/Homebrew/brew.git"
+    end
+  end
+
+  let(:f_skip) do
+    formula("test_skip") do
+      desc "Skipped test formula"
+      homepage "https://brew.sh"
+      url "https://brew.sh/test-0.0.1.tgz"
+
+      livecheck do
+        skip "Not maintained"
+      end
+    end
+  end
+
+  let(:f_versioned) do
+    formula("test@0.0.1") do
+      desc "Versioned test formula"
+      homepage "https://brew.sh"
+      url "https://brew.sh/test-0.0.1.tgz"
+    end
+  end
+
+  let(:args) { double("livecheck_args", full_name?: false, json?: false, quiet?: false, verbose?: true) }
+
+  describe "::formula_name" do
+    it "returns the name of the formula" do
+      expect(livecheck.formula_name(f, args: args)).to eq("test")
+    end
+
+    it "returns the full name" do
+      allow(args).to receive(:full_name?).and_return(true)
+
+      expect(livecheck.formula_name(f, args: args)).to eq("test")
+    end
+  end
+
+  describe "::status_hash" do
+    it "returns a hash containing the livecheck status" do
+      expect(livecheck.status_hash(f, "blah", ["blah"], args: args))
+        .to eq({
+                 formula:  "test",
+                 status:   "blah",
+                 messages: ["blah"],
+                 meta:     {
+                   livecheckable: true,
+                 },
+               })
+    end
+  end
+
+  describe "::skip_conditions" do
+    it "skips a deprecated formula without a livecheckable" do
+      expect { livecheck.skip_conditions(f_deprecated, args: args) }
+        .to output("test_deprecated : deprecated\n").to_stdout
+        .and not_to_output.to_stderr
+    end
+
+    it "skips a versioned formula without a livecheckable" do
+      expect { livecheck.skip_conditions(f_versioned, args: args) }
+        .to output("test@0.0.1 : versioned\n").to_stdout
+        .and not_to_output.to_stderr
+    end
+
+    it "skips a HEAD-only formula if not installed" do
+      expect { livecheck.skip_conditions(f_head_only, args: args) }
+        .to output("test_head_only : HEAD only formula must be installed to be livecheckable\n").to_stdout
+        .and not_to_output.to_stderr
+    end
+
+    it "skips a formula with a GitHub Gist stable URL" do
+      expect { livecheck.skip_conditions(f_gist, args: args) }
+        .to output("test_gist : skipped - Stable URL is a GitHub Gist\n").to_stdout
+        .and not_to_output.to_stderr
+    end
+
+    it "skips a formula with a skip livecheckable" do
+      expect { livecheck.skip_conditions(f_skip, args: args) }
+        .to output("test_skip : skipped - Not maintained\n").to_stdout
+        .and not_to_output.to_stderr
+    end
+
+    it "returns false for a non-skippable formula" do
+      expect(livecheck.skip_conditions(f, args: args)).to eq(false)
+    end
+  end
+
+  describe "::checkable_urls" do
+    it "returns the list of URLs to check" do
+      expect(livecheck.checkable_urls(f))
+        .to eq(
+          ["https://github.com/Homebrew/brew.git", "https://brew.sh/test-0.0.1.tgz", "https://brew.sh"],
+        )
+    end
+  end
+
+  describe "::livecheck_formulae", :needs_network do
+    it "checks for the latest versions of the formulae" do
+      allow(args).to receive(:debug?).and_return(true)
+
+      expect(livecheck.livecheck_formulae([f], args))
+        .to eq(/Formula/)
+    end
+  end
+end