download_strategy.rb 31.8 KB
Newer Older
1
# typed: false
2
3
# frozen_string_literal: true

4
require "json"
5
require "time"
6
require "unpack_strategy"
7
8
require "lazy_object"
require "cgi"
9
require "lock_file"
10

11
12
13
require "mechanize/version"
require "mechanize/http/content_disposition_parser"

14
15
require "utils/curl"

Markus Reiter's avatar
Markus Reiter committed
16
17
18
# @abstract Abstract superclass for all download strategies.
#
# @api private
19
class AbstractDownloadStrategy
Markus Reiter's avatar
Markus Reiter committed
20
21
  extend T::Sig

22
  extend Forwardable
23
  include FileUtils
24
  include Context
25

Markus Reiter's avatar
Markus Reiter committed
26
27
28
  # Extension for bottle downloads.
  #
  # @api private
29
30
  module Pourable
    def stage
Markus Reiter's avatar
Markus Reiter committed
31
      ohai "Pouring #{basename}"
32
33
34
35
      super
    end
  end

36
  attr_reader :cache, :cached_location, :url, :meta, :name, :version
37

38
  private :meta, :name, :version
39

40
  def initialize(url, name, version, **meta)
41
    @url = url
42
    @name = name
43
    @version = version
44
    @cache = meta.fetch(:cache, HOMEBREW_CACHE)
45
    @meta = meta
46
    @quiet = false
47
    extend Pourable if meta[:bottle]
48
  end
49

Markus Reiter's avatar
Markus Reiter committed
50
51
52
  # Download and cache the resource at {#cached_location}.
  #
  # @api public
53
  def fetch; end
54

Markus Reiter's avatar
Markus Reiter committed
55
56
  # Disable any output during downloading.
  #
Markus Reiter's avatar
Markus Reiter committed
57
  # TODO: Deprecate once we have an explicitly documented alternative.
Markus Reiter's avatar
Markus Reiter committed
58
59
  #
  # @api public
Markus Reiter's avatar
Markus Reiter committed
60
  sig { void }
Markus Reiter's avatar
Markus Reiter committed
61
62
63
64
  def shutup!
    @quiet = true
  end

65
66
67
68
  def quiet?
    Context.current.quiet? || @quiet
  end

69
70
71
  # Unpack {#cached_location} into the current working directory, and possibly
  # chdir into the newly-unpacked directory.
  # Unlike {Resource#stage}, this does not take a block.
Markus Reiter's avatar
Markus Reiter committed
72
73
  #
  # @api public
74
  def stage
75
    UnpackStrategy.detect(cached_location,
76
                          prioritise_extension: true,
77
                          ref_type: @ref_type, ref: @ref)
78
79
                  .extract_nestedly(basename:             basename,
                                    prioritise_extension: true,
80
                                    verbose:              verbose? && !quiet?)
Markus Reiter's avatar
Markus Reiter committed
81
82
83
84
85
    chdir
  end

  def chdir
    entries = Dir["*"]
86
87
88
89
90
91
92
    raise "Empty archive" if entries.length.zero?
    return if entries.length != 1

    begin
      Dir.chdir entries.first
    rescue
      nil
Markus Reiter's avatar
Markus Reiter committed
93
    end
94
  end
Markus Reiter's avatar
Markus Reiter committed
95
  private :chdir
96

97
98
  # @!attribute [r] source_modified_time
  # Returns the most recent modified time for all files in the current working directory after stage.
Markus Reiter's avatar
Markus Reiter committed
99
100
  #
  # @api public
101
102
103
104
  def source_modified_time
    Pathname.pwd.to_enum(:find).select(&:file?).map(&:mtime).max
  end

105
106
  # Remove {#cached_location} and any other files associated with the resource
  # from the cache.
Markus Reiter's avatar
Markus Reiter committed
107
108
  #
  # @api public
109
  def clear_cache
110
    rm_rf(cached_location)
111
112
  end

113
  def basename
Markus Reiter's avatar
Markus Reiter committed
114
    cached_location.basename
115
  end
116
117
118

  private

Markus Reiter's avatar
Markus Reiter committed
119
120
121
122
123
124
125
126
  def puts(*args)
    super(*args) unless quiet?
  end

  def ohai(*args)
    super(*args) unless quiet?
  end

127
128
  def silent_command(*args, **options)
    system_command(*args, print_stderr: false, env: env, **options)
129
130
  end

131
132
  def command!(*args, **options)
    system_command!(
133
      *args,
134
135
136
137
138
139
140
141
      env: env.merge(options.fetch(:env, {})),
      **command_output_options,
      **options,
    )
  end

  def command_output_options
    {
142
143
144
      print_stdout: !quiet?,
      print_stderr: !quiet?,
      verbose:      verbose? && !quiet?,
145
    }
146
147
148
149
150
  end

  def env
    {}
  end
151
152
end

Markus Reiter's avatar
Markus Reiter committed
153
154
155
# @abstract Abstract superclass for all download strategies downloading from a version control system.
#
# @api private
156
class VCSDownloadStrategy < AbstractDownloadStrategy
157
  REF_TYPES = [:tag, :branch, :revisions, :revision].freeze
158

159
  def initialize(url, name, version, **meta)
160
    super
161
    @ref_type, @ref = extract_ref(meta)
162
    @revision = meta[:revision]
163
    @cached_location = @cache/"#{name}--#{cache_tag}"
164
165
  end

Markus Reiter's avatar
Markus Reiter committed
166
167
168
  # Download and cache the repository at {#cached_location}.
  #
  # @api public
169
  def fetch
170
    ohai "Cloning #{url}"
171
172
173
174
175
176
177
178
179
180
181

    if cached_location.exist? && repo_valid?
      puts "Updating #{cached_location}"
      update
    elsif cached_location.exist?
      puts "Removing invalid repository from cache"
      clear_cache
      clone_repo
    else
      clone_repo
    end
182

183
184
    version.update_commit(last_commit) if head?

Markus Reiter's avatar
Markus Reiter committed
185
186
187
    return unless @ref_type == :tag
    return unless @revision && current_revision
    return if current_revision == @revision
Markus Reiter's avatar
Markus Reiter committed
188

Markus Reiter's avatar
Markus Reiter committed
189
    raise <<~EOS
Markus Reiter's avatar
Markus Reiter committed
190
191
192
      #{@ref} tag should be #{@revision}
      but is actually #{current_revision}
    EOS
193
194
  end

195
196
197
198
199
200
201
202
203
204
  def fetch_last_commit
    fetch
    last_commit
  end

  def commit_outdated?(commit)
    @last_commit ||= fetch_last_commit
    commit != @last_commit
  end

205
206
207
  def head?
    version.respond_to?(:head?) && version.head?
  end
208

209
210
  # Return last commit's unique identifier for the repository.
  # Return most recent modified timestamp unless overridden.
Markus Reiter's avatar
Markus Reiter committed
211
212
  #
  # @api public
213
214
215
216
  def last_commit
    source_modified_time.to_i.to_s
  end

217
218
219
  private

  def cache_tag
220
    raise NotImplementedError
221
222
  end

223
  def repo_valid?
224
    raise NotImplementedError
225
226
  end

227
  def clone_repo; end
228

229
  def update; end
230

231
  def current_revision; end
232

233
234
  def extract_ref(specs)
    key = REF_TYPES.find { |type| specs.key?(type) }
BrewTestBot's avatar
BrewTestBot committed
235
    [key, specs[key]]
236
  end
237
238
end

Markus Reiter's avatar
Markus Reiter committed
239
240
241
# @abstract Abstract superclass for all download strategies downloading a single file.
#
# @api private
242
class AbstractFileDownloadStrategy < AbstractDownloadStrategy
Markus Reiter's avatar
Markus Reiter committed
243
244
245
  # Path for storing an incomplete download while the download is still in progress.
  #
  # @api public
246
247
  def temporary_path
    @temporary_path ||= Pathname.new("#{cached_location}.incomplete")
248
249
  end

Markus Reiter's avatar
Markus Reiter committed
250
251
252
253
  # Path of the symlink (whose name includes the resource name, version and extension)
  # pointing to {#cached_location}.
  #
  # @api public
254
255
  def symlink_location
    return @symlink_location if defined?(@symlink_location)
Markus Reiter's avatar
Markus Reiter committed
256

257
258
259
260
    ext = Pathname(parse_basename(url)).extname
    @symlink_location = @cache/"#{name}--#{version}#{ext}"
  end

261
  # Path for storing the completed download.
Markus Reiter's avatar
Markus Reiter committed
262
263
  #
  # @api public
264
265
266
267
268
  def cached_location
    return @cached_location if defined?(@cached_location)

    url_sha256 = Digest::SHA256.hexdigest(url)
    downloads = Pathname.glob(HOMEBREW_CACHE/"downloads/#{url_sha256}--*")
269
                        .reject { |path| path.extname.end_with?(".incomplete") }
270
271
272
273
274
275
276
277
278

    @cached_location = if downloads.count == 1
      downloads.first
    else
      HOMEBREW_CACHE/"downloads/#{url_sha256}--#{resolved_basename}"
    end
  end

  def basename
Mike McQuaid's avatar
Mike McQuaid committed
279
    cached_location.basename.sub(/^[\da-f]{64}--/, "")
280
281
  end

282
283
  private

284
285
286
287
288
289
290
291
292
293
294
295
  def resolved_url
    resolved_url, = resolved_url_and_basename
    resolved_url
  end

  def resolved_basename
    _, resolved_basename = resolved_url_and_basename
    resolved_basename
  end

  def resolved_url_and_basename
    return @resolved_url_and_basename if defined?(@resolved_url_and_basename)
Markus Reiter's avatar
Markus Reiter committed
296

297
298
299
300
    @resolved_url_and_basename = [url, parse_basename(url)]
  end

  def parse_basename(url)
301
    uri_path = if url.match?(URI::DEFAULT_PARSER.make_regexp)
302
303
304
305
306
307
308
309
310
311
      uri = URI(url)

      if uri.query
        query_params = CGI.parse(uri.query)
        query_params["response-content-disposition"].each do |param|
          query_basename = param[/attachment;\s*filename=(["']?)(.+)\1/i, 2]
          return query_basename if query_basename
        end
      end

Markus Reiter's avatar
Markus Reiter committed
312
313
      uri.query ? "#{uri.path}?#{uri.query}" : uri.path
    else
314
      url
Markus Reiter's avatar
Markus Reiter committed
315
316
    end

317
318
    uri_path = URI.decode_www_form_component(uri_path)

319
320
    # We need a Pathname because we've monkeypatched extname to support double
    # extensions (e.g. tar.gz).
321
322
    # Given a URL like https://example.com/download.php?file=foo-1.0.tar.gz
    # the basename we want is "foo-1.0.tar.gz", not "download.php".
Markus Reiter's avatar
Markus Reiter committed
323
324
    Pathname.new(uri_path).ascend do |path|
      ext = path.extname[/[^?&]+/]
325
      return path.basename.to_s[/[^?&]+#{Regexp.escape(ext)}/] if ext
326
    end
327
328

    File.basename(uri_path)
329
330
331
  end
end

Markus Reiter's avatar
Markus Reiter committed
332
333
334
# Strategy for downloading files using `curl`.
#
# @api public
335
class CurlDownloadStrategy < AbstractFileDownloadStrategy
336
337
  include Utils::Curl

338
  attr_reader :mirrors
339

340
  def initialize(url, name, version, **meta)
341
    super
342
    @mirrors = meta.fetch(:mirrors, [])
343
344
  end

Markus Reiter's avatar
Markus Reiter committed
345
346
347
  # Download and cache the file at {#cached_location}.
  #
  # @api public
348
  def fetch
349
350
351
    download_lock = LockFile.new(temporary_path.basename)
    download_lock.lock

352
    urls = [url, *mirrors]
353

354
355
356
357
358
    begin
      url = urls.shift

      ohai "Downloading #{url}"

359
360
361
362
      resolved_url, _, url_time = resolve_url_basename_time(url)

      fresh = if cached_location.exist? && url_time
        url_time <= cached_location.mtime
363
364
      elsif version.respond_to?(:latest?)
        !version.latest?
365
366
367
368
369
      else
        true
      end

      if cached_location.exist? && fresh
370
371
372
373
374
375
376
377
        puts "Already downloaded: #{cached_location}"
      else
        begin
          _fetch(url: url, resolved_url: resolved_url)
        rescue ErrorDuringExecution
          raise CurlDownloadStrategyError, url
        end
        ignore_interrupts do
378
          cached_location.dirname.mkpath
379
380
381
          temporary_path.rename(cached_location)
          symlink_location.dirname.mkpath
        end
382
      end
383
384

      FileUtils.ln_s cached_location.relative_path_from(symlink_location.dirname), symlink_location, force: true
385
386
    rescue CurlDownloadStrategyError
      raise if urls.empty?
Markus Reiter's avatar
Markus Reiter committed
387

388
389
      puts "Trying a mirror..."
      retry
390
    end
391
  ensure
392
    download_lock&.unlock
393
    download_lock&.path&.unlink
394
395
396
397
398
399
400
401
402
  end

  def clear_cache
    super
    rm_rf(temporary_path)
  end

  private

403
  def resolved_url_and_basename
404
    resolved_url, basename, = resolve_url_basename_time(url)
405
    [resolved_url, basename]
406
  end
407

408
  def resolve_url_basename_time(url)
409
410
411
    @resolved_info_cache ||= {}
    return @resolved_info_cache[url] if @resolved_info_cache.include?(url)

Mike McQuaid's avatar
Mike McQuaid committed
412
    if (domain = Homebrew::EnvConfig.artifact_domain)
Mike McQuaid's avatar
Mike McQuaid committed
413
      url = url.sub(%r{^((ht|f)tps?://)?}, "#{domain.chomp("/")}/")
414
415
    end

Markus Reiter's avatar
Markus Reiter committed
416
    out, _, status= curl_output("--location", "--silent", "--head", "--request", "GET", url.to_s)
417
418
419
420
421
422
423

    lines = status.success? ? out.lines.map(&:chomp) : []

    locations = lines.map { |line| line[/^Location:\s*(.*)$/i, 1] }
                     .compact

    redirect_url = locations.reduce(url) do |current_url, location|
424
425
426
427
      if location.start_with?("//")
        uri = URI(current_url)
        "#{uri.scheme}:#{location}"
      elsif location.start_with?("/")
428
429
        uri = URI(current_url)
        "#{uri.scheme}://#{uri.host}#{location}"
430
431
432
      elsif location.start_with?("./")
        uri = URI(current_url)
        "#{uri.scheme}://#{uri.host}#{Pathname(uri.path).dirname/location}"
433
434
435
436
437
      else
        location
      end
    end

438
439
440
    content_disposition_parser = Mechanize::HTTP::ContentDispositionParser.new

    parse_content_disposition = lambda do |line|
441
      next unless content_disposition = content_disposition_parser.parse(line.sub(/; *$/, ""), true)
442

443
444
      filename = nil

445
446
      if filename_with_encoding = content_disposition.parameters["filename*"]
        encoding, encoded_filename = filename_with_encoding.split("''", 2)
447
        filename = URI.decode_www_form_component(encoded_filename).encode(encoding) if encoding && encoded_filename
448
      end
449
450

      filename || content_disposition.filename
451
452
453
    end

    filenames = lines.map(&parse_content_disposition).compact
454

455
    time =
Mike McQuaid's avatar
Mike McQuaid committed
456
      lines.map { |line| line[/^Last-Modified:\s*(.+)/i, 1] }
457
           .compact
458
           .map { |t| t.match?(/^\d+$/) ? Time.at(t.to_i) : Time.parse(t) }
459
460
           .last

461
462
    basename = filenames.last || parse_basename(redirect_url)

463
    @resolved_info_cache[url] = [redirect_url, basename, time]
464
465
466
467
468
  end

  def _fetch(url:, resolved_url:)
    ohai "Downloading from #{resolved_url}" if url != resolved_url

Mike McQuaid's avatar
Mike McQuaid committed
469
    if Homebrew::EnvConfig.no_insecure_redirect? &&
470
471
472
473
474
475
       url.start_with?("https://") && !resolved_url.start_with?("https://")
      $stderr.puts "HTTPS to HTTP redirect detected & HOMEBREW_NO_INSECURE_REDIRECT is set."
      raise CurlDownloadStrategyError, url
    end

    curl_download resolved_url, to: temporary_path
476
477
  end

478
  # Curl options to be always passed to curl,
Markus Reiter's avatar
Markus Reiter committed
479
  # with raw head calls (`curl --head`) or with actual `fetch`.
480
481
482
  def _curl_args
    args = []

Markus Reiter's avatar
Markus Reiter committed
483
    args += ["-b", meta.fetch(:cookies).map { |k, v| "#{k}=#{v}" }.join(";")] if meta.key?(:cookies)
484
485
486
487
488

    args += ["-e", meta.fetch(:referer)] if meta.key?(:referer)

    args += ["--user", meta.fetch(:user)] if meta.key?(:user)

Chris Tompkinson's avatar
Chris Tompkinson committed
489
    args += [meta[:header], meta[:headers]].flatten.compact.flat_map { |h| ["--header", h.strip] }
490

491
492
493
    args
  end

494
  def _curl_opts
495
    return { user_agent: meta.fetch(:user_agent) } if meta.key?(:user_agent)
Markus Reiter's avatar
Markus Reiter committed
496

497
    {}
498
499
  end

500
501
502
503
  def curl_output(*args, **options)
    super(*_curl_args, *args, **_curl_opts, **options)
  end

Markus Reiter's avatar
Markus Reiter committed
504
  def curl(*args, **options)
505
    args << "--connect-timeout" << "15" unless mirrors.empty?
506
    super(*_curl_args, *args, **_curl_opts, **command_output_options, **options)
507
508
509
  end
end

Markus Reiter's avatar
Markus Reiter committed
510
511
512
# Strategy for downloading a file from an Apache Mirror URL.
#
# @api public
513
class CurlApacheMirrorDownloadStrategy < CurlDownloadStrategy
514
  def mirrors
515
516
517
518
519
520
    combined_mirrors
  end

  private

  def combined_mirrors
521
522
523
524
525
526
    return @combined_mirrors if defined?(@combined_mirrors)

    backup_mirrors = apache_mirrors.fetch("backup", [])
                                   .map { |mirror| "#{mirror}#{apache_mirrors["path_info"]}" }

    @combined_mirrors = [*@mirrors, *backup_mirrors]
527
528
  end

529
  def resolve_url_basename_time(url)
530
    if url == self.url
531
      super("#{apache_mirrors["preferred"]}#{apache_mirrors["path_info"]}")
532
533
534
    else
      super
    end
535
  end
536

537
538
  def apache_mirrors
    return @apache_mirrors if defined?(@apache_mirrors)
Markus Reiter's avatar
Markus Reiter committed
539

540
541
542
    json, = curl_output("--silent", "--location", "#{url}&asjson=1")
    @apache_mirrors = JSON.parse(json)
  rescue JSON::ParserError
543
    raise CurlDownloadStrategyError, "Couldn't determine mirror, try again later."
544
545
546
  end
end

Markus Reiter's avatar
Markus Reiter committed
547
# Strategy for downloading via an HTTP POST request using `curl`.
548
# Query parameters on the URL are converted into POST parameters.
Markus Reiter's avatar
Markus Reiter committed
549
550
#
# @api public
551
class CurlPostDownloadStrategy < CurlDownloadStrategy
552
553
554
  private

  def _fetch(url:, resolved_url:)
Markus Reiter's avatar
Markus Reiter committed
555
    args = if meta.key?(:data)
556
      escape_data = ->(d) { ["-d", URI.encode_www_form([d])] }
557
      [url, *meta[:data].flat_map(&escape_data)]
558
    else
559
      url, query = url.split("?", 2)
Markus Reiter's avatar
Markus Reiter committed
560
      query.nil? ? [url, "-X", "POST"] : [url, "-d", query]
561
562
    end

Markus Reiter's avatar
Markus Reiter committed
563
    curl_download(*args, to: temporary_path)
564
565
566
  end
end

Markus Reiter's avatar
Markus Reiter committed
567
568
569
570
# Strategy for downloading archives without automatically extracting them.
# (Useful for downloading `.jar` files.)
#
# @api public
571
class NoUnzipCurlDownloadStrategy < CurlDownloadStrategy
572
  def stage
573
    UnpackStrategy::Uncompressed.new(cached_location)
574
                                .extract(basename: basename,
575
                                         verbose:  verbose? && !quiet?)
576
577
578
  end
end

Markus Reiter's avatar
Markus Reiter committed
579
580
581
# Strategy for extracting local binary packages.
#
# @api private
582
class LocalBottleDownloadStrategy < AbstractFileDownloadStrategy
Mike McQuaid's avatar
Mike McQuaid committed
583
  def initialize(path) # rubocop:disable Lint/MissingSuper
584
    @cached_location = path
585
586
587
  end
end

Markus Reiter's avatar
Markus Reiter committed
588
589
590
# Strategy for downloading a Subversion repository.
#
# @api public
591
class SubversionDownloadStrategy < VCSDownloadStrategy
Markus Reiter's avatar
Markus Reiter committed
592
593
  extend T::Sig

594
  def initialize(url, name, version, **meta)
595
    super
Jack Nagel's avatar
Jack Nagel committed
596
    @url = @url.sub("svn+http://", "")
597
598
  end

Markus Reiter's avatar
Markus Reiter committed
599
600
601
  # Download and cache the repository at {#cached_location}.
  #
  # @api public
602
  def fetch
603
    if @url.chomp("/") != repo_url || !silent_command("svn", args: ["switch", @url, cached_location]).success?
604
605
      clear_cache
    end
606
607
    super
  end
608

Markus Reiter's avatar
Markus Reiter committed
609
  # (see AbstractDownloadStrategy#source_modified_time)
Markus Reiter's avatar
Markus Reiter committed
610
  sig { returns(Time) }
611
  def source_modified_time
Markus Reiter's avatar
Markus Reiter committed
612
    time = if Version.create(Utils::Svn.version) >= Version.create("1.9")
613
      out, = silent_command("svn", args: ["info", "--show-item", "last-changed-date"], chdir: cached_location)
614
615
      out
    else
616
      out, = silent_command("svn", args: ["info"], chdir: cached_location)
617
618
619
      out[/^Last Changed Date: (.+)$/, 1]
    end
    Time.parse time
620
621
  end

Markus Reiter's avatar
Markus Reiter committed
622
  # (see VCSDownloadStrategy#source_modified_time)
623
  def last_commit
624
    out, = silent_command("svn", args: ["info", "--show-item", "revision"], chdir: cached_location)
625
    out.strip
626
627
  end

628
629
  private

feu's avatar
feu committed
630
  def repo_url
631
    out, = silent_command("svn", args: ["info"], chdir: cached_location)
632
    out.strip[/^URL: (.+)$/, 1]
633
634
  end

635
  def externals
636
    out, = silent_command("svn", args: ["propget", "svn:externals", @url])
637
    out.chomp.split("\n").each do |line|
638
      name, url = line.split(/\s+/)
639
640
641
642
      yield name, url
    end
  end

Mike McQuaid's avatar
Mike McQuaid committed
643
  def fetch_repo(target, url, revision = nil, ignore_externals: false)
644
    # Use "svn update" when the repository already exists locally.
645
646
    # This saves on bandwidth and will have a similar effect to verifying the
    # cache as it will make any changes to get the right revision.
Melvyn Depeyrot's avatar
Melvyn Depeyrot committed
647
    args = []
648
    args << "--quiet" unless verbose?
649

650
    if revision
651
      ohai "Checking out #{@ref}"
652
653
      args << "-r" << revision
    end
654

BrewTestBot's avatar
BrewTestBot committed
655
    args << "--ignore-externals" if ignore_externals
656
657
658
659
660
661

    if meta[:trust_cert] == true
      args << "--trust-server-cert"
      args << "--non-interactive"
    end

662
    if target.directory?
663
      command!("svn", args: ["update", *args], chdir: target.to_s)
feu's avatar
feu committed
664
    else
665
      command!("svn", args: ["checkout", url, target, *args])
feu's avatar
feu committed
666
    end
667
  end
Jack Nagel's avatar
Jack Nagel committed
668

Markus Reiter's avatar
Markus Reiter committed
669
  sig { returns(String) }
Jack Nagel's avatar
Jack Nagel committed
670
671
672
  def cache_tag
    head? ? "svn-HEAD" : "svn"
  end
Jack Nagel's avatar
Jack Nagel committed
673
674

  def repo_valid?
675
    (cached_location/".svn").directory?
Jack Nagel's avatar
Jack Nagel committed
676
  end
677
678
679
680

  def clone_repo
    case @ref_type
    when :revision
681
      fetch_repo cached_location, @url, @ref
682
683
684
    when :revisions
      # nil is OK for main_revision, as fetch_repo will then get latest
      main_revision = @ref[:trunk]
Mike McQuaid's avatar
Mike McQuaid committed
685
      fetch_repo cached_location, @url, main_revision, ignore_externals: true
686

687
      externals do |external_name, external_url|
Mike McQuaid's avatar
Mike McQuaid committed
688
        fetch_repo cached_location/external_name, external_url, @ref[external_name], ignore_externals: true
689
690
      end
    else
691
      fetch_repo cached_location, @url
692
693
    end
  end
Markus Reiter's avatar
Markus Reiter committed
694
  alias update clone_repo
Adam Vandenberg's avatar
Adam Vandenberg committed
695
696
end

Markus Reiter's avatar
Markus Reiter committed
697
698
699
# Strategy for downloading a Git repository.
#
# @api public
700
class GitDownloadStrategy < VCSDownloadStrategy
701
  SHALLOW_CLONE_ALLOWLIST = [
702
703
704
    %r{git://},
    %r{https://github\.com},
    %r{http://git\.sv\.gnu\.org},
705
706
    %r{http://llvm\.org},
  ].freeze
707

708
  def initialize(url, name, version, **meta)
709
    super
710
711
    @ref_type ||= :branch
    @ref ||= "master"
Mike McQuaid's avatar
Mike McQuaid committed
712
    @shallow = meta.fetch(:shallow, true)
713
714
  end

Markus Reiter's avatar
Markus Reiter committed
715
  # (see AbstractDownloadStrategy#source_modified_time)
Markus Reiter's avatar
Markus Reiter committed
716
  sig { returns(Time) }
717
  def source_modified_time
718
    out, = silent_command("git", args: ["--git-dir", git_dir, "show", "-s", "--format=%cD"])
719
    Time.parse(out)
720
721
  end

Markus Reiter's avatar
Markus Reiter committed
722
  # (see VCSDownloadStrategy#source_modified_time)
723
  def last_commit
724
    out, = silent_command("git", args: ["--git-dir", git_dir, "rev-parse", "--short=7", "HEAD"])
725
    out.chomp
726
727
  end

728
729
  private

Markus Reiter's avatar
Markus Reiter committed
730
  sig { returns(String) }
Jack Nagel's avatar
Jack Nagel committed
731
732
733
734
  def cache_tag
    "git"
  end

Markus Reiter's avatar
Markus Reiter committed
735
  sig { returns(Integer) }
736
737
738
739
  def cache_version
    0
  end

740
  def update
741
742
743
744
745
    config_repo
    update_repo
    checkout
    reset
    update_submodules if submodules?
746
747
  end

748
749
750
751
  def shallow_clone?
    @shallow && support_depth?
  end

Markus Reiter's avatar
Markus Reiter committed
752
  def shallow_dir?
753
    (git_dir/"shallow").exist?
754
755
  end

756
  def support_depth?
757
    @ref_type != :revision && SHALLOW_CLONE_ALLOWLIST.any? { |regex| @url =~ regex }
758
759
  end

760
  def git_dir
761
    cached_location/".git"
762
763
  end

Markus Reiter's avatar
Markus Reiter committed
764
  def ref?
765
    silent_command("git",
766
                   args: ["--git-dir", git_dir, "rev-parse", "-q", "--verify", "#{@ref}^{commit}"])
767
      .success?
768
769
  end

770
  def current_revision
771
    out, = silent_command("git", args: ["--git-dir", git_dir, "rev-parse", "-q", "--verify", "HEAD"])
772
    out.strip
773
774
  end

775
  def repo_valid?
776
    silent_command("git", args: ["--git-dir", git_dir, "status", "-s"]).success?
777
778
  end

779
  def submodules?
780
    (cached_location/".gitmodules").exist?
781
782
  end

Markus Reiter's avatar
Markus Reiter committed
783
  sig { returns(T::Array[String]) }
784
  def clone_args
BrewTestBot's avatar
BrewTestBot committed
785
786
    args = %w[clone]
    args << "--depth" << "1" if shallow_clone?
787

788
    case @ref_type
Markus Reiter's avatar
Markus Reiter committed
789
790
    when :branch, :tag
      args << "--branch" << @ref
791
      args << "-c" << "advice.detachedHead=false" # silences detached head warning
792
793
    end

794
    args << @url << cached_location
795
796
  end

Markus Reiter's avatar
Markus Reiter committed
797
  sig { returns(String) }
798
  def refspec
799
    case @ref_type
BrewTestBot's avatar
BrewTestBot committed
800
801
    when :branch then "+refs/heads/#{@ref}:refs/remotes/origin/#{@ref}"
    when :tag    then "+refs/tags/#{@ref}:refs/tags/#{@ref}"
802
803
804
805
806
    else              "+refs/heads/master:refs/remotes/origin/master"
    end
  end

  def config_repo
807
808
809
810
811
812
813
814
815
    command! "git",
             args:  ["config", "remote.origin.url", @url],
             chdir: cached_location
    command! "git",
             args:  ["config", "remote.origin.fetch", refspec],
             chdir: cached_location
    command! "git",
             args:  ["config", "remote.origin.tagOpt", "--no-tags"],
             chdir: cached_location
816
817
  end

818
  def update_repo
Markus Reiter's avatar
Markus Reiter committed
819
820
821
    return unless @ref_type == :branch || !ref?

    if !shallow_clone? && shallow_dir?
822
823
824
      command! "git",
               args:  ["fetch", "origin", "--unshallow"],
               chdir: cached_location
Markus Reiter's avatar
Markus Reiter committed
825
    else
826
827
828
      command! "git",
               args:  ["fetch", "origin"],
               chdir: cached_location
829
    end
830
831
832
  end

  def clone_repo
833
    command! "git", args: clone_args
834

835
836
837
    command! "git",
             args:  ["config", "homebrew.cacheversion", cache_version],
             chdir: cached_location
838
839
    checkout
    update_submodules if submodules?
840
841
  end

842
  def checkout
843
    ohai "Checking out #{@ref_type} #{@ref}" if @ref_type && @ref
844
    command! "git", args: ["checkout", "-f", @ref, "--"], chdir: cached_location
845
  end
846

847
  def reset
848
    ref = case @ref_type
Markus Reiter's avatar
Markus Reiter committed
849
850
851
852
853
    when :branch
      "origin/#{@ref}"
    when :revision, :tag
      @ref
    end
854

855
856
857
    command! "git",
             args:  ["reset", "--hard", *ref, "--"],
             chdir: cached_location
858
859
  end

860
  def update_submodules
861
862
863
864
865
866
    command! "git",
             args:  ["submodule", "foreach", "--recursive", "git submodule sync"],
             chdir: cached_location
    command! "git",
             args:  ["submodule", "update", "--init", "--recursive"],
             chdir: cached_location
867
868
869
    fix_absolute_submodule_gitdir_references!
  end

870
871
872
873
874
875
876
  # When checking out Git repositories with recursive submodules, some Git
  # versions create `.git` files with absolute instead of relative `gitdir:`
  # pointers. This works for the cached location, but breaks various Git
  # operations once the affected Git resource is staged, i.e. recursively
  # copied to a new location. (This bug was introduced in Git 2.7.0 and fixed
  # in 2.8.3. Clones created with affected version remain broken.)