Skip to content
Snippets Groups Projects
Commit 8e0e1642 authored by Mike McQuaid's avatar Mike McQuaid Committed by GitHub
Browse files

Use `curl` for the GitHub API (#295) 

* Move GitHub API module to utils/github.rb.

* Move curl method to utils/curl.rb.

* global: use long curl arguments and an array.

This makes the code more self-documenting.

* utils/curl: support reading curl's output.

* utils/github: use curl instead of open-uri.

It has far better proxy support.

* pull: set Homebrew user agent.

* gist-logs: remove trailing whitespace.

* gist-logs: use first instead of [0].

Easier to read.

* gist-logs: use curl-based GitHub.open method.
parent b2c9625d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Library/Homebrew/cmd/gist-logs.rb
+ 13
− 77
View file @ 8e0e1642
......@@ -7,12 +7,10 @@
#: If `--new-issue` is passed, automatically create a new issue in the appropriate
#: GitHub repository as well as creating the Gist.
#:
#: If no logs are found, an error message is presented.
#: If no logs are found, an error message is presented.
require "formula"
require "system_config"
require "net/http"
require "net/https"
require "stringio"
require "socket"
......@@ -46,17 +44,14 @@ module Homebrew
url = create_gist(files, descr)
if ARGV.include?("--new-issue") || ARGV.switch?("n")
auth = :AUTH_TOKEN
if GitHub.api_credentials_type == :none
puts "You can create a personal access token: https://github.com/settings/tokens"
puts "and then set HOMEBREW_GITHUB_API_TOKEN as authentication method."
puts
auth = :AUTH_USER_LOGIN
login!
end
url = new_issue(f.tap, "#{f.name} failed to build on #{MacOS.full_version}", url, auth)
url = new_issue(f.tap, "#{f.name} failed to build on #{MacOS.full_version}", url)
end
puts url if url
......@@ -84,13 +79,12 @@ module Homebrew
result
end
def login(request)
def login!
print "GitHub User: "
user = $stdin.gets.chomp
ENV["HOMEBREW_GITHUB_API_USERNAME"] = $stdin.gets.chomp
print "Password: "
password = noecho_gets.chomp
ENV["HOMEBREW_GITHUB_API_PASSWORD"] = noecho_gets.chomp
puts
request.basic_auth(user, password)
end
def load_logs(dir)
......@@ -103,77 +97,19 @@ module Homebrew
logs
end
def create_gist(files, descr)
post("/gists", { "public" => true, "files" => files, "description" => descr })["html_url"]
end
def new_issue(repo, title, body, auth)
post("/repos/#{repo}/issues", { "title" => title, "body" => body }, auth)["html_url"]
end
def http
@http ||= begin
uri = URI.parse("https://api.github.com")
p = ENV["http_proxy"] ? URI.parse(ENV["http_proxy"]) : nil
if p.class == URI::HTTP || p.class == URI::HTTPS
@http = Net::HTTP.new(uri.host, uri.port, p.host, p.port, p.user, p.password)
else
@http = Net::HTTP.new(uri.host, uri.port)
end
@http.use_ssl = true
@http
end
def create_gist(files, description)
data = { "public" => true, "files" => files, "description" => description }
GitHub.open("https://api.github.com/gists", data)["html_url"]
end
def make_request(path, data, auth)
headers = GitHub.api_headers
headers["Content-Type"] = "application/json"
basic_auth_credentials = nil
if auth != :AUTH_USER_LOGIN
token, username = GitHub.api_credentials
case GitHub.api_credentials_type
when :keychain
basic_auth_credentials = [username, token]
when :environment
headers["Authorization"] = "token #{token}"
end
end
request = Net::HTTP::Post.new(path, headers)
request.basic_auth(*basic_auth_credentials) if basic_auth_credentials
login(request) if auth == :AUTH_USER_LOGIN
request.body = Utils::JSON.dump(data)
request
end
def post(path, data, auth = nil)
request = make_request(path, data, auth)
case response = http.request(request)
when Net::HTTPCreated
Utils::JSON.load get_body(response)
else
GitHub.api_credentials_error_message(response)
raise "HTTP #{response.code} #{response.message} (expected 201)"
end
end
def get_body(response)
if !response.body.respond_to?(:force_encoding)
response.body
elsif response["Content-Type"].downcase == "application/json; charset=utf-8"
response.body.dup.force_encoding(Encoding::UTF_8)
else
response.body.encode(Encoding::UTF_8, :undef => :replace)
end
def new_issue(repo, title, body)
data = { "title" => title, "body" => body }
GitHub.open("https://api.github.com/repos/MikeMcQuaid/test/issues", data)["html_url"]
end
def gist_logs
raise FormulaUnspecifiedError if ARGV.resolved_formulae.length != 1
gistify_logs(ARGV.resolved_formulae[0])
gistify_logs(ARGV.resolved_formulae.first)
end
end
Library/Homebrew/cmd/pull.rb
+ 1
− 0
View file @ 8e0e1642
......@@ -503,6 +503,7 @@ module Homebrew
url = URI(bottle_info.url)
puts "Verifying bottle: #{File.basename(url.path)}"
http = Net::HTTP.new(url.host, url.port)
http.initialize_http_header "User-Agent" => HOMEBREW_USER_AGENT_RUBY
http.use_ssl = true
retry_count = 0
http.start do
......
Library/Homebrew/global.rb
+ 7
− 1
View file @ 8e0e1642
......@@ -30,7 +30,13 @@ RUBY_BIN = RUBY_PATH.dirname
HOMEBREW_USER_AGENT_CURL = ENV["HOMEBREW_USER_AGENT_CURL"]
HOMEBREW_USER_AGENT_RUBY = "#{ENV["HOMEBREW_USER_AGENT"]} ruby/#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}"
HOMEBREW_CURL_ARGS = "-f#RLA"
HOMEBREW_CURL_ARGS = [
"--fail",
"--progress-bar",
"--remote-time",
"--location",
"--user-agent", HOMEBREW_USER_AGENT_CURL
].freeze
require "tap_constants"
......
Library/Homebrew/utils.rb
+ 2
− 240
View file @ 8e0e1642
......@@ -7,7 +7,8 @@ require "utils/popen"
require "utils/fork"
require "utils/git"
require "utils/analytics"
require "open-uri"
require "utils/github"
require "utils/curl"
class Tty
class << self
......@@ -343,21 +344,6 @@ def quiet_system(cmd, *args)
end
end
def curl(*args)
curl = Pathname.new ENV["HOMEBREW_CURL"]
curl = Pathname.new "/usr/bin/curl" unless curl.exist?
raise "#{curl} is not executable" unless curl.exist? && curl.executable?
flags = HOMEBREW_CURL_ARGS
flags = flags.delete("#") if ARGV.verbose?
args = [flags, HOMEBREW_USER_AGENT_CURL, *args]
args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"]
args << "--silent" if !$stdout.tty? || ENV["TRAVIS"]
safe_system curl, *args
end
def puts_columns(items)
return if items.empty?
......@@ -519,230 +505,6 @@ def shell_profile
end
end
module GitHub
extend self
ISSUES_URI = URI.parse("https://api.github.com/search/issues")
Error = Class.new(RuntimeError)
HTTPNotFoundError = Class.new(Error)
class RateLimitExceededError < Error
def initialize(reset, error)
super <<-EOS.undent
GitHub API Error: #{error}
Try again in #{pretty_ratelimit_reset(reset)}, or create a personal access token:
#{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset}
and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token"
EOS
end
def pretty_ratelimit_reset(reset)
pretty_duration(Time.at(reset) - Time.now)
end
end
class AuthenticationFailedError < Error
def initialize(error)
message = "GitHub #{error}\n"
if ENV["HOMEBREW_GITHUB_API_TOKEN"]
message << <<-EOS.undent
HOMEBREW_GITHUB_API_TOKEN may be invalid or expired; check:
#{Tty.em}https://github.com/settings/tokens#{Tty.reset}
EOS
else
message << <<-EOS.undent
The GitHub credentials in the OS X keychain may be invalid.
Clear them with:
printf "protocol=https\\nhost=github.com\\n" | git credential-osxkeychain erase
Or create a personal access token:
#{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset}
and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token"
EOS
end
super message
end
end
def api_credentials
@api_credentials ||= begin
if ENV["HOMEBREW_GITHUB_API_TOKEN"]
ENV["HOMEBREW_GITHUB_API_TOKEN"]
else
github_credentials = Utils.popen("git credential-osxkeychain get", "w+") do |io|
io.puts "protocol=https\nhost=github.com"
io.close_write
io.read
end
github_username = github_credentials[/username=(.+)/, 1]
github_password = github_credentials[/password=(.+)/, 1]
if github_username && github_password
[github_password, github_username]
else
[]
end
end
end
end
def api_credentials_type
token, username = api_credentials
if token && !token.empty?
if username && !username.empty?
:keychain
else
:environment
end
else
:none
end
end
def api_credentials_error_message(response_headers)
@api_credentials_error_message_printed ||= begin
unauthorized = (response_headers["status"] == "401 Unauthorized")
scopes = response_headers["x-accepted-oauth-scopes"].to_s.split(", ")
if !unauthorized && scopes.empty?
credentials_scopes = response_headers["x-oauth-scopes"].to_s.split(", ")
case GitHub.api_credentials_type
when :keychain
onoe <<-EOS.undent
Your OS X keychain GitHub credentials do not have sufficient scope!
Scopes they have: #{credentials_scopes}
Create a personal access token: https://github.com/settings/tokens
and then set HOMEBREW_GITHUB_API_TOKEN as the authentication method instead.
EOS
when :environment
onoe <<-EOS.undent
Your HOMEBREW_GITHUB_API_TOKEN does not have sufficient scope!
Scopes it has: #{credentials_scopes}
Create a new personal access token: https://github.com/settings/tokens
and then set the new HOMEBREW_GITHUB_API_TOKEN as the authentication method instead.
EOS
end
end
true
end
end
def api_headers
{
"User-Agent" => HOMEBREW_USER_AGENT_RUBY,
"Accept" => "application/vnd.github.v3+json"
}
end
def open(url, &_block)
# This is a no-op if the user is opting out of using the GitHub API.
return if ENV["HOMEBREW_NO_GITHUB_API"]
require "net/https"
headers = api_headers
token, username = api_credentials
case api_credentials_type
when :keychain
headers[:http_basic_authentication] = [username, token]
when :environment
headers["Authorization"] = "token #{token}"
end
begin
Kernel.open(url, headers) { |f| yield Utils::JSON.load(f.read) }
rescue OpenURI::HTTPError => e
handle_api_error(e)
rescue EOFError, SocketError, OpenSSL::SSL::SSLError => e
raise Error, "Failed to connect to: #{url}\n#{e.message}", e.backtrace
rescue Utils::JSON::Error => e
raise Error, "Failed to parse JSON response\n#{e.message}", e.backtrace
end
end
def handle_api_error(e)
if e.io.meta.fetch("x-ratelimit-remaining", 1).to_i <= 0
reset = e.io.meta.fetch("x-ratelimit-reset").to_i
error = Utils::JSON.load(e.io.read)["message"]
raise RateLimitExceededError.new(reset, error)
end
GitHub.api_credentials_error_message(e.io.meta)
case e.io.status.first
when "401", "403"
raise AuthenticationFailedError.new(e.message)
when "404"
raise HTTPNotFoundError, e.message, e.backtrace
else
error = Utils::JSON.load(e.io.read)["message"] rescue nil
raise Error, [e.message, error].compact.join("\n"), e.backtrace
end
end
def issues_matching(query, qualifiers = {})
uri = ISSUES_URI.dup
uri.query = build_query_string(query, qualifiers)
open(uri) { |json| json["items"] }
end
def repository(user, repo)
open(URI.parse("https://api.github.com/repos/#{user}/#{repo}")) { |j| j }
end
def build_query_string(query, qualifiers)
s = "q=#{uri_escape(query)}+"
s << build_search_qualifier_string(qualifiers)
s << "&per_page=100"
end
def build_search_qualifier_string(qualifiers)
{
:repo => "Homebrew/homebrew-core",
:in => "title"
}.update(qualifiers).map do |qualifier, value|
"#{qualifier}:#{value}"
end.join("+")
end
def uri_escape(query)
if URI.respond_to?(:encode_www_form_component)
URI.encode_www_form_component(query)
else
require "erb"
ERB::Util.url_encode(query)
end
end
def issues_for_formula(name, options = {})
tap = options[:tap] || CoreTap.instance
issues_matching(name, :state => "open", :repo => "#{tap.user}/homebrew-#{tap.repo}")
end
def print_pull_requests_matching(query)
return [] if ENV["HOMEBREW_NO_GITHUB_API"]
ohai "Searching pull requests..."
open_or_closed_prs = issues_matching(query, :type => "pr")
open_prs = open_or_closed_prs.select { |i| i["state"] == "open" }
if open_prs.any?
puts "Open pull requests:"
prs = open_prs
elsif open_or_closed_prs.any?
puts "Closed pull requests:"
prs = open_or_closed_prs
else
return
end
prs.each { |i| puts "#{i["title"]} (#{i["html_url"]})" }
end
def private_repo?(user, repo)
uri = URI.parse("https://api.github.com/repos/#{user}/#{repo}")
open(uri) { |json| json["private"] }
end
end
def disk_usage_readable(size_in_bytes)
if size_in_bytes >= 1_073_741_824
size = size_in_bytes.to_f / 1_073_741_824
......
Library/Homebrew/utils/curl.rb 0 → 100644
+ 24
− 0
View file @ 8e0e1642
require "pathname"
def curl_args(extra_args=[])
curl = Pathname.new ENV["HOMEBREW_CURL"]
curl = Pathname.new "/usr/bin/curl" unless curl.exist?
raise "#{curl} is not executable" unless curl.exist? && curl.executable?
flags = HOMEBREW_CURL_ARGS
flags -= ["--progress-bar"] if ARGV.verbose?
args = ["#{curl}"] + flags + extra_args
args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"]
args << "--silent" if !$stdout.tty? || ENV["TRAVIS"]
args
end
def curl(*args)
safe_system(*curl_args(args))
end
def curl_output(*args)
curl_args = curl_args(args) - ["--fail"]
Utils.popen_read_text(*curl_args)
end
Library/Homebrew/utils/github.rb 0 → 100644
+ 267
− 0
View file @ 8e0e1642
require "uri"
require "tempfile"
module GitHub
extend self
ISSUES_URI = URI.parse("https://api.github.com/search/issues")
Error = Class.new(RuntimeError)
HTTPNotFoundError = Class.new(Error)
class RateLimitExceededError < Error
def initialize(reset, error)
super <<-EOS.undent
GitHub API Error: #{error}
Try again in #{pretty_ratelimit_reset(reset)}, or create a personal access token:
#{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset}
and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token"
EOS
end
def pretty_ratelimit_reset(reset)
pretty_duration(Time.at(reset) - Time.now)
end
end
class AuthenticationFailedError < Error
def initialize(error)
message = "GitHub #{error}\n"
if ENV["HOMEBREW_GITHUB_API_TOKEN"]
message << <<-EOS.undent
HOMEBREW_GITHUB_API_TOKEN may be invalid or expired; check:
#{Tty.em}https://github.com/settings/tokens#{Tty.reset}
EOS
else
message << <<-EOS.undent
The GitHub credentials in the OS X keychain may be invalid.
Clear them with:
printf "protocol=https\\nhost=github.com\\n" | git credential-osxkeychain erase
Or create a personal access token:
#{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset}
and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token"
EOS
end
super message
end
end
def api_credentials
@api_credentials ||= begin
if ENV["HOMEBREW_GITHUB_API_TOKEN"]
ENV["HOMEBREW_GITHUB_API_TOKEN"]
elsif ENV["HOMEBREW_GITHUB_API_USERNAME"] && ENV["HOMEBREW_GITHUB_API_PASSWORD"]
[ENV["HOMEBREW_GITHUB_API_USERNAME"], ENV["HOMEBREW_GITHUB_API_PASSWORD"]]
else
github_credentials = Utils.popen("git credential-osxkeychain get", "w+") do |io|
io.puts "protocol=https\nhost=github.com"
io.close_write
io.read
end
github_username = github_credentials[/username=(.+)/, 1]
github_password = github_credentials[/password=(.+)/, 1]
if github_username && github_password
[github_password, github_username]
else
[]
end
end
end
end
def api_credentials_type
token, username = api_credentials
if token && !token.empty?
if username && !username.empty?
:keychain
else
:environment
end
else
:none
end
end
def api_credentials_error_message(response_headers)
return if response_headers.empty?
@api_credentials_error_message_printed ||= begin
unauthorized = (response_headers["http/1.1"] == "401 Unauthorized")
scopes = response_headers["x-accepted-oauth-scopes"].to_s.split(", ")
if !unauthorized && scopes.empty?
credentials_scopes = response_headers["x-oauth-scopes"].to_s.split(", ")
case GitHub.api_credentials_type
when :keychain
onoe <<-EOS.undent
Your OS X keychain GitHub credentials do not have sufficient scope!
Scopes they have: #{credentials_scopes}
Create a personal access token: https://github.com/settings/tokens
and then set HOMEBREW_GITHUB_API_TOKEN as the authentication method instead.
EOS
when :environment
onoe <<-EOS.undent
Your HOMEBREW_GITHUB_API_TOKEN does not have sufficient scope!
Scopes it has: #{credentials_scopes}
Create a new personal access token: https://github.com/settings/tokens
and then set the new HOMEBREW_GITHUB_API_TOKEN as the authentication method instead.
EOS
end
end
true
end
end
def open(url, data=nil)
# This is a no-op if the user is opting out of using the GitHub API.
return if ENV["HOMEBREW_NO_GITHUB_API"]
args = %W[--header application/vnd.github.v3+json --write-out \n%{http_code}]
args += curl_args
token, username = api_credentials
case api_credentials_type
when :keychain
args += %W[--user #{username}:#{token}]
when :environment
args += ["--header", "Authorization: token #{token}"]
end
data_tmpfile = nil
if data
begin
data = Utils::JSON.dump data
data_tmpfile = Tempfile.new("github_api_post", HOMEBREW_TEMP)
rescue Utils::JSON::Error => e
raise Error, "Failed to parse JSON request:\n#{e.message}\n#{data}", e.backtrace
end
end
headers_tmpfile = Tempfile.new("github_api_headers", HOMEBREW_TEMP)
begin
if data
data_tmpfile.write data
args += ["--data", "@#{data_tmpfile.path}"]
end
args += ["--dump-header", "#{headers_tmpfile.path}"]
output, _, http_code = curl_output(url.to_s, *args).rpartition("\n")
output, _, http_code = output.rpartition("\n") if http_code == "000"
headers = headers_tmpfile.read
ensure
if data_tmpfile
data_tmpfile.close
data_tmpfile.unlink
end
headers_tmpfile.close
headers_tmpfile.unlink
end
begin
if !http_code.start_with?("2") && !$?.success?
raise_api_error(output, http_code, headers)
end
json = Utils::JSON.load output
if block_given?
yield json
else
json
end
rescue Utils::JSON::Error => e
raise Error, "Failed to parse JSON response\n#{e.message}", e.backtrace
end
end
def raise_api_error(output, http_code, headers)
meta = {}
headers.lines.each do |l|
key, _, value = l.delete(":").partition(" ")
key = key.downcase.strip
next if key.empty?
meta[key] = value.strip
end
if meta.fetch("x-ratelimit-remaining", 1).to_i <= 0
reset = meta.fetch("x-ratelimit-reset").to_i
error = Utils::JSON.load(output)["message"]
raise RateLimitExceededError.new(reset, error)
end
GitHub.api_credentials_error_message(meta)
case http_code
when "401", "403"
raise AuthenticationFailedError.new(output)
when "404"
raise HTTPNotFoundError, output
else
error = Utils::JSON.load(output)["message"] rescue nil
error ||= output
raise Error, error
end
end
def issues_matching(query, qualifiers = {})
uri = ISSUES_URI.dup
uri.query = build_query_string(query, qualifiers)
open(uri) { |json| json["items"] }
end
def repository(user, repo)
open(URI.parse("https://api.github.com/repos/#{user}/#{repo}")) { |j| j }
end
def build_query_string(query, qualifiers)
s = "q=#{uri_escape(query)}+"
s << build_search_qualifier_string(qualifiers)
s << "&per_page=100"
end
def build_search_qualifier_string(qualifiers)
{
:repo => "Homebrew/homebrew-core",
:in => "title"
}.update(qualifiers).map do |qualifier, value|
"#{qualifier}:#{value}"
end.join("+")
end
def uri_escape(query)
if URI.respond_to?(:encode_www_form_component)
URI.encode_www_form_component(query)
else
require "erb"
ERB::Util.url_encode(query)
end
end
def issues_for_formula(name, options = {})
tap = options[:tap] || CoreTap.instance
issues_matching(name, :state => "open", :repo => "#{tap.user}/homebrew-#{tap.repo}")
end
def print_pull_requests_matching(query)
return [] if ENV["HOMEBREW_NO_GITHUB_API"]
ohai "Searching pull requests..."
open_or_closed_prs = issues_matching(query, :type => "pr")
open_prs = open_or_closed_prs.select { |i| i["state"] == "open" }
if open_prs.any?
puts "Open pull requests:"
prs = open_prs
elsif open_or_closed_prs.any?
puts "Closed pull requests:"
prs = open_or_closed_prs
else
return
end
prs.each { |i| puts "#{i["title"]} (#{i["html_url"]})" }
end
def private_repo?(user, repo)
uri = URI.parse("https://api.github.com/repos/#{user}/#{repo}")
open(uri) { |json| json["private"] }
end
end
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment