Use `curl` for the GitHub API (#295)

* Move GitHub API module to utils/github.rb.

* Move curl method to utils/curl.rb.

* global: use long curl arguments and an array.

This makes the code more self-documenting.

* utils/curl: support reading curl's output.

* utils/github: use curl instead of open-uri.

It has far better proxy support.

* pull: set Homebrew user agent.

* gist-logs: remove trailing whitespace.

* gist-logs: use first instead of [0].

Easier to read.

* gist-logs: use curl-based GitHub.open method.

2 files changed, 291 insertions, 0 deletions

diff --git a/Library/Homebrew/utils/curl.rb b/Library/Homebrew/utils/curl.rb
new file mode 100644
index 000000000..a1653d46c
--- /dev/null
+++ b/Library/Homebrew/utils/curl.rb
@@ -0,0 +1,24 @@
+require "pathname"
+
+def curl_args(extra_args=[])
+  curl = Pathname.new ENV["HOMEBREW_CURL"]
+  curl = Pathname.new "/usr/bin/curl" unless curl.exist?
+  raise "#{curl} is not executable" unless curl.exist? && curl.executable?
+
+  flags = HOMEBREW_CURL_ARGS
+  flags -= ["--progress-bar"] if ARGV.verbose?
+
+  args = ["#{curl}"] + flags + extra_args
+  args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"]
+  args << "--silent" if !$stdout.tty? || ENV["TRAVIS"]
+  args
+end
+
+def curl(*args)
+  safe_system(*curl_args(args))
+end
+
+def curl_output(*args)
+  curl_args = curl_args(args) - ["--fail"]
+  Utils.popen_read_text(*curl_args)
+end
diff --git a/Library/Homebrew/utils/github.rb b/Library/Homebrew/utils/github.rb
new file mode 100644
index 000000000..7fd484211
--- /dev/null
+++ b/Library/Homebrew/utils/github.rb
@@ -0,0 +1,267 @@
+require "uri"
+require "tempfile"
+
+module GitHub
+  extend self
+  ISSUES_URI = URI.parse("https://api.github.com/search/issues")
+
+  Error = Class.new(RuntimeError)
+  HTTPNotFoundError = Class.new(Error)
+
+  class RateLimitExceededError < Error
+    def initialize(reset, error)
+      super <<-EOS.undent
+        GitHub API Error: #{error}
+        Try again in #{pretty_ratelimit_reset(reset)}, or create a personal access token:
+          #{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset}
+        and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token"
+      EOS
+    end
+
+    def pretty_ratelimit_reset(reset)
+      pretty_duration(Time.at(reset) - Time.now)
+    end
+  end
+
+  class AuthenticationFailedError < Error
+    def initialize(error)
+      message = "GitHub #{error}\n"
+      if ENV["HOMEBREW_GITHUB_API_TOKEN"]
+        message << <<-EOS.undent
+          HOMEBREW_GITHUB_API_TOKEN may be invalid or expired; check:
+          #{Tty.em}https://github.com/settings/tokens#{Tty.reset}
+        EOS
+      else
+        message << <<-EOS.undent
+          The GitHub credentials in the OS X keychain may be invalid.
+          Clear them with:
+            printf "protocol=https\\nhost=github.com\\n" | git credential-osxkeychain erase
+          Or create a personal access token:
+            #{Tty.em}https://github.com/settings/tokens/new?scopes=&description=Homebrew#{Tty.reset}
+          and then set the token as: export HOMEBREW_GITHUB_API_TOKEN="your_new_token"
+        EOS
+      end
+      super message
+    end
+  end
+
+  def api_credentials
+    @api_credentials ||= begin
+      if ENV["HOMEBREW_GITHUB_API_TOKEN"]
+        ENV["HOMEBREW_GITHUB_API_TOKEN"]
+      elsif ENV["HOMEBREW_GITHUB_API_USERNAME"] && ENV["HOMEBREW_GITHUB_API_PASSWORD"]
+        [ENV["HOMEBREW_GITHUB_API_USERNAME"], ENV["HOMEBREW_GITHUB_API_PASSWORD"]]
+      else
+        github_credentials = Utils.popen("git credential-osxkeychain get", "w+") do |io|
+          io.puts "protocol=https\nhost=github.com"
+          io.close_write
+          io.read
+        end
+        github_username = github_credentials[/username=(.+)/, 1]
+        github_password = github_credentials[/password=(.+)/, 1]
+        if github_username && github_password
+          [github_password, github_username]
+        else
+          []
+        end
+      end
+    end
+  end
+
+  def api_credentials_type
+    token, username = api_credentials
+    if token && !token.empty?
+      if username && !username.empty?
+        :keychain
+      else
+        :environment
+      end
+    else
+      :none
+    end
+  end
+
+  def api_credentials_error_message(response_headers)
+    return if response_headers.empty?
+
+    @api_credentials_error_message_printed ||= begin
+      unauthorized = (response_headers["http/1.1"] == "401 Unauthorized")
+      scopes = response_headers["x-accepted-oauth-scopes"].to_s.split(", ")
+      if !unauthorized && scopes.empty?
+        credentials_scopes = response_headers["x-oauth-scopes"].to_s.split(", ")
+
+        case GitHub.api_credentials_type
+        when :keychain
+          onoe <<-EOS.undent
+            Your OS X keychain GitHub credentials do not have sufficient scope!
+            Scopes they have: #{credentials_scopes}
+            Create a personal access token: https://github.com/settings/tokens
+            and then set HOMEBREW_GITHUB_API_TOKEN as the authentication method instead.
+          EOS
+        when :environment
+          onoe <<-EOS.undent
+            Your HOMEBREW_GITHUB_API_TOKEN does not have sufficient scope!
+            Scopes it has: #{credentials_scopes}
+            Create a new personal access token: https://github.com/settings/tokens
+            and then set the new HOMEBREW_GITHUB_API_TOKEN as the authentication method instead.
+          EOS
+        end
+      end
+      true
+    end
+  end
+
+  def open(url, data=nil)
+    # This is a no-op if the user is opting out of using the GitHub API.
+    return if ENV["HOMEBREW_NO_GITHUB_API"]
+
+    args = %W[--header application/vnd.github.v3+json --write-out \n%{http_code}]
+    args += curl_args
+
+    token, username = api_credentials
+    case api_credentials_type
+    when :keychain
+      args += %W[--user #{username}:#{token}]
+    when :environment
+      args += ["--header", "Authorization: token #{token}"]
+    end
+
+    data_tmpfile = nil
+    if data
+      begin
+        data = Utils::JSON.dump data
+        data_tmpfile = Tempfile.new("github_api_post", HOMEBREW_TEMP)
+      rescue Utils::JSON::Error => e
+        raise Error, "Failed to parse JSON request:\n#{e.message}\n#{data}", e.backtrace
+      end
+    end
+
+    headers_tmpfile = Tempfile.new("github_api_headers", HOMEBREW_TEMP)
+    begin
+      if data
+        data_tmpfile.write data
+        args += ["--data", "@#{data_tmpfile.path}"]
+      end
+
+      args += ["--dump-header", "#{headers_tmpfile.path}"]
+
+      output, _, http_code = curl_output(url.to_s, *args).rpartition("\n")
+      output, _, http_code = output.rpartition("\n") if http_code == "000"
+      headers = headers_tmpfile.read
+    ensure
+      if data_tmpfile
+        data_tmpfile.close
+        data_tmpfile.unlink
+      end
+      headers_tmpfile.close
+      headers_tmpfile.unlink
+    end
+
+    begin
+      if !http_code.start_with?("2") && !$?.success?
+        raise_api_error(output, http_code, headers)
+      end
+      json = Utils::JSON.load output
+      if block_given?
+        yield json
+      else
+        json
+      end
+    rescue Utils::JSON::Error => e
+      raise Error, "Failed to parse JSON response\n#{e.message}", e.backtrace
+    end
+  end
+
+  def raise_api_error(output, http_code, headers)
+    meta = {}
+    headers.lines.each do |l|
+      key, _, value = l.delete(":").partition(" ")
+      key = key.downcase.strip
+      next if key.empty?
+      meta[key] = value.strip
+    end
+
+    if meta.fetch("x-ratelimit-remaining", 1).to_i <= 0
+      reset = meta.fetch("x-ratelimit-reset").to_i
+      error = Utils::JSON.load(output)["message"]
+      raise RateLimitExceededError.new(reset, error)
+    end
+
+    GitHub.api_credentials_error_message(meta)
+
+    case http_code
+    when "401", "403"
+      raise AuthenticationFailedError.new(output)
+    when "404"
+      raise HTTPNotFoundError, output
+    else
+      error = Utils::JSON.load(output)["message"] rescue nil
+      error ||= output
+      raise Error, error
+    end
+  end
+
+  def issues_matching(query, qualifiers = {})
+    uri = ISSUES_URI.dup
+    uri.query = build_query_string(query, qualifiers)
+    open(uri) { |json| json["items"] }
+  end
+
+  def repository(user, repo)
+    open(URI.parse("https://api.github.com/repos/#{user}/#{repo}")) { |j| j }
+  end
+
+  def build_query_string(query, qualifiers)
+    s = "q=#{uri_escape(query)}+"
+    s << build_search_qualifier_string(qualifiers)
+    s << "&per_page=100"
+  end
+
+  def build_search_qualifier_string(qualifiers)
+    {
+      :repo => "Homebrew/homebrew-core",
+      :in => "title"
+    }.update(qualifiers).map do |qualifier, value|
+      "#{qualifier}:#{value}"
+    end.join("+")
+  end
+
+  def uri_escape(query)
+    if URI.respond_to?(:encode_www_form_component)
+      URI.encode_www_form_component(query)
+    else
+      require "erb"
+      ERB::Util.url_encode(query)
+    end
+  end
+
+  def issues_for_formula(name, options = {})
+    tap = options[:tap] || CoreTap.instance
+    issues_matching(name, :state => "open", :repo => "#{tap.user}/homebrew-#{tap.repo}")
+  end
+
+  def print_pull_requests_matching(query)
+    return [] if ENV["HOMEBREW_NO_GITHUB_API"]
+    ohai "Searching pull requests..."
+
+    open_or_closed_prs = issues_matching(query, :type => "pr")
+
+    open_prs = open_or_closed_prs.select { |i| i["state"] == "open" }
+    if open_prs.any?
+      puts "Open pull requests:"
+      prs = open_prs
+    elsif open_or_closed_prs.any?
+      puts "Closed pull requests:"
+      prs = open_or_closed_prs
+    else
+      return
+    end
+
+    prs.each { |i| puts "#{i["title"]} (#{i["html_url"]})" }
+  end
+
+  def private_repo?(user, repo)
+    uri = URI.parse("https://api.github.com/repos/#{user}/#{repo}")
+    open(uri) { |json| json["private"] }
+  end
+end