Use JSON files for bottle upload data. (#166)

This means that we do not need to read formulae or evaluate Ruby at
upload time.

4 files changed, 165 insertions, 102 deletions

diff --git a/Library/Homebrew/cmd/bottle.rb b/Library/Homebrew/cmd/bottle.rb
index 2f8f7f615..c63f19240 100644
--- a/Library/Homebrew/cmd/bottle.rb
+++ b/Library/Homebrew/cmd/bottle.rb
@@ -281,7 +281,8 @@ module Homebrew
       bottle.prefix prefix
     end
     bottle.revision bottle_revision
-    bottle.sha256 bottle_path.sha256 => Utils::Bottles.tag
+    sha256 = bottle_path.sha256
+    bottle.sha256 sha256 => Utils::Bottles.tag
 
     old_spec = f.bottle_specification
     if ARGV.include?("--keep-old") && !old_spec.checksums.empty?
@@ -300,74 +301,104 @@ module Homebrew
     puts "./#{filename}"
     puts output
 
-    if ARGV.include? "--rb"
-      File.open("#{filename.prefix}.bottle.rb", "w") do |file|
-        file.write("\# #{f.full_name}\n")
-        file.write(output)
+    if ARGV.include? "--json"
+      json = {
+        f.full_name => {
+          "formula" => {
+            "pkg_version" => f.pkg_version.to_s,
+            "path" => f.path.to_s.strip_prefix("#{HOMEBREW_REPOSITORY}/"),
+          },
+          "bottle" => {
+            "root_url" => bottle.root_url,
+            "prefix" => bottle.prefix,
+            "cellar" => bottle.cellar.to_s,
+            "revision" => bottle.revision,
+            "tags" => {
+              Utils::Bottles.tag.to_s => {
+                "filename" => filename.to_s,
+                "sha256" => sha256,
+              },
+            }
+          },
+          "bintray" => {
+            "package" => Utils::Bottles::Bintray.package(f.full_name),
+            "repository" => Utils::Bottles::Bintray.repository(f.tap),
+          },
+        },
+      }
+      File.open("#{filename.prefix}.bottle.json", "w") do |file|
+        file.write Utils::JSON.dump json
       end
     end
   end
 
-  module BottleMerger
-    def bottle(&block)
-      instance_eval(&block)
-    end
-  end
-
   def merge
     write = ARGV.include? "--write"
-    keep_old = ARGV.include? "--keep-old"
-
-    merge_hash = {}
-    ARGV.named.each do |argument|
-      bottle_block = IO.read(argument)
-      formula_name = bottle_block.lines.first.sub(/^# /, "").chomp
-      merge_hash[formula_name] ||= []
-      merge_hash[formula_name] << bottle_block
+
+    bottles_hash = ARGV.named.reduce({}) do |hash, json_file|
+      hash.merge! Utils::JSON.load(IO.read(json_file))
     end
 
-    merge_hash.each do |formula_name, bottle_blocks|
+    bottles_hash.each do |formula_name, bottle_hash|
       ohai formula_name
-      f = Formulary.factory(formula_name)
 
-      if f.bottle_disabled?
-        ofail "Formula #{f.full_name} has disabled bottle"
-        puts f.bottle_disable_reason
-        next
+      bottle = BottleSpecification.new
+      bottle.root_url bottle_hash["bottle"]["root_url"]
+      cellar = bottle_hash["bottle"]["cellar"]
+      if cellar == "any" || cellar == "any_skip_relocation"
+        cellar = cellar.to_sym
       end
-
-      bottle = if keep_old
-        f.bottle_specification.dup
-      else
-        BottleSpecification.new
-      end
-      bottle.extend(BottleMerger)
-      bottle_blocks.each { |block| bottle.instance_eval(block) }
-
-      old_spec = f.bottle_specification
-      if keep_old && !old_spec.checksums.empty?
-         bad_fields = [:root_url, :prefix, :cellar, :revision].select do |field|
-           old_spec.send(field) != bottle.send(field)
-         end
-         bad_fields.delete(:cellar) if old_spec.cellar == :any && bottle.cellar == :any_skip_relocation
-         if bad_fields.any?
-           ofail "--keep-old is passed but there are changes in: #{bad_fields.join ", "}"
-           next
-         end
+      bottle.cellar cellar
+      bottle.prefix bottle_hash["bottle"]["prefix"]
+      bottle.revision bottle_hash["bottle"]["revision"]
+      bottle_hash["bottle"]["tags"].each do |tag, tag_hash|
+        bottle.sha256 tag_hash["sha256"] => tag.to_sym
       end
 
       output = bottle_output bottle
-      puts output
 
       if write
+        path = Pathname.new("#{HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"]}")
         update_or_add = nil
 
-        Utils::Inreplace.inreplace(f.path) do |s|
+        Utils::Inreplace.inreplace(path) do |s|
           if s.include? "bottle do"
             update_or_add = "update"
+            if ARGV.include? "--keep-old"
+              mismatches = []
+              s =~ /  bottle do(.+?)end\n/m
+              bottle_block_contents = $1
+              bottle_block_contents.lines.each do |line|
+                line = line.strip
+                next if line.empty?
+                key, value, _, tag = line.split " ", 4
+                value = value.to_s.delete ":'\""
+                tag = tag.to_s.delete ":"
+
+                if !tag.empty?
+                  if !bottle_hash["bottle"]["tags"][tag].to_s.empty?
+                    mismatches << "#{key} => #{tag}"
+                  else
+                    bottle.send(key, value => tag.to_sym)
+                  end
+                  next
+                end
+
+                old_value = bottle_hash["bottle"][key].to_s
+                next if key == "cellar" && old_value == "any" && value == "any_skip_relocation"
+                mismatches << key if old_value.empty? || value != old_value
+              end
+              if mismatches.any?
+                odie "--keep-old was passed but there were changes in #{mismatches.join(", ")}!"
+              end
+              output = bottle_output bottle
+            end
+            puts output
             string = s.sub!(/  bottle do.+?end\n/m, output)
             odie "Bottle block update failed!" unless string
           else
+            odie "--keep-old was passed but there was no existing bottle block!"
+            puts output
             update_or_add = "add"
             if s.include? "stable do"
               indent = s.slice(/^ +stable do/).length - "stable do".length
@@ -392,12 +423,16 @@ module Homebrew
         end
 
         unless ARGV.include? "--no-commit"
-          f.path.parent.cd do
+          pkg_version = bottle_hash["formula"]["pkg_version"]
+
+          path.parent.cd do
             safe_system "git", "commit", "--no-edit", "--verbose",
-              "--message=#{f.name}: #{update_or_add} #{f.pkg_version} bottle.",
-              "--", f.path
+              "--message=#{formula_name}: #{update_or_add} #{pkg_version} bottle.",
+              "--", path
           end
         end
+      else
+        puts output
       end
     end
   end
diff --git a/Library/Homebrew/cmd/pull.rb b/Library/Homebrew/cmd/pull.rb
index 4313ac894..536277f94 100644
--- a/Library/Homebrew/cmd/pull.rb
+++ b/Library/Homebrew/cmd/pull.rb
@@ -119,7 +119,7 @@ module Homebrew
       end
       patch_puller.apply_patch
 
-      changed_formulae = []
+      changed_formulae_names = []
 
       if tap
         Utils.popen_read(
@@ -128,7 +128,7 @@ module Homebrew
         ).each_line do |line|
           name = "#{tap.name}/#{File.basename(line.chomp, ".rb")}"
           begin
-            changed_formulae << Formula[name]
+            changed_formulae_names << name
           # Make sure we catch syntax errors.
           rescue Exception
             next
@@ -137,7 +137,10 @@ module Homebrew
       end
 
       fetch_bottles = false
-      changed_formulae.each do |f|
+      changed_formulae_names.each do |name|
+        next if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
+
+        f = Formula[name]
         if ARGV.include? "--bottle"
           if f.bottle_unneeded?
             ohai "#{f}: skipping unneeded bottle."
@@ -164,12 +167,16 @@ module Homebrew
         message += "\n#{close_message}" unless message.include? close_message
       end
 
-      if changed_formulae.empty?
+      if changed_formulae_names.empty?
         odie "cannot bump: no changed formulae found after applying patch" if do_bump
         is_bumpable = false
       end
-      if is_bumpable && !ARGV.include?("--clean")
-        formula = changed_formulae.first
+
+      is_bumpable = false if ARGV.include?("--clean")
+      is_bumpable = false if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
+
+      if is_bumpable
+        formula = Formula[changed_formulae_names.first]
         new_versions = current_versions_from_info_external(patch_changes[:formulae].first)
         orig_subject = message.empty? ? "" : message.lines.first.chomp
         bump_subject = subject_for_bump(formula, old_versions, new_versions)
@@ -219,7 +226,7 @@ module Homebrew
 
         # Publish bottles on Bintray
         unless ARGV.include? "--no-publish"
-          published = publish_changed_formula_bottles(tap, changed_formulae)
+          published = publish_changed_formula_bottles(tap, changed_formulae_names)
           bintray_published_formulae.concat(published)
         end
       end
@@ -239,11 +246,16 @@ module Homebrew
 
   private
 
-  def publish_changed_formula_bottles(tap, changed_formulae)
+  def publish_changed_formula_bottles(tap, changed_formulae_names)
+    if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
+      raise "Need to load formulae to publish them!"
+    end
+
     published = []
     bintray_creds = { :user => ENV["BINTRAY_USER"], :key => ENV["BINTRAY_KEY"] }
     if bintray_creds[:user] && bintray_creds[:key]
-      changed_formulae.each do |f|
+      changed_formulae_names.each do |name|
+        f = Formula[name]
         next if f.bottle_unneeded? || f.bottle_disabled?
         ohai "Publishing on Bintray: #{f.name} #{f.pkg_version}"
         publish_bottle_file_on_bintray(f, bintray_creds)
@@ -493,6 +505,11 @@ module Homebrew
   # version of a formula.
   def verify_bintray_published(formulae_names)
     return if formulae_names.empty?
+
+    if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
+      raise "Need to load formulae to verify their publication!"
+    end
+
     ohai "Verifying bottles published on Bintray"
     formulae = formulae_names.map { |n| Formula[n] }
     max_retries = 300 # shared among all bottles
diff --git a/Library/Homebrew/cmd/test-bot.rb b/Library/Homebrew/cmd/test-bot.rb
index db82f565b..00deca43f 100644
--- a/Library/Homebrew/cmd/test-bot.rb
+++ b/Library/Homebrew/cmd/test-bot.rb
@@ -114,7 +114,7 @@ module Homebrew
     end
 
     def command_short
-      (@command - %w[brew --force --retry --verbose --build-bottle --rb]).join(" ")
+      (@command - %w[brew --force --retry --verbose --build-bottle --json]).join(" ")
     end
 
     def passed?
@@ -570,15 +570,15 @@ module Homebrew
       test "brew", "audit", *audit_args
       if install_passed
         if formula.stable? && !ARGV.include?("--fast") && !ARGV.include?("--no-bottle") && !formula.bottle_disabled?
-          bottle_args = ["--verbose", "--rb", formula_name]
+          bottle_args = ["--verbose", "--json", formula_name]
           bottle_args << "--keep-old" if ARGV.include? "--keep-old"
           test "brew", "bottle", *bottle_args
           bottle_step = steps.last
           if bottle_step.passed? && bottle_step.has_output?
             bottle_filename =
               bottle_step.output.gsub(/.*(\.\/\S+#{Utils::Bottles::native_regex}).*/m, '\1')
-            bottle_rb_filename = bottle_filename.gsub(/\.(\d+\.)?tar\.gz$/, ".rb")
-            bottle_merge_args = ["--merge", "--write", "--no-commit", bottle_rb_filename]
+            bottle_json_filename = bottle_filename.gsub(/\.(\d+\.)?tar\.gz$/, ".json")
+            bottle_merge_args = ["--merge", "--write", "--no-commit", bottle_json_filename]
             bottle_merge_args << "--keep-old" if ARGV.include? "--keep-old"
             test "brew", "bottle", *bottle_merge_args
             test "brew", "uninstall", "--force", formula_name
@@ -768,6 +768,9 @@ module Homebrew
   end
 
   def test_ci_upload(tap)
+    # Don't trust formulae we're uploading
+    ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] = "1"
+
     jenkins = ENV["JENKINS_HOME"]
     job = ENV["UPSTREAM_JOB_NAME"]
     id = ENV["UPSTREAM_BUILD_ID"]
@@ -779,14 +782,13 @@ module Homebrew
       raise "Missing BINTRAY_USER or BINTRAY_KEY variables!"
     end
 
-    # Don't pass keys/cookies to subprocesses..
+    # Don't pass keys/cookies to subprocesses
     ENV["BINTRAY_KEY"] = nil
     ENV["HUDSON_SERVER_COOKIE"] = nil
     ENV["JENKINS_SERVER_COOKIE"] = nil
     ENV["HUDSON_COOKIE"] = nil
 
     ARGV << "--verbose"
-    ARGV << "--keep-old" if ENV["UPSTREAM_BOTTLE_KEEP_OLD"]
     ARGV << "--legacy" if ENV["UPSTREAM_BOTTLE_LEGACY"]
 
     bottles = Dir["#{jenkins}/jobs/#{job}/configurations/axis-version/*/builds/#{id}/archive/*.bottle*.*"]
@@ -817,53 +819,58 @@ module Homebrew
       end
     end
 
-    bottle_args = ["--merge", "--write", *Dir["*.bottle.rb"]]
-    bottle_args << "--keep-old" if ARGV.include? "--keep-old"
-    system "brew", "bottle", *bottle_args
+    json_files = Dir.glob("*.bottle.json")
+    system "brew", "bottle", "--merge", "--write", *json_files
 
     remote = "git@github.com:BrewTestBot/homebrew-#{tap.repo}.git"
-    tag = pr ? "pr-#{pr}" : "testing-#{number}"
+    git_tag = pr ? "pr-#{pr}" : "testing-#{number}"
+    safe_system "git", "push", "--force", remote, "master:master", ":refs/tags/#{git_tag}"
 
-    bintray_repo = Utils::Bottles::Bintray.repository(tap)
-    bintray_repo_url = "https://api.bintray.com/packages/homebrew/#{bintray_repo}"
     formula_packaged = {}
 
-    Dir.glob("*.bottle*.tar.gz") do |filename|
-      formula_name, canonical_formula_name = Utils::Bottles.resolve_formula_names filename
-      formula = Formulary.factory canonical_formula_name
-      version = formula.pkg_version
-      bintray_package = Utils::Bottles::Bintray.package formula_name
-
-      if system "curl", "-I", "--silent", "--fail", "--output", "/dev/null",
-                "#{BottleSpecification::DEFAULT_DOMAIN}/#{bintray_repo}/#{filename}"
-        raise <<-EOS.undent
-          #{filename} is already published. Please remove it manually from
-          https://bintray.com/homebrew/#{bintray_repo}/#{bintray_package}/view#files
-        EOS
-      end
+    bottles_hash = json_files.reduce({}) do |hash, json_file|
+      hash.merge! Utils::JSON.load(IO.read(json_file))
+    end
 
-      unless formula_packaged[formula_name]
-        package_url = "#{bintray_repo_url}/#{bintray_package}"
-        unless system "curl", "--silent", "--fail", "--output", "/dev/null", package_url
-          package_blob = <<-EOS.undent
-            {"name": "#{bintray_package}",
-             "public_download_numbers": true,
-             "public_stats": true}
+    bottles_hash.each do |formula_name, bottle_hash|
+      version = bottle_hash["formula"]["pkg_version"]
+      bintray_package = bottle_hash["bintray"]["package"]
+      bintray_repo = bottle_hash["bintray"]["repository"]
+      bintray_repo_url = "https://api.bintray.com/packages/homebrew/#{bintray_repo}"
+
+      bottle_hash["bottle"]["tags"].each do |tag, tag_hash|
+        filename = tag_hash["filename"]
+        if system "curl", "-I", "--silent", "--fail", "--output", "/dev/null",
+                  "#{BottleSpecification::DEFAULT_DOMAIN}/#{bintray_repo}/#{filename}"
+          raise <<-EOS.undent
+            #{filename} is already published. Please remove it manually from
+            https://bintray.com/homebrew/#{bintray_repo}/#{bintray_package}/view#files
           EOS
-          curl "--silent", "--fail", "-u#{bintray_user}:#{bintray_key}",
-               "-H", "Content-Type: application/json",
-               "-d", package_blob, bintray_repo_url
-          puts
         end
-        formula_packaged[formula_name] = true
-      end
 
-      content_url = "https://api.bintray.com/content/homebrew"
-      content_url += "/#{bintray_repo}/#{bintray_package}/#{version}/#{filename}"
-      content_url += "?override=1"
-      curl "--silent", "--fail", "-u#{bintray_user}:#{bintray_key}",
-           "-T", filename, content_url
-      puts
+        unless formula_packaged[formula_name]
+          package_url = "#{bintray_repo_url}/#{bintray_package}"
+          unless system "curl", "--silent", "--fail", "--output", "/dev/null", package_url
+            package_blob = <<-EOS.undent
+              {"name": "#{bintray_package}",
+               "public_download_numbers": true,
+               "public_stats": true}
+            EOS
+            curl "--silent", "--fail", "-u#{bintray_user}:#{bintray_key}",
+                 "-H", "Content-Type: application/json",
+                 "-d", package_blob, bintray_repo_url
+            puts
+          end
+          formula_packaged[formula_name] = true
+        end
+
+        content_url = "https://api.bintray.com/content/homebrew"
+        content_url += "/#{bintray_repo}/#{bintray_package}/#{version}/#{filename}"
+        content_url += "?override=1"
+        curl "--silent", "--fail", "-u#{bintray_user}:#{bintray_key}",
+             "-T", filename, content_url
+        puts
+      end
     end
 
     safe_system "git", "tag", "--force", tag
diff --git a/Library/Homebrew/formulary.rb b/Library/Homebrew/formulary.rb
index d75f63425..8a436531c 100644
--- a/Library/Homebrew/formulary.rb
+++ b/Library/Homebrew/formulary.rb
@@ -16,6 +16,10 @@ class Formulary
   end
 
   def self.load_formula(name, path, contents, namespace)
+    if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
+      raise "Formula loading disabled by HOMEBREW_DISABLE_LOAD_FORMULA!"
+    end
+
     mod = Module.new
     const_set(namespace, mod)
     mod.module_eval(contents, path)