brew-test-bot: make an internal command.

1 files changed, 666 insertions, 0 deletions

diff --git a/Library/Homebrew/cmd/test-bot.rb b/Library/Homebrew/cmd/test-bot.rb
new file mode 100755
index 000000000..95bbc4686
--- /dev/null
+++ b/Library/Homebrew/cmd/test-bot.rb
@@ -0,0 +1,666 @@
+# Comprehensively test a formula or pull request.
+#
+# Usage: brew test-bot [options...] <pull-request|formula>
+#
+# Options:
+# --keep-logs:    Write and keep log files under ./brewbot/
+# --cleanup:      Clean the Homebrew directory. Very dangerous. Use with care.
+# --clean-cache:  Remove all cached downloads. Use with care.
+# --skip-setup:   Don't check the local system is setup correctly.
+# --junit:        Generate a JUnit XML test results file.
+# --email:        Generate an email subject file.
+# --no-bottle:    Run brew install without --build-bottle
+# --HEAD:         Run brew install with --HEAD
+# --local:        Ask Homebrew to write verbose logs under ./logs/
+# --tap=<tap>:    Use the git repository of the given tap
+# --dry-run:      Just print commands, don't run them.
+#
+# --ci-master:         Shortcut for Homebrew master branch CI options.
+# --ci-pr:             Shortcut for Homebrew pull request CI options.
+# --ci-testing:        Shortcut for Homebrew testing CI options.
+# --ci-pr-upload:      Homebrew CI pull request bottle upload.
+# --ci-testing-upload: Homebrew CI testing bottle upload.
+
+require 'formula'
+require 'utils'
+require 'date'
+require 'rexml/document'
+require 'rexml/xmldecl'
+require 'rexml/cdata'
+
+module Homebrew
+  EMAIL_SUBJECT_FILE = "brew-test-bot.#{MacOS.cat}.email.txt"
+
+  def homebrew_git_repo tap=nil
+    if tap
+        HOMEBREW_LIBRARY/"Taps/#{tap}"
+      else
+        HOMEBREW_REPOSITORY
+      end
+  end
+
+  class Step
+    attr_reader :command, :name, :status, :output, :time
+
+    def initialize test, command, options={}
+      @test = test
+      @category = test.category
+      @command = command
+      @puts_output_on_success = options[:puts_output_on_success]
+      @name = command[1].delete("-")
+      @status = :running
+      @repository = options[:repository] || HOMEBREW_REPOSITORY
+      @time = 0
+    end
+
+    def log_file_path
+      file = "#{@category}.#{@name}.txt"
+      root = @test.log_root
+      root ? root + file : file
+    end
+
+    def status_colour
+      case @status
+      when :passed  then "green"
+      when :running then "orange"
+      when :failed  then "red"
+      end
+    end
+
+    def status_upcase
+      @status.to_s.upcase
+    end
+
+    def command_short
+      (@command - %w[brew --force --retry --verbose --build-bottle --rb]).join(" ")
+    end
+
+    def passed?
+      @status == :passed
+    end
+
+    def failed?
+      @status == :failed
+    end
+
+    def puts_command
+      cmd = @command.join(" ")
+      print "#{Tty.blue}==>#{Tty.white} #{cmd}#{Tty.reset}"
+      tabs = (80 - "PASSED".length + 1 - cmd.length) / 8
+      tabs.times{ print "\t" }
+      $stdout.flush
+    end
+
+    def puts_result
+      puts " #{Tty.send status_colour}#{status_upcase}#{Tty.reset}"
+    end
+
+    def has_output?
+      @output && !@output.empty?
+    end
+
+    def run
+      puts_command
+      if ARGV.include? "--dry-run"
+        puts
+        @status = :passed
+        return
+      end
+
+      start_time = Time.now
+
+      log = log_file_path
+
+      pid = fork do
+        File.open(log, "wb") do |f|
+          STDOUT.reopen(f)
+          STDERR.reopen(f)
+        end
+        Dir.chdir(@repository) if @command.first == "git"
+        exec(*@command)
+      end
+      Process.wait(pid)
+
+      @time = Time.now - start_time
+
+      @status = $?.success? ? :passed : :failed
+      puts_result
+
+      if File.exist?(log)
+        @output = File.read(log)
+        if has_output? and (failed? or @puts_output_on_success)
+          puts @output
+        end
+        FileUtils.rm(log) unless ARGV.include? "--keep-logs"
+      end
+    end
+  end
+
+  class Test
+    attr_reader :log_root, :category, :name, :steps
+
+    def initialize argument, tap=nil
+      @hash = nil
+      @url = nil
+      @formulae = []
+      @steps = []
+      @tap = tap
+      @repository = Homebrew.homebrew_git_repo @tap
+      @repository_requires_tapping = !@repository.directory?
+
+      url_match = argument.match HOMEBREW_PULL_OR_COMMIT_URL_REGEX
+
+      # Tap repository if required, this is done before everything else
+      # because Formula parsing and/or git commit hash lookup depends on it.
+      test "brew", "tap", @tap if @tap && @repository_requires_tapping
+
+      begin
+        formula = Formulary.factory(argument)
+      rescue FormulaUnavailableError
+      end
+
+      git "rev-parse", "--verify", "-q", argument
+      if $?.success?
+        @hash = argument
+      elsif url_match
+        @url = url_match[0]
+      elsif formula
+        @formulae = [argument]
+      else
+        odie "#{argument} is not a pull request URL, commit URL or formula name."
+      end
+
+      @category = __method__
+      @brewbot_root = Pathname.pwd + "brewbot"
+      FileUtils.mkdir_p @brewbot_root
+    end
+
+    def no_args?
+      @hash == 'HEAD'
+    end
+
+    def git(*args)
+      rd, wr = IO.pipe
+
+      pid = fork do
+        rd.close
+        STDERR.reopen("/dev/null")
+        STDOUT.reopen(wr)
+        wr.close
+        Dir.chdir @repository
+        exec("git", *args)
+      end
+      wr.close
+      Process.wait(pid)
+
+      rd.read
+    ensure
+      rd.close
+    end
+
+    def download
+      def shorten_revision revision
+        git("rev-parse", "--short", revision).strip
+      end
+
+      def current_sha1
+        shorten_revision 'HEAD'
+      end
+
+      def current_branch
+        git("symbolic-ref", "HEAD").gsub("refs/heads/", "").strip
+      end
+
+      def single_commit? start_revision, end_revision
+        git("rev-list", "--count", "#{start_revision}..#{end_revision}").to_i == 1
+      end
+
+      @category = __method__
+      @start_branch = current_branch
+
+      # Use Jenkins environment variables if present.
+      if no_args? and ENV['GIT_PREVIOUS_COMMIT'] and ENV['GIT_COMMIT'] \
+         and not ENV['ghprbPullId']
+        diff_start_sha1 = shorten_revision ENV['GIT_PREVIOUS_COMMIT']
+        diff_end_sha1 = shorten_revision ENV['GIT_COMMIT']
+        test "brew", "update" if current_branch == "master"
+      elsif @hash or @url
+        diff_start_sha1 = current_sha1
+        test "brew", "update" if current_branch == "master"
+        diff_end_sha1 = current_sha1
+      end
+
+      # Handle Jenkins pull request builder plugin.
+      if ENV['ghprbPullId'] and ENV['GIT_URL']
+        git_url = ENV['GIT_URL']
+        git_match = git_url.match %r{.*github.com[:/](\w+/\w+).*}
+        if git_match
+          github_repo = git_match[1]
+          pull_id = ENV['ghprbPullId']
+          @url = "https://github.com/#{github_repo}/pull/#{pull_id}"
+          @hash = nil
+        else
+          puts "Invalid 'ghprbPullId' environment variable value!"
+        end
+      end
+
+      if no_args?
+        if diff_start_sha1 == diff_end_sha1 or \
+          single_commit?(diff_start_sha1, diff_end_sha1)
+          @name = diff_end_sha1
+        else
+          @name = "#{diff_start_sha1}-#{diff_end_sha1}"
+        end
+      elsif @hash
+        test "git", "checkout", @hash
+        diff_start_sha1 = "#{@hash}^"
+        diff_end_sha1 = @hash
+        @name = @hash
+      elsif @url
+        test "git", "checkout", current_sha1
+        test "brew", "pull", "--clean", @url
+        diff_end_sha1 = current_sha1
+        @short_url = @url.gsub('https://github.com/', '')
+        if @short_url.include? '/commit/'
+          # 7 characters should be enough for a commit (not 40).
+          @short_url.gsub!(/(commit\/\w{7}).*/, '\1')
+          @name = @short_url
+        else
+          @name = "#{@short_url}-#{diff_end_sha1}"
+        end
+      else
+        diff_start_sha1 = diff_end_sha1 = current_sha1
+        @name = "#{@formulae.first}-#{diff_end_sha1}"
+      end
+
+      @log_root = @brewbot_root + @name
+      FileUtils.mkdir_p @log_root
+
+      return unless diff_start_sha1 != diff_end_sha1
+      return if @url and not steps.last.passed?
+
+      if @tap
+        formula_path = %w[Formula HomebrewFormula].find { |dir| (@repository/dir).directory? } || ""
+      else
+        formula_path = "Library/Formula"
+      end
+
+      git(
+        "diff-tree", "-r", "--name-only", "--diff-filter=AM",
+        diff_start_sha1, diff_end_sha1, "--", formula_path
+      ).each_line do |line|
+        @formulae << File.basename(line.chomp, ".rb")
+      end
+    end
+
+    def skip formula
+      puts "#{Tty.blue}==>#{Tty.white} SKIPPING: #{formula}#{Tty.reset}"
+    end
+
+    def satisfied_requirements? formula_object, spec
+      requirements = formula_object.send(spec).requirements
+
+      unsatisfied_requirements = requirements.reject do |requirement|
+        requirement.satisfied? || requirement.default_formula?
+      end
+
+      if unsatisfied_requirements.empty?
+        true
+      else
+        formula = formula_object.name
+        formula += " (#{spec})" unless spec == :stable
+        skip formula
+        unsatisfied_requirements.each {|r| puts r.message}
+        false
+      end
+    end
+
+    def setup
+      @category = __method__
+      return if ARGV.include? "--skip-setup"
+      test "brew", "doctor"
+      test "brew", "--env"
+      test "brew", "config"
+    end
+
+    def formula formula
+      @category = __method__.to_s + ".#{formula}"
+
+      test "brew", "uses", formula
+      dependencies = `brew deps #{formula}`.split("\n")
+      dependencies -= `brew list`.split("\n")
+      unchanged_dependencies = dependencies - @formulae
+      changed_dependences = dependencies - unchanged_dependencies
+      formula_object = Formulary.factory(formula)
+      return unless satisfied_requirements?(formula_object, :stable)
+
+      installed_gcc = false
+      deps = formula_object.stable.deps.to_a
+      reqs = formula_object.stable.requirements.to_a
+      if formula_object.devel && !ARGV.include?('--HEAD')
+        deps |= formula_object.devel.deps.to_a
+        reqs |= formula_object.devel.requirements.to_a
+      end
+
+      begin
+        deps.each { |d| CompilerSelector.select_for(d.to_formula) }
+        CompilerSelector.select_for(formula_object)
+      rescue CompilerSelectionError => e
+        unless installed_gcc
+          test "brew", "install", "gcc"
+          installed_gcc = true
+          OS::Mac.clear_version_cache
+          retry
+        end
+        skip formula
+        puts e.message
+        return
+      end
+
+      if (deps | reqs).any? { |d| d.name == "mercurial" && d.build? }
+        test "brew", "install", "mercurial"
+      end
+
+      test "brew", "fetch", "--retry", *unchanged_dependencies unless unchanged_dependencies.empty?
+      test "brew", "fetch", "--retry", "--build-bottle", *changed_dependences unless changed_dependences.empty?
+      formula_fetch_options = []
+      formula_fetch_options << "--build-bottle" unless ARGV.include? "--no-bottle"
+      formula_fetch_options << "--force" if ARGV.include? "--cleanup"
+      formula_fetch_options << formula
+      test "brew", "fetch", "--retry", *formula_fetch_options
+      test "brew", "uninstall", "--force", formula if formula_object.installed?
+      install_args = %w[--verbose]
+      install_args << "--build-bottle" unless ARGV.include? "--no-bottle"
+      install_args << "--HEAD" if ARGV.include? "--HEAD"
+      install_args << formula
+      # Don't care about e.g. bottle failures for dependencies.
+      ENV["HOMEBREW_DEVELOPER"] = nil
+      test "brew", "install", "--only-dependencies", *install_args unless dependencies.empty?
+      ENV["HOMEBREW_DEVELOPER"] = "1"
+      test "brew", "install", *install_args
+      install_passed = steps.last.passed?
+      test "brew", "audit", formula
+      if install_passed
+        unless ARGV.include? '--no-bottle'
+          test "brew", "bottle", "--rb", formula, :puts_output_on_success => true
+          bottle_step = steps.last
+          if bottle_step.passed? and bottle_step.has_output?
+            bottle_filename =
+              bottle_step.output.gsub(/.*(\.\/\S+#{bottle_native_regex}).*/m, '\1')
+            test "brew", "uninstall", "--force", formula
+            test "brew", "install", bottle_filename
+          end
+        end
+        test "brew", "test", "--verbose", formula if formula_object.test_defined?
+        test "brew", "uninstall", "--force", formula
+      end
+
+      if formula_object.devel && !ARGV.include?('--HEAD') \
+         && satisfied_requirements?(formula_object, :devel)
+        test "brew", "fetch", "--retry", "--devel", *formula_fetch_options
+        test "brew", "install", "--devel", "--verbose", formula
+        devel_install_passed = steps.last.passed?
+        test "brew", "audit", "--devel", formula
+        if devel_install_passed
+          test "brew", "test", "--devel", "--verbose", formula if formula_object.test_defined?
+          test "brew", "uninstall", "--devel", "--force", formula
+        end
+      end
+      test "brew", "uninstall", "--force", *unchanged_dependencies unless unchanged_dependencies.empty?
+    end
+
+    def homebrew
+      @category = __method__
+      test "brew", "tests"
+      test "brew", "readall"
+    end
+
+    def cleanup_before
+      @category = __method__
+      return unless ARGV.include? '--cleanup'
+      git "stash"
+      git "am", "--abort"
+      git "rebase", "--abort"
+      git "reset", "--hard"
+      git "checkout", "-f", "master"
+      git "clean", "--force", "-dx"
+    end
+
+    def cleanup_after
+      @category = __method__
+
+      checkout_args = []
+      if ARGV.include? '--cleanup'
+        test "git", "clean", "--force", "-dx"
+        checkout_args << "-f"
+      end
+
+      checkout_args << @start_branch
+
+      if ARGV.include? '--cleanup' or @url or @hash
+        test "git", "checkout", *checkout_args
+      end
+
+      if ARGV.include? '--cleanup'
+        test "git", "reset", "--hard"
+        git "stash", "pop"
+        test "brew", "cleanup"
+      end
+
+      test "brew", "untap", @tap if @tap && @repository_requires_tapping
+
+      FileUtils.rm_rf @brewbot_root unless ARGV.include? "--keep-logs"
+    end
+
+    def test(*args)
+      options = Hash === args.last ? args.pop : {}
+      options[:repository] = @repository
+      step = Step.new self, args, options
+      step.run
+      steps << step
+      step
+    end
+
+    def check_results
+      status = :passed
+      steps.each do |step|
+        case step.status
+        when :passed  then next
+        when :running then raise
+        when :failed  then status = :failed
+        end
+      end
+      status == :passed
+    end
+
+    def formulae
+      changed_formulae_dependents = {}
+      dependencies = []
+      non_dependencies = []
+
+      @formulae.each do |formula|
+        formula_dependencies = `brew deps #{formula}`.split("\n")
+        unchanged_dependencies = formula_dependencies - @formulae
+        changed_dependences = formula_dependencies - unchanged_dependencies
+        changed_dependences.each do |changed_formula|
+          changed_formulae_dependents[changed_formula] ||= 0
+          changed_formulae_dependents[changed_formula] += 1
+        end
+      end
+
+      changed_formulae = changed_formulae_dependents.sort do |a1,a2|
+        a2[1].to_i <=> a1[1].to_i
+      end
+      changed_formulae.map!(&:first)
+      unchanged_formulae = @formulae - changed_formulae
+      changed_formulae + unchanged_formulae
+    end
+
+    def run
+      cleanup_before
+      download
+      setup
+      homebrew
+      formulae.each do |f|
+        formula(f)
+      end
+      cleanup_after
+      check_results
+    end
+  end
+
+  def test_bot
+    tap = ARGV.value('tap')
+
+    if Pathname.pwd == HOMEBREW_PREFIX and ARGV.include? "--cleanup"
+      odie 'cannot use --cleanup from HOMEBREW_PREFIX as it will delete all output.'
+    end
+
+    if ARGV.include? "--email"
+      File.open EMAIL_SUBJECT_FILE, 'w' do |file|
+        # The file should be written at the end but in case we don't get to that
+        # point ensure that we have something valid.
+        file.write "#{MacOS.version}: internal error."
+      end
+    end
+
+    ENV['HOMEBREW_DEVELOPER'] = '1'
+    ENV['HOMEBREW_NO_EMOJI'] = '1'
+    if ARGV.include? '--ci-master' or ARGV.include? '--ci-pr' \
+       or ARGV.include? '--ci-testing'
+      ARGV << '--cleanup' << '--junit' << '--local'
+    end
+    if ARGV.include? '--ci-master'
+      ARGV << '--no-bottle' << '--email'
+    end
+
+    if ARGV.include? '--local'
+      ENV['HOMEBREW_LOGS'] = "#{Dir.pwd}/logs"
+    end
+
+    if ARGV.include? '--ci-pr-upload' or ARGV.include? '--ci-testing-upload'
+      jenkins = ENV['JENKINS_HOME']
+      job = ENV['UPSTREAM_JOB_NAME']
+      id = ENV['UPSTREAM_BUILD_ID']
+      raise "Missing Jenkins variables!" unless jenkins and job and id
+
+      ARGV << '--verbose'
+      cp_args = Dir["#{jenkins}/jobs/#{job}/configurations/axis-version/*/builds/#{id}/archive/*.bottle*.*"] + ["."]
+      return unless system "cp", *cp_args
+
+      ENV["GIT_COMMITTER_NAME"] = "BrewTestBot"
+      ENV["GIT_COMMITTER_EMAIL"] = "brew-test-bot@googlegroups.com"
+      ENV["GIT_WORK_TREE"] = Homebrew.homebrew_git_repo tap
+      ENV["GIT_DIR"] = "#{ENV["GIT_WORK_TREE"]}/.git"
+
+      pr = ENV['UPSTREAM_PULL_REQUEST']
+      number = ENV['UPSTREAM_BUILD_NUMBER']
+
+      system "git am --abort 2>/dev/null"
+      system "git rebase --abort 2>/dev/null"
+      safe_system "git", "checkout", "-f", "master"
+      safe_system "git", "reset", "--hard", "origin/master"
+      safe_system "brew", "update"
+
+      if ARGV.include? '--ci-pr-upload'
+        safe_system "brew", "pull", "--clean", pr
+      end
+
+      ENV["GIT_AUTHOR_NAME"] = ENV["GIT_COMMITTER_NAME"]
+      ENV["GIT_AUTHOR_EMAIL"] = ENV["GIT_COMMITTER_EMAIL"]
+      safe_system "brew", "bottle", "--merge", "--write", *Dir["*.bottle.rb"]
+
+      remote = "git@github.com:BrewTestBot/homebrew.git"
+      tag = pr ? "pr-#{pr}" : "testing-#{number}"
+      safe_system "git", "push", "--force", remote, "master:master", ":refs/tags/#{tag}"
+
+      path = "/home/frs/project/m/ma/machomebrew/Bottles/"
+      url = "BrewTestBot,machomebrew@frs.sourceforge.net:#{path}"
+
+      rsync_args = %w[--partial --progress --human-readable --compress]
+      rsync_args += Dir["*.bottle*.tar.gz"] + [url]
+
+      safe_system "rsync", *rsync_args
+      safe_system "git", "tag", "--force", tag
+      safe_system "git", "push", "--force", remote, "refs/tags/#{tag}"
+      return
+    end
+
+    tests = []
+    any_errors = false
+    if ARGV.named.empty?
+      # With no arguments just build the most recent commit.
+      test = Test.new('HEAD', tap)
+      any_errors = test.run
+      tests << test
+    else
+      ARGV.named.each do |argument|
+        test = Test.new(argument, tap)
+        any_errors = test.run or any_errors
+        tests << test
+      end
+    end
+
+    if ARGV.include? "--junit"
+      xml_document = REXML::Document.new
+      xml_document << REXML::XMLDecl.new
+      testsuites = xml_document.add_element 'testsuites'
+      tests.each do |test|
+        testsuite = testsuites.add_element 'testsuite'
+        testsuite.attributes['name'] = "brew-test-bot.#{MacOS.cat}"
+        testsuite.attributes['tests'] = test.steps.count
+        test.steps.each do |step|
+          testcase = testsuite.add_element 'testcase'
+          testcase.attributes['name'] = step.command_short
+          testcase.attributes['status'] = step.status
+          testcase.attributes['time'] = step.time
+          failure = testcase.add_element 'failure' if step.failed?
+          if step.has_output?
+            # Remove invalid XML CData characters from step output.
+            output = step.output
+            if output.respond_to?(:force_encoding) && !output.valid_encoding?
+              output.force_encoding(Encoding::UTF_8)
+            end
+            output = REXML::CData.new output.delete("\000\a\b\e\f")
+            if step.passed?
+              system_out = testcase.add_element 'system-out'
+              system_out.text = output
+            else
+              failure.attributes["message"] = "#{step.status}: #{step.command.join(" ")}"
+              failure.text = output
+            end
+          end
+        end
+      end
+
+      open("brew-test-bot.xml", "w") do |xml_file|
+        pretty_print_indent = 2
+        xml_document.write(xml_file, pretty_print_indent)
+      end
+    end
+
+    if ARGV.include? "--email"
+      failed_steps = []
+      tests.each do |test|
+        test.steps.each do |step|
+          next unless step.failed?
+          failed_steps << step.command_short
+        end
+      end
+
+      if failed_steps.empty?
+        email_subject = ''
+      else
+        email_subject = "#{MacOS.version}: #{failed_steps.join ', '}."
+      end
+
+      File.open EMAIL_SUBJECT_FILE, 'w' do |file|
+        file.write email_subject
+      end
+    end
+
+    safe_system "rm -rf #{HOMEBREW_CACHE}/*" if ARGV.include? "--clean-cache"
+
+    Homebrew.failed = any_errors
+  end
+end