openproject/script/bulk_run_rspec

#!/usr/bin/env ruby

# Runs the given specs multiple times to check if they are reliable.
#
# Run with --help for more information

# rubocop:disable Rails
require "fileutils"
require "optparse"
require "ostruct"
require "pathname"

RAILS_ROOT = Pathname.new File.expand_path("../", __dir__)
RSPEC_EXAMPLE_STATUS_PERSISTENCE_FILE_PATH = RAILS_ROOT.join("tmp/spec_examples.txt")
PATTERN = ENV["BULK_RUN_PATTERN"] ? Regexp.new(ENV["BULK_RUN_PATTERN"]) : /features/

class Options
  DEFAULTS = {
    run_count: ENV["BULK_RUN_COUNT"] ? ENV["BULK_RUN_COUNT"].to_i : 5
  }.freeze

  BANNER = <<~BANNER.freeze
    Usage: #{$0} [options] [spec ...]

    Runs the given specs multiple times to check if they are reliable.

    Tests to run are fetched from tmp/spec_examples.txt matching /features/
    pattern.

    Results are stored in tmp/bulk_run:
      - results.txt   summary
      - logs/*.log    one log file per test run

    Options:
  BANNER

  class << self
    def options
      return @options if defined?(@options)

      @options = DEFAULTS.dup
      parse_options!
      @options
    end

    def method_missing(name, *)
      if DEFAULTS.key?(name)
        options[name]
      else
        super
      end
    end

    def respond_to_missing?(method_name, include_private = false)
      DEFAULTS.key?(name) || super
    end

    def parse_options!
      options.merge!(parse_args)
    end

    def parse_args
      options = {}
      opt_parser = OptionParser.new do |parser|
        parser.banner = BANNER

        parser.on("-c", "--run-count COUNT", "number of runs (BULK_RUN_COUNT)") do |count|
          options[:run_count] = count.to_i
        end

        parser.on("-h", "--help", "Prints this help") do
          puts parser
          exit
        end
      end
      opt_parser.parse!
      ensure_specs_args_present!(opt_parser)
      options
    end

    def ensure_specs_args_present!(parser)
      if ARGV.empty?
        puts "Error: missing spec path"
        puts parser
        exit 1
      end
    end
  end
end

def normalized(path)
  path.sub(/^\.\//, "").tr("/", "__").tr("[]:", "_")
end

class String
  # Colors, colors everywhere
  {
    red: 31,
    green: 32,
    brown: 33
  }.each do |color, ansi_code|
    define_method(color) { "\e[#{ansi_code}m#{self}\e[0m" }
  end
end

class Test
  attr_accessor :path, :runs

  def initialize(path)
    @path = path
    @runs = []
  end

  def to_line
    [path, normalized_name, runs.count, runs.count(&:passed?), runs.count(&:failed?)].join(" | ")
  end

  def normalized_name
    normalized(path)
  end
end

class Run
  attr_accessor :output
  attr_reader :result, :path, :start, :end

  def initialize(path)
    @path = path
    @result = nil
    @start = Time.now
    @end = nil
    @output = nil
  end

  def failed!
    self.result = :failed
  end

  def failed?
    result == :failed
  end

  def passed!
    self.result = :passed
  end

  def passed?
    result == :passed
  end

  def result=(result)
    @result = result
    @end = Time.now
  end
end

class BulkRunner
  def found_tests
    @found_tests ||=
      if ARGV.any?
        ARGV
      else
        lines = File.readlines(RSPEC_EXAMPLE_STATUS_PERSISTENCE_FILE_PATH)
        lines = lines.grep(PATTERN)
        lines.map { |line| line.split("|").first.strip }
      end
  end

  def tests
    @tests ||= found_tests.map { |test_path| Test.new(test_path) }
  end

  def copy_screenshots_from_capybara_dir_to_bulk_run_dir
    capybara_screenshots_dir.glob("screenshot_*").each do |screenshot|
      FileUtils.cp(screenshot, bulk_run_dir("screenshots/"))
    end
  end

  def copy_screenshots_from_bulk_run_dir_to_capybara_dir
    bulk_run_dir.glob("screenshots/screenshot_*").each do |screenshot|
      FileUtils.cp(screenshot, capybara_screenshots_dir)
    end
  end

  # rubocop:disable Metrics/AbcSize
  def run
    FileUtils.rm_rf(bulk_run_dir("logs")) if bulk_run_dir("logs").exist?
    FileUtils.mkdir_p([bulk_run_dir("logs"),
                       bulk_run_dir("screenshots"),
                       capybara_screenshots_dir])
    puts "#{tests.count} tests to run"
    puts "logs: #{bulk_run_dir('logs')}"
    tests.each do |test|
      puts "=" * 80
      puts "Running #{test.path} #{Options.run_count} times"
      puts "=" * 80
      Options.run_count.times do |i|
        puts " Run #{i} ".center(80, "-")
        run = Run.new(test.path)
        run.output = `DISABLE_PRY=1 CI=true bundle exec rspec '#{run.path}' 2>&1`
        if $?.success?
          puts "ok".green
          run.passed!
        else
          puts "ko".red
          copy_screenshots_from_capybara_dir_to_bulk_run_dir
          run.failed!
        end
        File.write(bulk_run_dir("logs/#{test.normalized_name}.#{run.result}.#{i}.log"), run.output)
        test.runs << run
      end

      if test.runs.all?(&:passed?)
        puts "PASSED".green
      else
        puts "FAILED".red
        copy_screenshots_from_bulk_run_dir_to_capybara_dir
      end
    end
  end
  # rubocop:enable Metrics/AbcSize

  def bulk_run_dir(path = "")
    RAILS_ROOT.join("tmp/bulk_run").join(path)
  end

  def capybara_screenshots_dir
    RAILS_ROOT.join("tmp/capybara")
  end

  def save
    puts "Saving"
    File.open(bulk_run_dir("results.txt"), "w") do |results|
      results.puts("test_path | log_prefix | count | passed | failed")
      tests.each do |test|
        results.puts(test.to_line)
      end
    end
    puts "Done"
  end
end

Options.options
bulk_runner = BulkRunner.new
begin
  bulk_runner.run
rescue Interrupt
  puts "Interrupted. Stopping and Saving.".brown
ensure
  bulk_runner.save
end
puts "results in #{bulk_runner.bulk_run_dir('')}"

# rubocop:enable Rails