openproject/script/github_pr_errors

#!/usr/bin/env ruby
# frozen_string_literal: true

require "rubygems"
require "bundler"
Bundler.setup(:default, :development)

require "colored2"
require "json"
require "optparse"
require "base64"
require "pathname"
require "pry"
require "time"
require "yaml"
require "httpx"

GITHUB_API_OPENPROJECT_PREFIX = "https://api.github.com/repos/opf/openproject"
GITHUB_HTML_OPENPROJECT_PREFIX = "https://github.com/opf/openproject"
RAILS_ROOT = Pathname.new(__dir__).dirname
EXCLUDED_JOB_NAMES = %w[eslint rubocop].freeze

if !ENV["GITHUB_USERNAME"]
  raise "Missing GITHUB_USERNAME env"
elsif !ENV["GITHUB_TOKEN"]
  raise "Missing GITHUB_TOKEN env, go to https://github.com/settings/tokens and create one with 'repo' access"
end

class Options
  DEFAULTS = {
    compact: false,
    display_rerun_info: false,
    display_images: false,
    failed_jobs_logs: [],
    no_cache: false,
    run_id: nil,
    job_id: nil,
    verbose: false
  }.freeze

  BANNER = <<~BANNER.freeze
    Usage: #{$0} [options] [url]

    Fetches rspec failures from last completed GitHub actions on current branch,
    and outputs them on standard output, one by line.

    If given an url, it will fetch the failures from the given url using the workflow
    id and the job id if present instead of the tip of the current branch.

    Information is printed on standard error to preserve standard output.

    Use this script with xargs to run failing specs locally:

        #{$0} | xargs --no-run-if-empty bundle exec rspec


    Options:
  BANNER

  class << self
    def options
      return @options if defined?(@options)

      @options = DEFAULTS.dup
      parse_options!
      @options
    end

    def method_missing(name, *)
      if DEFAULTS.key?(name)
        options[name]
      else
        super
      end
    end

    def respond_to_missing?(method_name, include_private = false)
      DEFAULTS.key?(name) || super
    end

    def parse_options!
      options.merge!(parse_args)
      options.merge!(parse_url(ARGV.first)) if ARGV.any?
    end

    def parse_args
      options = {}
      opt_parser = OptionParser.new do |parser|
        parser.banner = BANNER

        parser.on("-c", "--compact", "Output all failing rspec files on one line") do
          options[:compact] = true
        end

        parser.on("-d", "--display-rerun-info",
                  "Displays rspec rerun instructions like CI with seed") do
          options[:display_rerun_info] = true
        end

        parser.on("-f PATH", "--failed-job-log PATH",
                  "Use given file as failed job log instead of downloading from GitHub. Can be used multiple times.") do |path|
          options[:failed_jobs_logs] << path
        end

        parser.on("-i", "--images", "Download and display screenshots of failed feature tests") do
          options[:display_images] = true
        end

        parser.on("-n", "--no-cache", "Do not use cached replies from GitHub API calls") do
          options[:no_cache] = true
        end

        parser.on("-r RUN_ID", "--run-id RUN_ID", Integer,
                  "The workflow run id to use (in github url: actions/runs/{id})") do |value|
          options[:run_id] = value
        end

        parser.on("-h", "--help", "Prints this help") do
          puts parser
          exit
        end

        parser.on("-v", "--verbose",
                  "Print more information, mostly useful for debugging") do
          options[:verbose] = true
        end
      end
      opt_parser.parse!
      options
    end

    def parse_url(url)
      case url
      when %r{^https://github.com/opf/openproject/actions/runs/(\d+)(?:/job/(\d+)(?:\?.*)?)?$}
        run_id = $1.to_i
        job_id = $2&.to_i
        say_verbose("Extracted run id #{run_id}#{" and job id #{job_id}" if job_id} from #{url}")
        { run_id:, job_id: }
      else
        warn "Unrecognized url #{url}"
        exit 1
      end
    end
  end
end

def say_verbose(text)
  return unless Options.verbose

  warn "| #{text}"
end

# Returns current branch
def current_branch_name
  @current_branch_name ||= `git rev-parse --abbrev-ref HEAD`.strip
end

def github_url(path)
  if path.start_with?("http")
    path
  else
    "#{GITHUB_API_OPENPROJECT_PREFIX}/#{path}"
  end
end

def http
  HTTPX
    .plugin(:follow_redirects)
    .plugin(:basic_auth)
    .basic_auth(ENV.fetch("GITHUB_USERNAME"), ENV.fetch("GITHUB_TOKEN"))
end

def get_http(path)
  url = github_url(path)
  say_verbose("HTTP GET #{url}")

  response = http.get(url)

  say_verbose("HTTP Response #{response.status}")
  response.raise_for_status
  response.to_s
rescue HTTPX::HTTPError => e
  warn error_details(e)
  exit 1
rescue StandardError => e
  warn "Failed to perform API request GET #{url}: #{e}"
  exit 1
end

def error_details(error)
  response = error.response
  response_body = response.json

  parts = []
  parts << "Failed to perform API request GET #{response.uri}: #{error}"
  parts << "  #{response_body['message']}"
  parts << "  See #{response_body['documentation_url']}"
  parts += error.backtrace.map { "    #{it}" }
  parts.join("\n")
end

def get_json(path)
  JSON.parse(get_http(path))
end

def path_to_cache_key(path)
  path
    .gsub(/\?.*$/, "") # remove query parameter
    .gsub(/^#{GITHUB_API_OPENPROJECT_PREFIX}\/?/o, "") # remove https://.../
    .gsub(/\W/, "_") # transform non alphanum chars
end

def get_jobs(workflow_run)
  workflow_run["jobs_url"]
  cache_key = [
    path_to_cache_key(workflow_run["jobs_url"]),
    workflow_run["updated_at"].delete(":")
  ].join("_")
  cached(cache_key) { get_json(workflow_run["jobs_url"]) }
end

def get_log(job)
  cached("job_#{job['id']}.log") do
    get_http("actions/jobs/#{job['id']}/logs")
  end
end

def cached(unique_name)
  if Options.no_cache
    return yield
  end

  cached_file = RAILS_ROOT.join("tmp/github_pr_errors/#{unique_name}")
  if cached_file.file?
    say_verbose("Reading from cached file #{cached_file}")
    content = cached_file.read
    content.start_with?("---") ? YAML::load(content) : content
  else
    content = yield
    say_verbose("Writing to cached file #{cached_file}")
    cached_file.dirname.mkpath
    cached_file.write(content.is_a?(String) ? content : YAML::dump(content))
    content
  end
end

def last_with_status(workflow_runs, status)
  workflow_runs
    .select { |entry| entry["status"] == status }
    .max_by { |entry| entry["run_number"] }
end

def get_last_workflow_run(branch_name)
  test_workflow_runs =
    get_json("actions/runs?branch=#{CGI.escape(branch_name)}")
      .then { |response| response["workflow_runs"] }
      .select { |entry| entry["name"] == "Test suite" }

  last_completed = last_with_status(test_workflow_runs, "completed")
  last_in_progress = last_with_status(test_workflow_runs, "in_progress")

  last_in_progress || last_completed or raise "No workflow run found for branch #{branch_name}"
end

def get_workflow_run(run_id)
  if run_id
    warn "Looking for the workflow run with id #{run_id.to_s.bold}"
    get_json("actions/runs/#{CGI.escape(run_id.to_s)}")
  else
    warn "Looking for the last 'Test suite' workflow run in current branch #{current_branch_name.bold}"
    get_last_workflow_run(current_branch_name)
  end
end

class Report
  attr_accessor :errors,
                :failures_explanation,
                :head_branch,
                :head_sha,
                :commit_message,
                :run_started_at,
                :merge_branch_sha
end

class Error
  attr_accessor :location, :page_html, :page_screenshot, :tests_group, :loading_error
end

# rubocop:disable Layout/LineLength
# Looks like this in the job log:
# Process 28: TEST_ENV_NUMBER=28 RUBYOPT=-I/usr/local/bundle/bundler/gems/turbo_tests-3148ae6c3482/lib -r/usr/local/bundle/gems/bundler-2.5.23/lib/bundler/setup -W0 RSPEC_SILENCE_FILTER_ANNOUNCEMENTS=1 /usr/local/bundle/gems/bundler-2.5.23/exe/bundle exec rspec --seed 52674 --format TurboTests::JsonRowsFormatter --out tmp/test-pipes/subprocess-28 --format ParallelTests::RSpec::RuntimeLogger --out spec/support/turbo_runtime_features.log spec/features/api_docs/index_spec.rb spec/features/custom_fields/reorder_options_spec.rb spec/features/projects/projects_portfolio_spec.rb spec/features/projects/template_spec.rb spec/features/versions/edit_spec.rb spec/features/work_packages/details/markdown/description_editor_spec.rb spec/features/work_packages/table/hierarchy/hierarchy_parent_below_spec.rb spec/features/work_packages/table/inline_create/inline_create_refresh_spec.rb spec/features/work_packages/table/invalid_query_spec.rb spec/features/work_packages/tabs/activity_revisions_spec.rb
# rubocop:enable Layout/LineLength
class TestsGroup
  attr_accessor :test_env_number, :seed, :files

  def initialize
    @files = []
  end

  def include_error?(error)
    return false if error.location.nil?

    files.any? { |file| error.location.include?(file) }
  end

  def inspect
    "#<#{self.class} @test_env_number=#{test_env_number} @seed=#{seed} (#{files.count} files)>"
  end
end

class JobErrorsFinder
  SPEC_FAILURES_PATTERN = %r{^\S+ rspec (\S+) #.+$}
  SPEC_LOADING_ERRORS_PATTERN = %r{^\S+ An error occurred while loading (\S+)\.\r?$}
  SCREENSHOT_PATTERN = /\{"message":"Screenshot captured for failed feature test"[^\n]+$/
  TESTS_GROUP_PATTERN = /Process \d+: TEST_ENV_NUMBER=\d+ [^\n]+$/
  BRANCH_MERGE_PATTERN = /Merge \w{40} into (\w{40})$/

  attr_reader :failures_explanation, :merge_branch_sha

  def self.scan_logs(report, logs)
    finder = new
    logs.each do |log|
      finder.scan_log(log)
    end
    report.errors = finder.errors
    report.failures_explanation = finder.failures_explanation
    report.merge_branch_sha = finder.merge_branch_sha
    report
  end

  def scan_log(log)
    find_failures(log)
    find_failures_explanation(log)
    find_loading_errors(log)
    find_screenshots(log)
    find_tests_groups(log)
    find_merge_branch_info(log)
  end

  def errors
    @errors.values
  end

  protected

  def initialize
    @errors = {}
  end

  def create_error(location)
    return if location.nil?

    error = Error.new
    error.location = location
    @errors[location] ||= error
  end

  def with_matching_error(location: nil, id: nil)
    error = @errors[id] || @errors[location]
    yield error if error && block_given?
    error
  end

  def find_failures(log)
    log.scan(SPEC_FAILURES_PATTERN)
      .flatten
      .uniq
      .sort
      .each do |rerun_location|
        create_error(rerun_location)
      end
  end

  def find_failures_explanation(log)
    explanations = []
    log.split("\n").each do |line|
      if line.end_with?("Failures:") .. line.end_with?("Failed examples:")
        explanations << line
      end
    end
    explanations.map! { it[29..] } # Remove leading timestamp (like "2024-02-05T08:37:54.5175930Z")
    explanations.reject! do |line|
      line == "Failures:" ||
        line == "Failed examples:" ||
        line.include?("gems/rspec-retry-") ||
        line.include?("gems/webmock-")
    end
    @failures_explanation = explanations.join("\n")
  end

  def find_loading_errors(log)
    log.scan(SPEC_LOADING_ERRORS_PATTERN)
      .flatten
      .uniq
      .sort
      .each do |location|
        error = create_error(location)
        error.loading_error = true
      end
  end

  def find_screenshots(log)
    log.scan(SCREENSHOT_PATTERN)
      .map { JSON.parse it }
      .each do |screenshot_info|
        id = screenshot_info["test_id"]
        location = screenshot_info["test_location"]
        with_matching_error(location:, id:) do |error|
          error.page_html = screenshot_info["html"]
          error.page_screenshot = screenshot_info["image"]
        end
      end
  end

  def find_tests_groups(log)
    tests_groups = log
      .scan(TESTS_GROUP_PATTERN)
      .flatten
      .map { build_tests_group_from_command(it) }

    errors.each do |error|
      error.tests_group = tests_groups.find { it.include_error?(error) }
    end
  end

  def find_merge_branch_info(log)
    merge_branch_sha = log.scan(BRANCH_MERGE_PATTERN).flatten.first
    @merge_branch_sha = merge_branch_sha if merge_branch_sha
  end

  def build_tests_group_from_command(line)
    tests_group = TestsGroup.new
    parts = line.split
    while parts.any?
      case part = parts.shift
      when /^TEST_ENV_NUMBER=/
        tests_group.test_env_number = part.delete_prefix("TEST_ENV_NUMBER=")
      when "--seed"
        tests_group.seed = parts.shift
      when /_spec.rb$/
        tests_group.files << part
      end
    end
    tests_group
  end
end

class Formatter
  def initialize(compact: false)
    @compact = compact
  end

  def compact?
    @compact
  end

  def display_workflow_run_info(report, workflow_run)
    report.head_branch = workflow_run["head_branch"]
    report.head_sha = workflow_run["head_sha"]
    report.commit_message = commit_message(workflow_run)
    report.run_started_at = Time.parse(workflow_run["run_started_at"]).utc
    warn "  Branch: #{report.head_branch.bold}"
    warn "  Commit SHA: #{report.head_sha.bold}"
    warn "  Commit message: #{report.commit_message.bold}"
    warn "  Last attempted run started at: #{report.run_started_at.localtime.to_s.bold}"
    display_pull_request_info(workflow_run)
  end

  def display_workflow_status(workflow_run)
    warn "  Run attempt: #{workflow_run['run_attempt']}"
    warn "  #{status_line(workflow_run)}"
  end

  def display_job_status(job)
    warn "    #{status_line(job)}"
  end

  def display_report(report)
    display_failures_explanation(report.failures_explanation)
    display_errors(report.errors)
    display_checkout_test_commit_information(report) if Options.display_rerun_info
  end

  def display_failures_explanation(failures_explanation)
    return if failures_explanation.nil? || failures_explanation.empty?

    warn "Failures explanation:".bold
    warn failures_explanation
    warn ""
  end

  def display_errors(errors)
    if errors.empty?
      warn "No rspec errors found :-/"
    elsif compact?
      display_errors_compact(errors)
    else
      display_errors_detailed(errors)
    end
  end

  private

  def display_errors_compact(errors)
    puts errors.map { escaped_location(it) }.join(" ")
  end

  def display_errors_detailed(errors)
    if Options.display_rerun_info
      errors
        .sort_by { |error| [error.tests_group.test_env_number.to_i, error.location] }
        .group_by(&:tests_group)
        .each do |tests_group, tests_group_errors|
          display_tests_group_info(tests_group)
          tests_group_errors.each { display_error(it) }
          display_tests_group_rerun_commands(tests_group)
        end
    else
      errors
        .sort_by(&:location)
        .each { display_error(it) }
    end
  end

  def display_error(error)
    puts escaped_location(error)
    display_error_attribute("loading error", error.loading_error)
    display_error_attribute("html", error.page_html)
    display_error_screenshot(error.page_screenshot)
  end

  def display_error_screenshot(screenshot_url)
    return unless screenshot_url

    display_error_attribute("screenshot", screenshot_url)
    if Options.display_images
      if screenshot_url.start_with?("/")
        # external pull requests do not have access to AWS credentials to upload screenshots
        warn "      (cannot display screenshot: it is local file stored on the runner)".yellow
        return
      end
      begin
        Open3.popen2e("imgcat", "--width", "50%", "--url", screenshot_url) do |stdin, stdout_and_stderr, _thread|
          stdin.close
          warn stdout_and_stderr.read
        end
      rescue Errno::ENOENT => e
        warn e
        warn "To display image inline, you need to have a compatible terminal like iTerm2 or Konsole, and the 'imgcat' command installed."
        warn "See https://iterm2.com/documentation-images.html to know more."
      end
    end
  end

  def display_error_attribute(name, value)
    return unless value

    warn [
      "    ",
      "↳".blue.bold,
      " ",
      name.blue,
      ": ",
      value.to_s.blue
    ].join
  end

  def display_checkout_test_commit_information(report)
    if report.merge_branch_sha
      warn <<~INSTRUCTIONS.white.dark
        GitHub Action run merged #{report.merge_branch_sha} into #{report.head_sha} before running.

        To be with the exact same source files as CI, use these commands (warning: stash your modifications first):
          git checkout -B repro_ci_failures #{report.head_sha}
          git merge --no-edit --no-verify #{report.merge_branch_sha}
      INSTRUCTIONS
    else
      warn <<~INSTRUCTIONS.white.dark
        To be with the exact same source files as CI, use this command:
          git checkout #{report.head_sha}
      INSTRUCTIONS
    end
    warn <<~INSTRUCTIONS.white.dark
      Then run a test group with the rspec command above.
    INSTRUCTIONS
  end

  def display_tests_group_info(tests_group)
    return unless tests_group

    warn "Tests group ##{tests_group.test_env_number}".bold
  end

  def display_tests_group_rerun_commands(tests_group)
    return unless tests_group

    warn "To run the tests group in the same order as CI, use this command:".white.dark
    warn "CI=true bundle exec rspec --seed #{tests_group.seed} #{tests_group.files.join(' ')}".white.dark
    warn ""
  end

  def display_pull_request_info(workflow_run)
    return unless workflow_run["event"] == "pull_request"

    if pr = workflow_run["pull_requests"].first
      pr_number = "##{pr['number']}"
      pr_html_url = "#{GITHUB_HTML_OPENPROJECT_PREFIX}/pull/#{pr['number']}"
      pr_display_title = "#{workflow_run['display_title']} #{pr_number.white.dark} #{pr_html_url.white.dark}"
      warn "  Pull Request: #{pr_display_title} "
    else
      warn "  Pull Request: not found; perhaps it is already merged or closed?"
    end
  end

  def commit_message(workflow_run)
    workflow_run["head_commit"]
      .then { |commit| commit["message"] }
      .then { |message| message.split("\n", 2).first }
  end

  def status_icon(job)
    case job["status"]
    when "queued", "in_progress"
      "●".yellow
    else
      case job["conclusion"]
      when "success"
        "✓".green
      when "failure"
        "✗".red
      else
        "-"
      end
    end
  end

  def status_url(job)
    return if job["conclusion"] == "success"

    job["html_url"].white.dark
  end

  def status_line(job)
    [
      "#{status_icon(job)} #{job['name']}: #{job['conclusion'] || job['status']}",
      status_url(job)
    ].compact.join("  ")
  end

  def escaped_location(error)
    "'#{error.location}'"
  end
end

def get_jobs_from_job_id(job_id)
  warn "  Looking for the job with id #{job_id.to_s.bold}"

  # no need to fetch anything as we only use the job id to get the logs
  # mock the same structure as the jobs response
  [{ "id" => job_id }]
end

def get_failing_jobs_from_workflow_last_attempt(workflow_run, formatter)
  get_jobs(workflow_run)
    .then { |jobs_response| jobs_response["jobs"] }
    .sort_by { it["name"] }
    .each { |job| formatter.display_job_status(job) }
    .select { it["conclusion"] == "failure" }
    .reject { EXCLUDED_JOB_NAMES.include?(it["name"]) }
end

def get_relevant_jobs(job_id, workflow_run, formatter)
  if job_id
    get_jobs_from_job_id(job_id)
  else
    formatter.display_workflow_status(workflow_run)
    get_failing_jobs_from_workflow_last_attempt(workflow_run, formatter)
  end
end

def get_failed_jobs_logs_from_github(report, formatter)
  workflow_run = get_workflow_run(Options.run_id)

  formatter.display_workflow_run_info(report, workflow_run)

  job_logs = get_relevant_jobs(Options.job_id, workflow_run, formatter)
    .map { |job| get_log(job) }

  [job_logs, job_logs.any? ? "error" : workflow_run["status"]]
end

def get_failed_jobs_logs_from_args
  Options.failed_jobs_logs.map { |path| File.read(path) }
end

def get_failed_jobs_logs(report, formatter)
  if Options.failed_jobs_logs.any?
    [get_failed_jobs_logs_from_args, failed_jobs_logs.none? ? "completed" : "error"]
  else
    get_failed_jobs_logs_from_github(report, formatter)
  end
end

##########

# report stores all found information about the workflow run
report = Report.new
formatter = Formatter.new(compact: Options.compact)

failed_jobs_logs, status = get_failed_jobs_logs(report, formatter)

JobErrorsFinder.scan_logs(report, failed_jobs_logs)

case status
when "completed"
  warn "All jobs successful 🎉"
when "in_progress"
  # NOOP
when "error"
  formatter.display_report(report)
end