rspec retry create issue

Gemfile

@@ -69,8 +69,10 @@ end
 
 group :test do
   gem 'faker', '~> 2.13'
+  gem 'octokit', '~> 7.1'
   gem 'pg_query', '~> 4.2.3'
   gem 'prosopite', '~> 1.3.3'
+  gem 'rspec-retry', github: 'rootstrap/rspec-retry', branch: 'add-intermittent-callback'
   gem 'shoulda-matchers', '~> 5.3'
   gem 'simplecov', '~> 0.22.0', require: false
   gem 'webmock', '~> 3.19'

Gemfile.lock

@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/rootstrap/rspec-retry.git
+  revision: 8266536fb6f4bc8d005c3863bcdfe8d7a6c9b203
+  branch: add-intermittent-callback
+  specs:
+    rspec-retry (0.6.2)
+      rspec-core (> 3.3)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -178,6 +186,11 @@ GEM
       railties (>= 5.0.0)
     faker (2.23.0)
       i18n (>= 1.8.11, < 2)
+    faraday (2.7.11)
+      base64
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-net_http (3.0.2)
     ffi (1.15.5)
     flipper (1.0.0)
       brow (~> 0.4.1)
@@ -288,6 +301,9 @@ GEM
     nokogiri (1.15.4)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
+    octokit (7.2.0)
+      faraday (>= 1, < 3)
+      sawyer (~> 0.9)
     oj (3.16.1)
     orm_adapter (0.5.0)
     pagy (6.0.4)
@@ -454,6 +470,9 @@ GEM
       sprockets (> 3.0)
       sprockets-rails
       tilt
+    sawyer (0.9.2)
+      addressable (>= 2.3.5)
+      faraday (>= 0.17.3, < 3)
     sendgrid (1.2.4)
       json
     sexp_processor (4.16.1)
@@ -528,6 +547,7 @@ DEPENDENCIES
   listen (~> 3.8)
   lograge (~> 0.13)
   newrelic_rpm (~> 9.4)
+  octokit (~> 7.1)
   oj (~> 3.16)
   pagy (~> 6.0)
   parallel_tests (~> 4.2)
@@ -543,6 +563,7 @@ DEPENDENCIES
   rails_best_practices (~> 1.20)
   reek (~> 6.1, >= 6.1.1)
   rspec-rails (~> 6.0)
+  rspec-retry!
   rspec_api_documentation (~> 6.1.0)
   rubocop (~> 1.55)
   rubocop-factory_bot (~> 2.23, >= 2.23.1)

spec/models/user_spec.rb

@@ -83,4 +83,12 @@
       end
     end
   end
+
+  describe 'flaky' do
+    $raise = [true, false]
+
+    it 'fails' do
+      raise if $raise.shift
+    end
+  end
 end

spec/rails_helper.rb

@@ -20,9 +20,14 @@
 require 'rspec/core'
 require 'spec_helper'
 require 'rspec/rails'
+require 'rspec/retry'
+require 'support/retry/message_formatter'
 
 ActiveRecord::Migration.maintain_test_schema!
-WebMock.disable_net_connect!(allow_localhost: true)
+WebMock.disable_net_connect!(
+  allow_localhost: true,
+  allow: ['api.github.com']
+)
 
 RSpec.configure do |config|
   config.render_views = true
@@ -46,6 +51,19 @@
 
   # Reset previous flipper instance
   config.before { Flipper.instance = nil }
+
+  # rspec-retry gem
+  # Show retry status in spec process
+  config.verbose_retry = true
+  # Print what reason forced the retry
+  config.display_try_failure_messages = true
+  # Try tests twice in the CI and once locally
+  config.default_retry_count = ENV.fetch('CI', false) ? 3 : 3
+  # Callback for intermittent tests
+  config.intermittent_callback = proc do |ex|
+    text = Retry::MessageFormatter.new(ex).to_s
+    Retry::PullRequestComment.new.comment(text)
+  end
 end
 
 Shoulda::Matchers.configure do |config|

spec/support/retry/message_formatter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Retry
+  class MessageFormatter
+    attr_reader :example
+
+    def initialize(example)
+      @example = example
+    end
+
+    def to_s
+      "\n*Failed test:*" \
+        "\n> Scenario: _#{description}_" \
+        "\n> File: #{error_location}" \
+        "\n> Seed: #{RSpec.configuration.seed}" \
+        "\n> Error: ```#{error}```\n"
+    end
+
+    private
+
+    def description
+      example.metadata[:full_description]
+    end
+
+    def error
+      example.execution_result.exception.to_s.split("\nDiff").first.presence ||
+        example.metadata[:retry_exceptions].first
+    end
+
+    def error_location
+      example.metadata[:location]
+    end
+  end
+end

spec/support/retry/pull_request_comment.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Retry
+  class PullRequestComment
+    attr_reader :repo, :pull_request_number, :client
+
+    def initialize
+      @repo = ENV.fetch('REPO', nil)
+      @pull_request_number = ENV.fetch('PULL_REQUEST_NUMBER', nil)
+      @github_token = ENV.fetch('GITHUB_TOKEN', nil)
+      @client = Octokit::Client.new(access_token: @github_token)
+    end
+
+    def comment(message)
+      pp "repo: #{repo}"
+      pp "Payload number: #{ENV.fetch('PAYLOAD_NUMBER', nil)}"
+      pp "pr: #{pull_request_number}"
+      pp message
+
+      options = { event: 'COMMENT', body: message }
+      client.create_pull_request_review(repo, pull_request_number, options)
+    rescue StandardError => e
+      p e.message
+    end
+  end
+end