add Git::SearchCommits callable that sends backend API request to che…

…ck which commits are already known to the backend

lib/datadog/ci/ext/environment.rb

@@ -3,6 +3,8 @@
 require_relative "git"
 require_relative "environment/extractor"
 
+require_relative "../utils/git"
+
 module Datadog
   module CI
     module Ext
@@ -21,8 +23,6 @@ module Environment
         TAG_NODE_NAME = "ci.node.name"
         TAG_CI_ENV_VARS = "_dd.ci.env_vars"
 
-        HEX_NUMBER_REGEXP = /[0-9a-f]{40}/i.freeze
-
         module_function
 
         def tags(env)
@@ -57,24 +57,19 @@ def validate_repository_url(repo_url)
         end
 
         def validate_git_sha(git_sha)
-          message = "DD_GIT_COMMIT_SHA must be a full-length git SHA."
+          return if Utils::Git.valid_commit_sha?(git_sha)
 
-          if git_sha.nil? || git_sha.empty?
-            message += " No value was set and no SHA was automatically extracted."
-            Datadog.logger.error(message)
-            return
-          end
+          message = "DD_GIT_COMMIT_SHA must be a full-length git SHA."
 
-          if git_sha.length < Git::SHA_LENGTH
-            message += " Expected SHA length #{Git::SHA_LENGTH}, was #{git_sha.length}."
-            Datadog.logger.error(message)
-            return
+          message += if git_sha.nil? || git_sha.empty?
+            " No value was set and no SHA was automatically extracted."
+          elsif git_sha.length < Git::SHA_LENGTH
+            " Expected SHA length #{Git::SHA_LENGTH}, was #{git_sha.length}."
+          else
+            " Expected SHA to be a valid HEX number, got #{git_sha}."
           end
 
-          unless HEX_NUMBER_REGEXP.match?(git_sha)
-            message += " Expected SHA to be a valid HEX number, got #{git_sha}."
-            Datadog.logger.error(message)
-          end
+          Datadog.logger.error(message)
         end
       end
     end

lib/datadog/ci/ext/transport.rb

@@ -27,6 +27,7 @@ module Transport
         TEST_COVERAGE_INTAKE_PATH = "/api/v2/citestcov"
 
         DD_API_HOST_PREFIX = "api"
+
         DD_API_SETTINGS_PATH = "/api/v2/libraries/tests/services/setting"
         DD_API_SETTINGS_TYPE = "ci_app_test_service_libraries_settings"
         DD_API_SETTINGS_RESPONSE_DIG_KEYS = %w[data attributes].freeze
@@ -36,6 +37,8 @@ module Transport
         DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
         DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
 
+        DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"
+
         CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
         CONTENT_TYPE_JSON = "application/json"
         CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data"

lib/datadog/ci/git/search_commits.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "json"
+require "set"
+
+require_relative "../ext/transport"
+require_relative "../utils/git"
+
+module Datadog
+  module CI
+    module Git
+      class SearchCommits
+        class ApiError < StandardError; end
+
+        attr_reader :api
+
+        def initialize(api:)
+          @api = api
+        end
+
+        def call(repository_url, commits)
+          raise ApiError, "test visibility API is not configured" if api.nil?
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH,
+            payload: request_payload(repository_url, commits)
+          )
+          raise ApiError, "Failed to search commits: #{http_response.body}" unless http_response.ok?
+
+          response_payload = parse_json_response(http_response)
+          extract_commits(response_payload)
+        end
+
+        private
+
+        def request_payload(repository_url, commits)
+          {
+            meta: {
+              repository_url: repository_url
+            },
+            data: commits.filter_map do |commit|
+              next unless Utils::Git.valid_commit_sha?(commit)
+
+              {
+                id: commit,
+                type: "commit"
+              }
+            end
+          }.to_json
+        end
+
+        def parse_json_response(http_response)
+          JSON.parse(http_response.payload)
+        rescue JSON::ParserError => e
+          raise ApiError, "Failed to parse search commits response: #{e}. Payload was: #{http_response.payload}"
+        end
+
+        def extract_commits(response_payload)
+          result = Set.new
+
+          response_payload.fetch("data").each do |commit_json|
+            raise ApiError, "Invalid commit type response #{commit_json}" unless commit_json["type"] == "commit"
+
+            commit_sha = commit_json["id"]
+            raise ApiError, "Invalid commit SHA response #{commit_sha}" unless Utils::Git.valid_commit_sha?(commit_sha)
+
+            result.add(commit_sha)
+          end
+
+          result
+        rescue KeyError => e
+          raise ApiError, "Malformed search commits response: #{e}. Payload was: #{response_payload}"
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/utils/git.rb

@@ -7,6 +7,12 @@ module Datadog
   module CI
     module Utils
       module Git
+        def self.valid_commit_sha?(sha)
+          return false if sha.nil?
+
+          sha.match?(/\A[0-9a-f]{40}\Z/) || sha.match?(/\A[0-9a-f]{64}\Z/)
+        end
+
         def self.normalize_ref(ref)
           return nil if ref.nil?
 

sig/datadog/ci/ext/environment.rbs

@@ -27,8 +27,6 @@ module Datadog
 
         TAG_CI_ENV_VARS: String
 
-        HEX_NUMBER_REGEXP: Regexp
-
         PROVIDERS: ::Array[Array[String | Symbol]]
 
         def self?.tags: (untyped env) -> Hash[String, String]

sig/datadog/ci/ext/transport.rbs

@@ -32,7 +32,7 @@ module Datadog
 
         DD_API_HOST_PREFIX: "api"
 
-        DD_API_SETTINGS_PATH: "/api/v2/ci/libraries/tests/services/setting"
+        DD_API_SETTINGS_PATH: "/api/v2/libraries/tests/services/setting"
 
         DD_API_SETTINGS_TYPE: "ci_app_test_service_libraries_settings"
 
@@ -48,6 +48,8 @@ module Datadog
 
         DD_API_SETTINGS_RESPONSE_DEFAULT: Hash[String, untyped]
 
+        DD_API_GIT_SEARCH_COMMITS_PATH: "/api/v2/git/repository/search_commits"
+
         CONTENT_TYPE_MESSAGEPACK: "application/msgpack"
 
         CONTENT_TYPE_JSON: "application/json"

sig/datadog/ci/git/search_commits.rbs

@@ -0,0 +1,26 @@
+module Datadog
+  module CI
+    module Git
+      class SearchCommits
+        @api: Datadog::CI::Transport::Api::Base?
+
+        attr_reader api: Datadog::CI::Transport::Api::Base?
+
+        class ApiError < StandardError
+        end
+
+        def initialize: (api: Datadog::CI::Transport::Api::Base?) -> void
+
+        def call: (String repository_url, Array[String] commits) -> Set[String]
+
+        private
+
+        def request_payload: (String repository_url, Array[String] commits) -> String
+
+        def parse_json_response: (Datadog::Core::Transport::Response response) -> Hash[String, untyped]
+
+        def extract_commits: (Hash[String, untyped] response) -> Set[String]
+      end
+    end
+  end
+end

sig/datadog/ci/utils/git.rbs

@@ -5,6 +5,8 @@ module Datadog
         @root: String?
         @repository_name: String?
 
+        def self.valid_commit_sha?: (String? sha) -> bool
+
         def self.normalize_ref: (String? name) -> String?
 
         def self.is_git_tag?: (String? ref) -> bool

spec/datadog/ci/git/search_commits_spec.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require_relative "../../../../lib/datadog/ci/git/search_commits"
+
+RSpec.describe Datadog::CI::Git::SearchCommits do
+  let(:api) { double("api") }
+  subject(:search_commits) { described_class.new(api: api) }
+
+  describe "#call" do
+    let(:repository_url) { "https://datadoghq.com/git/test.git" }
+    let(:commits) { ["c7f893648f656339f62fb7b4d8a6ecdf7d063835"] }
+
+    context "when the API is not configured" do
+      let(:api) { nil }
+
+      it "raises an error" do
+        expect { search_commits.call(repository_url, commits) }
+          .to raise_error(Datadog::CI::Git::SearchCommits::ApiError, "test visibility API is not configured")
+      end
+    end
+
+    context "when the API is configured" do
+      before do
+        allow(api).to receive(:api_request).and_return(http_response)
+      end
+
+      context "when the API request fails" do
+        let(:http_response) { double("http_response", ok?: false, body: "error message") }
+
+        it "raises an error" do
+          expect { search_commits.call(repository_url, commits) }
+            .to raise_error(Datadog::CI::Git::SearchCommits::ApiError, "Failed to search commits: error message")
+        end
+      end
+
+      context "when the API request is successful" do
+        let(:http_response) { double("http_response", ok?: true, payload: response_payload) }
+        let(:response_payload) do
+          {
+            "data" => [
+              {
+                "id" => "c7f893648f656339f62fb7b4d8a6ecdf7d063835",
+                "type" => "commit"
+              }
+            ]
+          }.to_json
+        end
+
+        it "returns the list of commit SHAs" do
+          expect(api).to receive(:api_request).with(
+            path: Datadog::CI::Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH,
+            payload: "{\"meta\":{\"repository_url\":\"https://datadoghq.com/git/test.git\"},\"data\":[{\"id\":\"c7f893648f656339f62fb7b4d8a6ecdf7d063835\",\"type\":\"commit\"}]}"
+          ).and_return(http_response)
+
+          expect(search_commits.call(repository_url, commits)).to eq(Set.new(["c7f893648f656339f62fb7b4d8a6ecdf7d063835"]))
+        end
+
+        context "when the request contains an invalid commit SHA" do
+          let(:commits) { ["INVALID_SHA", "c7f893648f656339f62fb7b4d8a6ecdf7d063835"] }
+
+          it "does not include the invalid commit SHA in the request" do
+            expect(api).to receive(:api_request).with(
+              path: Datadog::CI::Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH,
+              payload: "{\"meta\":{\"repository_url\":\"https://datadoghq.com/git/test.git\"},\"data\":[{\"id\":\"c7f893648f656339f62fb7b4d8a6ecdf7d063835\",\"type\":\"commit\"}]}"
+            ).and_return(http_response)
+
+            expect(search_commits.call(repository_url, commits)).to eq(Set.new(["c7f893648f656339f62fb7b4d8a6ecdf7d063835"]))
+          end
+        end
+
+        context "when the response contains an invalid commit type" do
+          let(:response_payload) do
+            {
+              "data" => [
+                {
+                  "id" => "c7f893648f656339f62fb7b4d8a6ecdf7d063835",
+                  "type" => "invalid"
+                }
+              ]
+            }.to_json
+          end
+
+          it "raises an error" do
+            expect { search_commits.call(repository_url, commits) }
+              .to raise_error(
+                Datadog::CI::Git::SearchCommits::ApiError,
+                "Invalid commit type response {\"id\"=>\"c7f893648f656339f62fb7b4d8a6ecdf7d063835\", \"type\"=>\"invalid\"}"
+              )
+          end
+        end
+
+        context "when the response contains an invalid commit SHA" do
+          let(:response_payload) do
+            {
+              "data" => [
+                {
+                  "id" => "INVALID_SHA",
+                  "type" => "commit"
+                }
+              ]
+            }.to_json
+          end
+
+          it "raises an error" do
+            expect { search_commits.call(repository_url, commits) }
+              .to raise_error(Datadog::CI::Git::SearchCommits::ApiError, "Invalid commit SHA response INVALID_SHA")
+          end
+        end
+
+        context "when the response is not a valid JSON" do
+          let(:response_payload) { "invalid json" }
+
+          it "raises an error" do
+            expect { search_commits.call(repository_url, commits) }
+              .to raise_error(
+                Datadog::CI::Git::SearchCommits::ApiError,
+                "Failed to parse search commits response: unexpected token at 'invalid json'. Payload was: invalid json"
+              )
+          end
+        end
+
+        context "when the response is missing the data key" do
+          let(:response_payload) { {}.to_json }
+
+          it "raises an error" do
+            expect { search_commits.call(repository_url, commits) }
+              .to raise_error(
+                Datadog::CI::Git::SearchCommits::ApiError,
+                "Malformed search commits response: key not found: \"data\". Payload was: {}"
+              )
+          end
+        end
+
+        context "when the response is missing the commit type" do
+          let(:response_payload) do
+            {
+              "data" => [
+                {
+                  "id" => "c7f893648f656339f62fb7b4d8a6ecdf7d063835"
+                }
+              ]
+            }.to_json
+          end
+
+          it "raises an error" do
+            expect { search_commits.call(repository_url, commits) }
+              .to raise_error(
+                Datadog::CI::Git::SearchCommits::ApiError,
+                "Invalid commit type response {\"id\"=>\"c7f893648f656339f62fb7b4d8a6ecdf7d063835\"}"
+              )
+          end
+        end
+      end
+    end
+  end
+end