add snippet upload

ahrefs · Feb 6, 2025 · 02d8677 · 02d8677
1 parent 4924b24
commit 02d8677
Show file tree

Hide file tree

Showing 5 changed files with 216 additions and 5 deletions.
diff --git a/lib/action.ml b/lib/action.ml
@@ -372,7 +372,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) (Buildkite_api :
         | None -> Lwt.return_unit
         | Some handler ->
         try
-          handler res;
+          let%lwt () = handler res in
           Lwt.return_unit
         with exn -> handler_error (Printexc.to_string exn))
       | Ok None -> Lwt.return_unit

diff --git a/lib/dune b/lib/dune
@@ -18,6 +18,7 @@
   ptime.clock
   re2
   sexplib0
+  slack.lib
   uri
   yojson)
  (preprocess

diff --git a/lib/slack.atd b/lib/slack.atd
@@ -158,6 +158,33 @@ type permalink_res = {
   ?error: string nullable;
 }
 
+type get_upload_url_res = {
+  ok: bool;
+  upload_url: string;
+  file_id: string;
+  ?error: string nullable;
+}
+
+type file_v2 = {
+  id: string;
+  ?title : string nullable
+}
+
+type files_v2 = file_v2 list
+
+type complete_upload_ext_req = {
+  files: files_v2;
+  ?channel_id : string nullable;
+  ?initial_comment: string nullable;
+  ?thread_ts: timestamp nullable
+}
+
+type complete_upload_ext_res = {
+  ok: bool;
+  files: files_v2;
+  ?error: string nullable;
+}
+
 type ('ok, 'err) http_response <ocaml predef module="Result" t="t"> = [
   | Ok of 'ok
   | Error of 'err

diff --git a/lib/slack.ml b/lib/slack.ml
@@ -86,9 +86,12 @@ let thread_state_handler ~ctx ~channel ~repo_url ~html_url action (response : Sl
   match action with
   | `Add ->
     State.add_thread_if_new ctx.Context.state ~repo_url ~pr_url:html_url
-      { cid = response.channel; channel; ts = response.ts }
-  | `Delete -> State.delete_thread ctx.state ~repo_url ~pr_url:html_url
-  | `Noop -> ()
+      { cid = response.channel; channel; ts = response.ts };
+    Lwt.return_unit
+  | `Delete ->
+    State.delete_thread ctx.state ~repo_url ~pr_url:html_url;
+    Lwt.return_unit
+  | `Noop -> Lwt.return_unit
 
 let thread_state_action_of_pr_action : pr_action -> _ = function
   | Opened | Ready_for_review | Labeled
@@ -418,7 +421,96 @@ let generate_status_notification ?slack_user_id ?failed_steps ~(ctx : Context.t)
         failed_steps
   in
   let attachment = { empty_attachments with mrkdwn_in = Some [ "fields"; "text" ]; color = Some color_info; text } in
-  make_message ~text:summary ~attachments:[ attachment ] ~channel:(Status_notification.to_slack_channel channel) ()
+  let handler ({ channel; _ } : Slack_t.post_message_res) =
+    match is_failed_build_notification with
+    | false -> Lwt.return_unit
+    | true ->
+      let logs = Text.[ log1 ] in
+      let secrets = Context.get_secrets_exn ctx in
+      (match secrets.slack_access_token with
+      | None -> failwith " failed to retrieve Slack access token to upload files"
+      | Some access_token ->
+        let http_request ?headers ?body meth path =
+          let setup h =
+            Curl.set_followlocation h true;
+            Curl.set_maxredirs h 1
+          in
+          match%lwt Web.http_request_lwt ~setup ~ua:"monorobot" ~verbose:true ?headers ?body meth path with
+          | `Ok s -> Lwt.return @@ Ok s
+          | `Error e -> Lwt.return @@ Error e
+        in
+        let slack_api_request ?headers ?body meth url read =
+          match%lwt http_request ?headers ?body meth url with
+          | Error e -> Lwt.return_error (query_error_msg url e)
+          | Ok s ->
+          match Slack_j.slack_response_of_string read s with
+          | res -> Lwt.return res
+          | exception exn -> Lwt.return_error (query_error_msg url (Exn.to_string exn))
+        in
+        let request_token_auth ~name ?headers ?body meth path read =
+          print_endline @@ sprintf "%s: starting request" name;
+          let headers = bearer_token_header access_token :: Option.default [] headers in
+          let url = sprintf "https://slack.com/api/%s" path in
+          match%lwt slack_api_request ?body ~headers meth url read with
+          | Ok res -> Lwt.return @@ Ok res
+          | Error e -> Lwt.return @@ fmt_error "%s: failure : %s" name e
+        in
+        let get_upload_url_external (filename, content) =
+          let data = Web.make_url_args [ "filename", filename; "length", Int.to_string (String.length content) ] in
+          print_endline @@ sprintf "data to upload file: %s" filename;
+          match%lwt
+            request_token_auth ~name:(sprintf "file.upload (%s)" filename) `GET
+              (sprintf "files.getUploadURLExternal?%s" data)
+              Slack_j.read_get_upload_url_res
+          with
+          | Error (s : string) -> failwith @@ sprintf "couldn't get get_upload_url for %s: %s" filename s
+          | Ok (res : Slack_t.get_upload_url_res) when res.ok = false ->
+            failwith @@ sprintf "bad request fetching get_upload_url for %s: %s" filename (Option.default "" res.error)
+          | Ok (res : Slack_t.get_upload_url_res) ->
+            (match%lwt http_request ~body:(`Raw ("text/plain", content)) `POST res.upload_url with
+            | Error e -> failwith @@ sprintf "failed uploading file for %s: %s" filename e
+            | Ok s ->
+              print_endline @@ sprintf "uploaded file for %s. Reply: %s" filename s;
+              Lwt.return res)
+        in
+        let complete_upload_external (files : (string * string) list) =
+          print_endline @@ sprintf "completing upload url for ";
+          let data : Slack_t.complete_upload_ext_req =
+            {
+              files = List.map (fun (id, title) -> { id; title = Some title }) files;
+              thread_ts = None;
+              channel_id = Some (Slack_channel.Ident.project channel);
+              initial_comment = None;
+            }
+          in
+          print_endline
+          @@ sprintf "================\ndata to upload req: %s\n=================="
+               (Slack_j.string_of_complete_upload_ext_req data);
+          let body = `Raw ("application/json", Slack_j.string_of_complete_upload_ext_req data) in
+          (* print_endline @@ sprintf "data to upload req: %s" data; *)
+          match%lwt
+            request_token_auth
+              ~name:(sprintf "files.completeUploadExternal ")
+              ~body `POST "files.completeUploadExternal" Slack_j.read_complete_upload_ext_res
+          with
+          | Error e -> failwith @@ sprintf "failed completing uploading file: %s" e
+          | Ok _ ->
+            print_endline @@ sprintf "uploaded files";
+            Lwt.return_unit
+        in
+        let%lwt files =
+          Lwt_list.map_p
+            (fun (filename, content) ->
+              let%lwt uploaded = get_upload_url_external (filename, content) in
+              Lwt.return (uploaded.file_id, filename))
+            logs
+        in
+        let%lwt _ = complete_upload_external files in
+        Lwt.return_unit)
+  in
+  make_message ~text:summary ~attachments:[ attachment ]
+    ~channel:(Status_notification.to_slack_channel channel)
+    ~reply_broadcast:false ~handler ()
 
 let generate_commit_comment_notification ~slack_match_func api_commit notification channel =
   let { commit; _ } = api_commit in

diff --git a/lib/text.ml b/lib/text.ml
@@ -0,0 +1,91 @@
+let log1 =
+  ( "log1.txt",
+    "_bk;t=1738172391664~~~ Running global environment hook\n\
+     _bk;t=1738172391665[90m$[0m /etc/buildkite-agent/hooks/environment\n\
+     _bk;t=1738172391711~~~ Preparing working directory\n\
+     _bk;t=1738172391712[90m$[0m cd /home/user/builds/buildbot104/ahrefs/monorepo\n\
+     _bk;t=1738172391780[90m# Host \"git.ahrefs.com\" already in list of known hosts at \
+     \"/home/user/.ssh/known_hosts\"[0m\n\
+     _bk;t=1738172391782[90m$[0m git submodule foreach --recursive \"git clean -ffdx -e _build -e _opam -e .opam \
+     -e node_modules\"\n\
+     _bk;t=1738172391823Entering 'backend/cdr/cachetools'\n\
+     _bk;t=1738172391833[90m$[0m git clean -ffdx -e _build -e _opam -e .opam -e node_modules\n\
+     _bk;t=1738172391979Removing typescript\n\
+     _bk;t=1738172391980[90m$[0m git fetch -v --prune -- origin 61a85c041436cfd2fbc247978bf212383842fe69\n\
+     _bk;t=1738172398066From git.ahrefs.com:ahrefs/monorepo\n\
+     _bk;t=1738172398066 * branch                    61a85c041436cfd2fbc247978bf212383842fe69 -> FETCH_HEAD\n\
+     _bk;t=1738172398099[90m$[0m git checkout -f 61a85c041436cfd2fbc247978bf212383842fe69\n\
+     _bk;t=1738172399552Previous HEAD position was 19fcfa0cfaf infra: ch: mv `core::clickhouse::dashboard` to `devsg`\n\
+     _bk;t=1738172399554HEAD is now at 61a85c04143 Revert \"infra: add aa_wc_admin to publish\"\n\
+     _bk;t=1738172399569[90m# Git submodules detected[0m\n\
+     _bk;t=1738172399569[90m$[0m git submodule sync --recursive\n\
+     _bk;t=1738172399611Synchronizing submodule url for 'backend/cdr/cachetools'\n\
+     _bk;t=1738172399618[90m$[0m git submodule update --init --recursive --force\n\
+     _bk;t=1738172399669Submodule path 'backend/cdr/cachetools': checked out \
+     'c9700a0b29a8d220c1afdb867e9eed9357cd3351'\n\
+     _bk;t=1738172399674[90m$[0m git submodule foreach --recursive \"git reset --hard\"\n\
+     _bk;t=1738172399713Entering 'backend/cdr/cachetools'\n\
+     _bk;t=1738172399718HEAD is now at c9700a0 use long for internal counters\n\
+     _bk;t=1738172399723[90m# Cleaning again to catch any post-checkout changes[0m\n\
+     _bk;t=1738172399723[90m$[0m git clean -ffdx -e _build -e _opam -e .opam -e node_modules\n\
+     _bk;t=1738172399865[90m$[0m git submodule foreach --recursive \"git clean -ffdx -e _build -e _opam -e .opam \
+     -e node_modules\"\n\
+     _bk;t=1738172399908Entering 'backend/cdr/cachetools'\n\
+     _bk;t=1738172399917[90m# Checking to see if git commit information needs to be sent to Buildkite...[0m\n\
+     _bk;t=1738172399917[90m$[0m buildkite-agent meta-data exists buildkite:git:commit\n\
+     _bk;t=1738172400612[90m# Sending Git commit information back to Buildkite[0m\n\
+     _bk;t=1738172400617[90m$[0m buildkite-agent meta-data set buildkite:git:commit < /dev/stdin\n\
+     _bk;t=1738172400646[38;5;48m2025-01-29 17:40:00 INFO  [0m [0mReading meta-data value from STDIN[0m\n\
+     _bk;t=1738172401344~~~ Running commands\n\
+     _bk;t=1738172401344[90m$[0m /home/user/buildkite_pipeline.sh | buildkite-agent pipeline upload\n\
+     _bk;t=1738172402024[38;5;48m2025-01-29 17:40:02 INFO  [0m [0mReading pipeline config from STDIN[0m\n\
+     _bk;t=1738172402027[38;5;48m2025-01-29 17:40:02 INFO  [0m [0mUpdating BUILDKITE_COMMIT to \
+     \"61a85c041436cfd2fbc247978bf212383842fe69\"[0m\n\
+     _bk;t=1738172404182[38;5;48m2025-01-29 17:40:04 INFO  [0m [0mSuccessfully uploaded and parsed pipeline \
+     config[0m\n" )
+
+let log2 =
+  ( "log2.txt",
+    "_bk;t=1738172391664~~~ Running global environment hook\n\
+     _bk;t=1738172391665[90m$[0m /etc/buildkite-agent/hooks/environment\n\
+     _bk;t=1738172391711~~~ Preparing working directory\n\
+     _bk;t=1738172391712[90m$[0m cd /home/user/builds/buildbot104/ahrefs/monorepo\n\
+     _bk;t=1738172391780[90m# Host \"git.ahrefs.com\" already in list of known hosts at \
+     \"/home/user/.ssh/known_hosts\"[0m\n\
+     _bk;t=1738172391782[90m$[0m git submodule foreach --recursive \"git clean -ffdx -e _build -e _opam -e .opam \
+     -e node_modules\"\n\
+     _bk;t=1738172391823Entering 'backend/cdr/cachetools'\n\
+     _bk;t=1738172391833[90m$[0m git clean -ffdx -e _build -e _opam -e .opam -e node_modules\n\
+     _bk;t=1738172391979Removing typescript\n\
+     _bk;t=1738172391980[90m$[0m git fetch -v --prune -- origin 61a85c041436cfd2fbc247978bf212383842fe69\n\
+     _bk;t=1738172398066From git.ahrefs.com:ahrefs/monorepo\n\
+     _bk;t=1738172398066 * branch                    61a85c041436cfd2fbc247978bf212383842fe69 -> FETCH_HEAD\n\
+     _bk;t=1738172398099[90m$[0m git checkout -f 61a85c041436cfd2fbc247978bf212383842fe69\n\
+     _bk;t=1738172399552Previous HEAD position was 19fcfa0cfaf infra: ch: mv `core::clickhouse::dashboard` to `devsg`\n\
+     _bk;t=1738172399554HEAD is now at 61a85c04143 Revert \"infra: add aa_wc_admin to publish\"\n\
+     _bk;t=1738172399569[90m# Git submodules detected[0m\n\
+     _bk;t=1738172399569[90m$[0m git submodule sync --recursive\n\
+     _bk;t=1738172399611Synchronizing submodule url for 'backend/cdr/cachetools'\n\
+     _bk;t=1738172399618[90m$[0m git submodule update --init --recursive --force\n\
+     _bk;t=1738172399669Submodule path 'backend/cdr/cachetools': checked out \
+     'c9700a0b29a8d220c1afdb867e9eed9357cd3351'\n\
+     _bk;t=1738172399674[90m$[0m git submodule foreach --recursive \"git reset --hard\"\n\
+     _bk;t=1738172399713Entering 'backend/cdr/cachetools'\n\
+     _bk;t=1738172399718HEAD is now at c9700a0 use long for internal counters\n\
+     _bk;t=1738172399723[90m# Cleaning again to catch any post-checkout changes[0m\n\
+     _bk;t=1738172399723[90m$[0m git clean -ffdx -e _build -e _opam -e .opam -e node_modules\n\
+     _bk;t=1738172399865[90m$[0m git submodule foreach --recursive \"git clean -ffdx -e _build -e _opam -e .opam \
+     -e node_modules\"\n\
+     _bk;t=1738172399908Entering 'backend/cdr/cachetools'\n\
+     _bk;t=1738172399917[90m# Checking to see if git commit information needs to be sent to Buildkite...[0m\n\
+     _bk;t=1738172399917[90m$[0m buildkite-agent meta-data exists buildkite:git:commit\n\
+     _bk;t=1738172400612[90m# Sending Git commit information back to Buildkite[0m\n\
+     _bk;t=1738172400617[90m$[0m buildkite-agent meta-data set buildkite:git:commit < /dev/stdin\n\
+     _bk;t=1738172400646[38;5;48m2025-01-29 17:40:00 INFO  [0m [0mReading meta-data value from STDIN[0m\n\
+     _bk;t=1738172401344~~~ Running commands\n\
+     _bk;t=1738172401344[90m$[0m /home/user/buildkite_pipeline.sh | buildkite-agent pipeline upload\n\
+     _bk;t=1738172402024[38;5;48m2025-01-29 17:40:02 INFO  [0m [0mReading pipeline config from STDIN[0m\n\
+     _bk;t=1738172402027[38;5;48m2025-01-29 17:40:02 INFO  [0m [0mUpdating BUILDKITE_COMMIT to \
+     \"61a85c041436cfd2fbc247978bf212383842fe69\"[0m\n\
+     _bk;t=1738172404182[38;5;48m2025-01-29 17:40:04 INFO  [0m [0mSuccessfully uploaded and parsed pipeline \
+     config[0m\n" )