From c7bdb35fa1129a131e1eb2b02be358b461736f2e Mon Sep 17 00:00:00 2001
From: Drew Tate <drew.tate@elastic.co>
Date: Tue, 28 May 2024 10:02:20 -0600
Subject: [PATCH] [ES|QL] add function definition sync to CI (#184225)

## Summary

Runs the script added in https://github.com/elastic/kibana/pull/179584
on the weekly schedule.

Successful run:
https://buildkite.com/elastic/kibana-es-ql-grammar-sync/builds/21#018fab3b-d051-4852-a03e-889cf156fb70/838

---------

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
---
 .buildkite/pipelines/esql_grammar_sync.yml    | 11 ++-
 .../esql_generate_function_definitions.sh     | 76 +++++++++++++++++++
 .github/CODEOWNERS                            |  1 +
 .../scripts/generate_function_definitions.ts  |  2 -
 4 files changed, 87 insertions(+), 3 deletions(-)
 create mode 100755 .buildkite/scripts/steps/esql_generate_function_definitions.sh

diff --git a/.buildkite/pipelines/esql_grammar_sync.yml b/.buildkite/pipelines/esql_grammar_sync.yml
index ffa532101cf3c49..2bf14af24b20ede 100644
--- a/.buildkite/pipelines/esql_grammar_sync.yml
+++ b/.buildkite/pipelines/esql_grammar_sync.yml
@@ -1,6 +1,15 @@
 steps:
   - command: .buildkite/scripts/steps/esql_grammar_sync.sh
-    label: ES|QL Grammar Sync
+    label: Grammar Sync
+    timeout_in_minutes: 10
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
+  - command: .buildkite/scripts/steps/esql_generate_function_definitions.sh
+    label: Generate Function Definitions
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
diff --git a/.buildkite/scripts/steps/esql_generate_function_definitions.sh b/.buildkite/scripts/steps/esql_generate_function_definitions.sh
new file mode 100755
index 000000000000000..ecb5b996df4c2db
--- /dev/null
+++ b/.buildkite/scripts/steps/esql_generate_function_definitions.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+report_main_step () {
+  echo "--- $1"
+}
+
+main () {
+  cd "$PARENT_DIR"
+
+  report_main_step "Cloning Elasticsearch repository"
+
+  rm -rf elasticsearch
+  git clone https://github.com/elastic/elasticsearch --depth 1
+
+  report_main_step "Bootstrapping Kibana"
+
+  cd "$KIBANA_DIR"
+
+  .buildkite/scripts/bootstrap.sh
+
+  cd "$KIBANA_DIR/packages/kbn-esql-validation-autocomplete"
+
+  report_main_step "Generate function definitions"
+
+  yarn make:defs $PARENT_DIR/elasticsearch
+
+  report_main_step "Generate function validation tests"
+  
+  yarn make:tests
+
+  # Check for differences
+  set +e
+  git diff --exit-code --quiet .
+  if [ $? -eq 0 ]; then
+    echo "No differences found. Our work is done here."
+    exit
+  fi
+  set -e
+
+  report_main_step "Differences found. Checking for an existing pull request."
+
+  KIBANA_MACHINE_USERNAME="kibanamachine"
+  git config --global user.name "$KIBANA_MACHINE_USERNAME"
+  git config --global user.email '42973632+kibanamachine@users.noreply.github.com'
+
+  PR_TITLE='[ES|QL] Update function definitions'
+  PR_BODY='This PR updates the function definitions based on the latest metadata from Elasticsearch.'
+
+  # Check if a PR already exists
+  pr_search_result=$(gh pr list --search "$PR_TITLE" --state open --author "$KIBANA_MACHINE_USERNAME"  --limit 1 --json title -q ".[].title")
+
+  if [ "$pr_search_result" == "$PR_TITLE" ]; then
+    echo "PR already exists. Exiting."
+    exit
+  fi
+
+  echo "No existing PR found. Committing changes."
+
+  # Make a commit
+  BRANCH_NAME="esql_generate_function_definitions_$(date +%s)"
+
+  git checkout -b "$BRANCH_NAME"
+
+  git add ./**/*
+  git commit -m "Update function definitions"
+
+  report_main_step "Changes committed. Creating pull request."
+
+  git push origin "$BRANCH_NAME"
+
+  # Create a PR
+  gh pr create --title "$PR_TITLE" --body "$PR_BODY" --base main --head "${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:ESQL' 
+}
+
+main
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 3d9ca954ecb7d68..7bb36681643aa7d 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1219,6 +1219,7 @@ x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
 /WORKSPACE.bazel @elastic/kibana-operations
 /.buildkite/ @elastic/kibana-operations
 /.buildkite/scripts/steps/esql_grammar_sync.sh @elastic/kibana-esql
+/.buildkite/scripts/steps/esql_generate_function_definitions.sh @elastic/kibana-esql
 /.buildkite/pipelines/esql_grammar_sync.yml @elastic/kibana-esql
 /kbn_pm/ @elastic/kibana-operations
 /x-pack/dev-tools @elastic/kibana-operations
diff --git a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts
index b7cd905c63baf08..65e2c8fe3852603 100644
--- a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts
+++ b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts
@@ -354,7 +354,6 @@ import type { FunctionDefinition } from './types';
   const pathToElasticsearch = process.argv[2];
 
   const ESFunctionDefinitionsDirectory = join(
-    __dirname,
     pathToElasticsearch,
     'docs/reference/esql/functions/kibana/definition'
   );
@@ -365,7 +364,6 @@ import type { FunctionDefinition } from './types';
   );
 
   const evalFunctionDefinitions: FunctionDefinition[] = [];
-  // const aggFunctionDefinitions = [];
   for (const ESDefinition of ESFunctionDefinitions) {
     if (aliases.has(ESDefinition.name) || excludedFunctions.has(ESDefinition.name)) {
       continue;