Merge pull request #17 from DataRecce/feature/drc-430-setup-devcontai…

…ner-with-codespace

[Implement] Setup Recce with CodeSpace on our own OSO repo

.devcontainer/Dockerfile

@@ -0,0 +1,35 @@
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.11
+
+# Add GitHub CLI 
+RUN (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
+&& sudo mkdir -p -m 755 /etc/apt/keyrings \
+&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
+&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+
+RUN apt-get update && apt-get install -y python3-dev gh && rm -rf /var/lib/apt/lists/*
+
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+RUN if [ "$USER_GID" != "1000" ] || [ "$USER_UID" != "1000" ]; then groupmod --gid $USER_GID vscode && usermod --uid $USER_UID --gid $USER_GID vscode; fi
+
+RUN pip3 install --upgrade pip
+RUN pip3 install pipx
+RUN pipx install poetry
+
+# Downloading gcloud package
+RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz
+
+# Installing the package
+RUN mkdir -p /usr/local/gcloud \
+  && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \
+  && /usr/local/gcloud/google-cloud-sdk/install.sh
+
+# Adding the package path to local
+ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
+
+# Install osos python dependencies
+COPY . /tmp/oso
+RUN cd /tmp/oso && pip3 install .
+RUN pip3 install recce-nightly

.devcontainer/devcontainer.json

@@ -0,0 +1,22 @@
+{
+    "name": "Recce CodeSpace",
+    "image": "infuseai/oso-dev-container:3.11",
+    "containerEnv": {
+        "DBT_GOOGLE_PROJECT": "infuseai-dev",
+        "DBT_GOOGLE_DATASET": "oso_playground",
+        "DBT_GOOGLE_DEV_DATASET": "oso_playground_dev",
+        "DBT_GOOGLE_KEYFILE": "/home/vscode/.config/gcloud/google-service-account.json",
+        "RECCE_CI_WORKFLOW_NAME": "OSO Recce CI",
+        "RECCE_DAILY_CI_WORKFLOW_NAME": "OSO Recce Staging CI"
+    },
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "terminal.integrated.shell.linux": "/bin/bash",
+                "python.terminal.activateEnvironment": true
+            }
+        },
+        "extensions": []
+        },
+    "forwardPorts": [8000]
+}

.devcontainer/docker-build.sh

@@ -0,0 +1,3 @@
+#! /bin/bash
+
+docker buildx build --platform linux/amd64 -t infuseai/oso-dev-container:3.11 .. -f Dockerfile

.devcontainer/github_codespace_env.sh

@@ -0,0 +1,30 @@
+#! /bin/bash
+
+if [ "${CODESPACES}" == "true" ]; then
+  # Set the default git repository if running in GitHub Codespaces
+  echo "Setting the default git repository to $GITHUB_REPOSITORY"
+  gh repo set-default $GITHUB_REPOSITORY
+
+  current_branch=$(git branch --show-current)
+  # Check if the current branch is under a pull request
+  if gh pr view > /dev/null ; then
+    # Check if the Recce state file is downloaded
+    run_id=$(gh run list -b ${current_branch} -s success --limit 1 -w "${RECCE_CI_WORKFLOW_NAME}" --json databaseId | jq .[].databaseId)
+    if [ -z "$run_id" ]; then
+      echo "No successful Recce run found for the current branch."
+    else
+      echo "Downloading the Recce state file for the last successful run."
+      gh run download $run_id --dir .recce
+      echo "The Recce state file is downloaded to '.recce/recce_state_file/recce_state.json'."
+    fi
+  fi
+
+  # Check daily staging artifact files
+  default_branch=$(gh repo view --json defaultBranchRef --jq .defaultBranchRef.name)
+  daily_artifact_workflow_id=$(gh run list -w "${RECCE_DAILY_CI_WORKFLOW_NAME}" --status success -b dev --limit 1 --json databaseId | jq .[].databaseId)
+  gh run download $daily_artifact_workflow_id --dir .recce
+  if [ -d ".recce/dbt-artifacts" ]; then
+    mv .recce/dbt-artifacts target-base
+    echo "The daily staging artifact files are downloaded to 'target-base'."
+  fi
+fi

.devcontainer/launch_recce_server.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+if [ -f "${DIR}/../.recce/recce_state_file/recce_state.json" ]; then
+    echo "Launching the Recce server in review mode. The Recce state file is found."
+    cp ${DIR}/../.recce/recce_state_file/recce_state.json recce_state.json
+    recce server --review recce_state.json
+else
+    echo "Launching the Recce server."
+    recce server
+fi

.devcontainer/setup_required_env.sh

@@ -0,0 +1,61 @@
+#! /bin/bash
+
+GREEN="\033[0;32m"
+YELLOW="\033[1;33m"
+ENDCOLOR="\033[0m"
+
+function show_env_hint() {
+  echo -e "[${YELLOW}Required ENV${ENDCOLOR}] $1 is not set."
+  cat << EOF
+
+Please set the following environment variables in your GitHub Codespaces Secrets.
+You can set the secret in your GitHub Codespaces Secrets by going to:
+    GitHub Personal Account -> Settings -> Codespaces -> Codespaces secrets
+
+Then add the secret with the name $1 and the value of the secret.
+After adding the secret, please restart the Codespaces to apply the changes.
+EOF
+}
+
+# Check the ENV variables should be provided by the user
+if [ -z "$DBT_GOOGLE_PROJECT" ]; then
+  show_env_hint "DBT_GOOGLE_PROJECT"
+  exit 1
+fi
+
+if [ -z "$DBT_GOOGLE_DATASET" ]; then
+  show_env_hint "DBT_GOOGLE_DATASET"
+  exit 1
+fi
+
+if [ -z "$DBT_GOOGLE_DEV_DATASET" ]; then
+  show_env_hint "DBT_GOOGLE_DEV_DATASET"
+  exit 1
+fi
+
+mkdir -p $HOME/.config/gcloud
+DBT_GOOGLE_KEYFILE=$HOME/.config/gcloud/google-service-account.json
+
+# Setup dbt profiles.yml
+if [ "$DBT_PROFILES_YML_CONTENT" != '' ]; then
+  echo "$DBT_PROFILES_YML_CONTENT" > $HOME/.dbt/profiles.yml
+  echo "dbt profiles.yml is saved to $HOME/.dbt/profiles.yml"
+fi
+
+
+# Check if the user is already logged in
+if [ -z "$GOOGLE_CLOUD_SERVICE_ACCOUNT_KEY_CONTENT" ]; then
+  # Change to use OAuth2 to login
+  if [ -f "${DBT_GOOGLE_KEYFILE}" ]; then
+    echo "User is already logged in Google cloud"
+    gcloud auth list
+    exit 0
+  else
+    echo -e "[${GREEN}Action${ENDCOLOR}] Please login to Google cloud to continue."
+    gcloud auth application-default login
+  fi
+else
+  # Use the service account key to login
+  echo "$GOOGLE_CLOUD_SERVICE_ACCOUNT_KEY_CONTENT" > ${DBT_GOOGLE_KEYFILE}
+  echo "Google cloud service account key is saved to ${DBT_GOOGLE_KEYFILE}"
+fi

.gitignore

@@ -63,4 +63,10 @@ dbt_packages/
 
 supabase/.temp/
 **/supabase/.temp/
-*/**/supabase/.temp/
+*/**/supabase/.temp/
+
+# Recce & DBT
+.recce/
+target/
+target-base/
+recce_state.json

.vscode/tasks.json

@@ -0,0 +1,40 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Check GitHub CodeSpace environment",
+            "type": "shell",
+            "command": "bash .devcontainer/github_codespace_env.sh",
+        },
+        {
+            "label": "Install Project dependencies",
+            "type": "shell",
+            "command": "poetry install && poetry env use python",
+        },
+        {
+            "label": "Setup required environment variables",
+            "type": "shell",
+            "command": "bash .devcontainer/setup_required_env.sh",
+        },
+        {
+            "label": "Run DBT",
+            "type": "shell",
+            "command": "dbt deps && dbt build && dbt docs generate",
+            "dependsOn": [
+                "Setup required environment variables",
+                "Check GitHub CodeSpace environment"
+            ],
+        },
+        {
+            "label": "Launch Recce server",
+            "type": "shell",
+            "command": "bash .devcontainer/launch_recce_server.sh",
+            "dependsOn": ["Run DBT"],
+            "runOptions": {
+                "runOn": "folderOpen"
+            }
+        }
+    ]
+}