Merge branch 'main' into refactor/cluster

databendlabs · Jun 29, 2024 · 24383ea · 24383ea
2 parents 0d9ea68 + b045ba1
commit 24383ea
Show file tree

Hide file tree

Showing 134 changed files with 1,264 additions and 2,261 deletions.
diff --git a/.github/actions/artifact_download/action.yml b/.github/actions/artifact_download/action.yml
@@ -54,11 +54,7 @@ runs:
       run: |
         artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
-          if [[ "${artifact}" == "open-sharing" ]]; then
-            chmod +x ${{ steps.info.outputs.path }}/open-sharing || true
-          else
-            chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
-          fi
+          chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
         done
 
     - name: Download artifact from s3
@@ -67,15 +63,9 @@ runs:
       run: |
         artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
-          if [[ "${artifact}" == "open-sharing" ]]; then
-            echo "==> fetching open-sharing ..."
-            aws s3 cp s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/open-sharing ${{ steps.info.outputs.path }}/open-sharing --no-progress || true
-            chmod +x ${{ steps.info.outputs.path }}/open-sharing || true
-          else
-            echo "==> fetching databend-$artifact ..."
-            aws s3 cp s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/databend-$artifact ${{ steps.info.outputs.path }}/databend-$artifact --no-progress
-            chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
-          fi
+          echo "==> fetching databend-$artifact ..."
+          aws s3 cp s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/databend-$artifact ${{ steps.info.outputs.path }}/databend-$artifact --no-progress
+          chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
         done
 
     - name: Download artifact from gcs
@@ -85,15 +75,9 @@ runs:
         gcloud config set storage/parallel_composite_upload_enabled False
         artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
-          if [[ "${artifact}" == "open-sharing" ]]; then
-            echo "==> fetching open-sharing ..."
-            gcloud storage cp gs://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/open-sharing ${{ steps.info.outputs.path }}/open-sharing || true
-            chmod +x ${{ steps.info.outputs.path }}/open-sharing || true
-          else
-            echo "==> fetching databend-$artifact ..."
-            gcloud storage cp gs://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/databend-$artifact ${{ steps.info.outputs.path }}/databend-$artifact
-            chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
-          fi
+          echo "==> fetching databend-$artifact ..."
+          gcloud storage cp gs://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/databend-$artifact ${{ steps.info.outputs.path }}/databend-$artifact
+          chmod +x ${{ steps.info.outputs.path }}/databend-$artifact
         done
 
     - name: Check Disk Space

diff --git a/.github/actions/artifact_upload/action.yml b/.github/actions/artifact_upload/action.yml
@@ -37,21 +37,13 @@ runs:
         name: ${{ env.BUILD_PROFILE }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
         path: ${{ steps.info.outputs.path }}/databend-*
 
-    - name: Upload open-sharing to github
-      if: env.RUNNER_PROVIDER == 'github' && inputs.category == 'default'
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ env.BUILD_PROFILE }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
-        path: ${{ steps.info.outputs.path }}/open-sharing
-
     - name: Upload artifacts to s3
       if: env.RUNNER_PROVIDER == 'aws'
       shell: bash
       run: |
         aws s3 sync ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/ \
           s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/ \
           --exclude "*" --include "databend-*" --no-progress
-        aws s3 cp ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/open-sharing s3://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/ --no-progress || true
 
     - name: Upload artifact to gcs
       if: env.RUNNER_PROVIDER == 'gcp'
@@ -60,5 +52,3 @@ runs:
         gcloud config set storage/parallel_composite_upload_enabled False
         gcloud storage cp --quiet --recursive ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-* \
           gs://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/
-        gcloud storage cp --quiet ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/open-sharing \
-          gs://databend-ci/${{ env.BUILD_PROFILE }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/ || true
diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml
@@ -72,7 +72,7 @@ runs:
           echo "==> building databend-$artifact ..."
           cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
         done
-        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin open-sharing
+        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }}
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-*
 
     - name: Build Debug for specific artifacts

diff --git a/.github/actions/test_stateful_sharing/action.yml b/.github/actions/test_stateful_sharing/action.yml
diff --git a/.github/workflows/reuse.benchmark.yml b/.github/workflows/reuse.benchmark.yml
@@ -209,7 +209,7 @@ jobs:
             aws s3 sync "s3://benchmark/clickbench/release/${dataset}/latest/" ./results/${dataset}/ --exclude "*" --include "*.json"
             ./update_results.py --dataset $dataset --pr ${{ inputs.source_id }}
             aws s3 cp ./results/${dataset}.html ${REPORT_S3_PREFIX}/${dataset}.html
-            echo "* **${dataset}**: https://benchmark.databend.rs/clickbench/pr/${{ inputs.source_id }}/${{ inputs.run_id }}/${dataset}.html" >> /tmp/body
+            echo "* **${dataset}**: https://benchmark.databend.com/clickbench/pr/${{ inputs.source_id }}/${{ inputs.run_id }}/${dataset}.html" >> /tmp/body
           done
       - name: Comment on PR
         uses: everpcpc/comment-on-pr-action@v1

diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml
@@ -234,20 +234,6 @@ jobs:
         with:
           name: test-stateful-standalone-linux
 
-  test_stateful_sharing:
-    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
-    needs: [build, check]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/test_stateful_sharing
-        if: env.BUILD_PROFILE == 'debug'
-        timeout-minutes: 10
-      - name: Upload failure
-        if: failure()
-        uses: ./.github/actions/artifact_failure
-        with:
-          name: test-stateful-sharing-linux
-
   test_stateful_cluster:
     runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     needs: [build, check]

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -34,7 +34,6 @@ members = [
     "src/query/config",
     "src/query/catalog",
     "src/query/sharing",
-    "src/query/sharing_endpoint",
     "src/query/datavalues",
     "src/query/expression",
     "src/query/formats",
@@ -73,6 +72,7 @@ members = [
     "src/query/storages/parquet",
     "src/query/storages/result_cache",
     "src/query/users",
+    "src/query/ee_features/attach_table",
     "src/query/ee_features/vacuum_handler",
     "src/query/ee_features/background_service",
     "src/query/ee_features/aggregating_index",
@@ -177,6 +177,7 @@ databend-common-tracing = { path = "src/common/tracing" }
 databend-common-users = { path = "src/query/users" }
 databend-common-vector = { path = "src/common/vector" }
 databend-enterprise-aggregating-index = { path = "src/query/ee_features/aggregating_index" }
+databend-enterprise-attach-table = { path = "src/query/ee_features/attach_table" }
 databend-enterprise-background-service = { path = "src/query/ee_features/background_service" }
 databend-enterprise-data-mask-feature = { path = "src/query/ee_features/data_mask" }
 databend-enterprise-inverted-index = { path = "src/query/ee_features/inverted_index" }
@@ -189,7 +190,6 @@ databend-enterprise-vacuum-handler = { path = "src/query/ee_features/vacuum_hand
 databend-enterprise-virtual-column = { path = "src/query/ee_features/virtual_column" }
 databend-meta = { path = "src/meta/service" }
 databend-query = { path = "src/query/service" }
-databend-sharing-endpoint = { path = "src/query/sharing_endpoint" }
 databend-sqllogictests = { path = "tests/sqllogictests" }
 databend-storages-common-blocks = { path = "src/query/storages/common/blocks" }
 databend-storages-common-cache = { path = "src/query/storages/common/cache" }
@@ -390,11 +390,13 @@ arrow-udf-js = { git = "https://github.com/datafuse-extras/arrow-udf", rev = "df
 arrow-udf-python = { git = "https://github.com/datafuse-extras/arrow-udf", rev = "dfa1390" }
 arrow-udf-wasm = { git = "https://github.com/datafuse-extras/arrow-udf", rev = "dfa1390" }
 async-backtrace = { git = "https://github.com/zhang2014/async-backtrace.git", rev = "dea4553" }
+async-recursion = { git = "https://github.com/zhang2014/async-recursion.git", rev = "a353334" }
 backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "6145fe6b" }
 deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "81593e9" }
 ethnum = { git = "https://github.com/ariesdevil/ethnum-rs", rev = "4cb05f1" }
 icelake = { git = "https://github.com/icelake-io/icelake", rev = "be8b2c2" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
 orc-rust = { git = "https://github.com/youngsofun/datafusion-orc", branch = "pub" }
+recursive = { git = "https://github.com/zhang2014/recursive.git", rev = "6af35a1" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }
diff --git a/benchmark/clickbench/hits/load.sql b/benchmark/clickbench/hits/load.sql
@@ -1,5 +1,5 @@
 COPY INTO hits
-FROM 'https://datasets.databend.rs/hits_100m_obfuscated_v1.tsv.xz' FILE_FORMAT = (
+FROM 'https://datasets.databend.org/hits_100m_obfuscated_v1.tsv.xz' FILE_FORMAT = (
         type = TSV compression = XZ field_delimiter = '\t' record_delimiter = '\n' skip_header = 0
     );
 ANALYZE TABLE hits;
diff --git a/benchmark/clickbench/index.jinja b/benchmark/clickbench/index.jinja
@@ -305,7 +305,7 @@
         <span class="nowrap themes"><span id="toggle-dark">🌚</span><span id="toggle-light">🌞</span></span>
         <h1>{{ title }}</h1>
         <a href="https://github.com/ClickHouse/ClickBench/">Clickbench</a> |
-        <a href="https://benchmark.databend.rs/clickbench/release/{{ dataset }}.html">Release Versions</a> |
+        <a href="https://benchmark.databend.com/clickbench/release/{{ dataset }}.html">Release Versions</a> |
         <a href="{{ url }}">Changeset Detail</a>
     </div>