Merge branch 'main' into parquet

.github/actions/fuse_compat/action.yml

@@ -30,6 +30,8 @@ runs:
           bash ./tests/fuse-compat/test-fuse-compat.sh 1.1.39 base
           bash ./tests/fuse-compat/test-fuse-compat.sh 1.1.46 base
           bash ./tests/fuse-compat/test-fuse-compat.sh 1.2.306 rbac
+          bash ./tests/fuse-compat/test-fuse-forward-compat.sh 1.2.307 rbac
+          bash ./tests/fuse-compat/test-fuse-forward-compat.sh 1.2.318 rbac
     - name: Upload failure
       if: failure()
       uses: ./.github/actions/artifact_failure

src/common/license/Cargo.toml

@@ -16,3 +16,5 @@ databend-common-base = { path = "../base" }
 databend-common-exception = { path = "../exception" }
 jwt-simple = "0.11.0"
 serde = { workspace = true }
+strum = "0.24.1"
+strum_macros = "0.24.3"

src/common/license/src/license.rs

@@ -17,8 +17,11 @@ use std::fmt::Formatter;
 
 use serde::Deserialize;
 use serde::Serialize;
+use strum::IntoEnumIterator;
+use strum_macros::EnumIter;
 
 // All enterprise features are defined here.
+#[derive(Debug, PartialEq, EnumIter)]
 pub enum Feature {
     LicenseInfo,
     Vacuum,
@@ -77,3 +80,23 @@ pub struct LicenseInfo {
     pub tenants: Option<Vec<String>>,
     pub features: Option<Vec<String>>,
 }
+
+impl LicenseInfo {
+    pub fn display_features(&self) -> String {
+        // sort all features in alphabet order and ignore test feature
+        let mut binding = self.features.clone().unwrap_or_default();
+        if binding.is_empty() {
+            binding = Feature::iter().map(|f| f.to_string()).collect::<Vec<_>>();
+        }
+        let mut features = binding
+            .iter()
+            .filter(|f| *f != &Feature::Test.to_string())
+            .collect::<Vec<_>>();
+        features.sort();
+        features
+            .iter()
+            .map(|f| f.to_string())
+            .collect::<Vec<_>>()
+            .join(",")
+    }
+}

src/meta/app/src/principal/user_privilege.rs

@@ -19,6 +19,7 @@ use enumflags2::bitflags;
 use enumflags2::make_bitflags;
 use enumflags2::BitFlags;
 
+// Note: If add new privilege type, need add forward test
 #[bitflags]
 #[repr(u64)]
 #[derive(

src/meta/proto-conv/src/share_from_to_protobuf_impl.rs

@@ -129,23 +129,19 @@ impl FromToProto for mt::ShareGrantEntry {
     where Self: Sized {
         reader_check_msg(p.ver, p.min_reader_ver)?;
 
-        let privileges = BitFlags::<mt::ShareGrantObjectPrivilege, u64>::from_bits(p.privileges);
-        match privileges {
-            Ok(privileges) => Ok(mt::ShareGrantEntry {
-                object: mt::ShareGrantObject::from_pb(p.object.ok_or_else(|| Incompatible {
-                    reason: "ShareGrantEntry.object can not be None".to_string(),
-                })?)?,
-                privileges,
-                grant_on: DateTime::<Utc>::from_pb(p.grant_on)?,
-                update_on: match p.update_on {
-                    Some(t) => Some(DateTime::<Utc>::from_pb(t)?),
-                    None => None,
-                },
-            }),
-            Err(e) => Err(Incompatible {
-                reason: format!("UserPrivilegeType error: {}", e),
-            }),
-        }
+        let privileges =
+            BitFlags::<mt::ShareGrantObjectPrivilege, u64>::from_bits_truncate(p.privileges);
+        Ok(mt::ShareGrantEntry {
+            object: mt::ShareGrantObject::from_pb(p.object.ok_or_else(|| Incompatible {
+                reason: "ShareGrantEntry.object can not be None".to_string(),
+            })?)?,
+            privileges,
+            grant_on: DateTime::<Utc>::from_pb(p.grant_on)?,
+            update_on: match p.update_on {
+                Some(t) => Some(DateTime::<Utc>::from_pb(t)?),
+                None => None,
+            },
+        })
     }
 
     fn to_pb(&self) -> Result<pb::ShareGrantEntry, Incompatible> {

src/meta/proto-conv/src/user_from_to_protobuf_impl.rs

@@ -241,18 +241,14 @@ impl FromToProto for mt::principal::GrantEntry {
     where Self: Sized {
         reader_check_msg(p.ver, p.min_reader_ver)?;
 
-        let privileges = BitFlags::<mt::principal::UserPrivilegeType, u64>::from_bits(p.privileges);
-        match privileges {
-            Ok(privileges) => Ok(mt::principal::GrantEntry::new(
-                mt::principal::GrantObject::from_pb(p.object.ok_or_else(|| Incompatible {
-                    reason: "GrantEntry.object can not be None".to_string(),
-                })?)?,
-                privileges,
-            )),
-            Err(e) => Err(Incompatible {
-                reason: format!("UserPrivilegeType error: {}", e),
-            }),
-        }
+        let privileges =
+            BitFlags::<mt::principal::UserPrivilegeType, u64>::from_bits_truncate(p.privileges);
+        Ok(mt::principal::GrantEntry::new(
+            mt::principal::GrantObject::from_pb(p.object.ok_or_else(|| Incompatible {
+                reason: "GrantEntry.object can not be None".to_string(),
+            })?)?,
+            privileges,
+        ))
     }
 
     fn to_pb(&self) -> Result<pb::GrantEntry, Incompatible> {

src/meta/proto-conv/src/util.rs

@@ -107,6 +107,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (75, "2024-01-15: ADD: user.proto/CsvFileFormatParams add field `binary_format` and `output_header`", ),
     (76, "2024-01-18: ADD: ownership.proto and role.proto", ),
     (77, "2024-01-22: Remove: allow_anonymous in S3 Config", ),
+    (78, "2024-01-29: Refactor: GrantEntry::UserPrivilegeType and ShareGrantEntry::ShareGrantObjectPrivilege use from_bits_truncate deserialize", ),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -81,3 +81,4 @@ mod v074_table_db_meta;
 mod v075_csv_format_params;
 mod v076_role_ownership_info;
 mod v077_s3_remove_allow_anonymous;
+mod v078_grantentry;

src/meta/proto-conv/tests/it/v078_grantentry.rs

@@ -0,0 +1,72 @@
+// Copyright 2023 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use chrono::TimeZone;
+use chrono::Utc;
+use databend_common_meta_app as mt;
+use databend_common_meta_app::principal::UserPrivilegeType;
+use databend_common_meta_app::share::ShareGrantObjectPrivilege;
+use enumflags2::make_bitflags;
+use minitrace::func_name;
+
+use crate::common;
+
+// These bytes are built when a new version in introduced,
+// and are kept for backward compatibility test.
+//
+// *************************************************************
+// * These messages should never be updated,                   *
+// * only be added when a new version is added,                *
+// * or be removed when an old version is no longer supported. *
+// *************************************************************
+//
+
+#[test]
+fn test_decode_v78_grant_entry() -> anyhow::Result<()> {
+    let grant_entry_v78 = vec![
+        10, 8, 10, 0, 160, 6, 78, 168, 6, 24, 16, 254, 255, 55, 160, 6, 78, 168, 6, 24,
+    ];
+
+    let want = || {
+        mt::principal::GrantEntry::new(
+            mt::principal::GrantObject::Global,
+            make_bitflags!(UserPrivilegeType::{Create | Select | Insert | Update | Delete | Drop | Alter | Super | CreateUser | DropUser | CreateRole | DropRole | Grant | CreateStage | Set | CreateDataMask | Read | Write }),
+        )
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), grant_entry_v78.as_slice(), 78, want())?;
+
+    Ok(())
+}
+
+#[test]
+fn test_decode_v78_share_grant_entry() -> anyhow::Result<()> {
+    let share_grant_entry_v78 = vec![
+        10, 8, 16, 19, 160, 6, 78, 168, 6, 24, 16, 4, 26, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50,
+        56, 32, 49, 50, 58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 160, 6, 78, 168, 6, 24,
+    ];
+
+    let want = || mt::share::ShareGrantEntry {
+        object: mt::share::ShareGrantObject::Table(19),
+        privileges: make_bitflags!(ShareGrantObjectPrivilege::{Select}),
+        grant_on: Utc.with_ymd_and_hms(2014, 11, 28, 12, 0, 9).unwrap(),
+        update_on: None,
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), share_grant_entry_v78.as_slice(), 78, want())?;
+
+    Ok(())
+}

src/query/ee/src/license/license_mgr.rs

@@ -131,8 +131,9 @@ impl RealLicenseManager {
         let features = l.custom.features.as_ref().unwrap();
         if !features.contains(&feature.to_string()) {
             return Err(ErrorCode::LicenseKeyInvalid(format!(
-                "license key does not support feature {}, supported features: {:?}",
-                feature, features
+                "license key does not support feature {}, supported features: {}",
+                feature,
+                l.custom.display_features()
             )));
         }
         Ok(())

src/query/service/src/table_functions/others/license_info.rs

@@ -69,6 +69,7 @@ impl LicenseInfoTable {
             TableField::new("expire_at", TableDataType::Timestamp),
             // formatted string calculate the available time from now to expiry of license
             TableField::new("available_time_until_expiry", TableDataType::String),
+            TableField::new("features", TableDataType::String),
         ])
     }
 
@@ -156,6 +157,8 @@ impl LicenseInfoSource {
         let available_time = info.expires_at.unwrap_or_default().sub(now).as_micros();
         let human_readable_available_time =
             HumanDuration::from(Duration::from_micros(available_time)).to_string();
+
+        let feature_str = info.custom.display_features();
         Ok(DataBlock::new(
             vec![
                 BlockEntry::new(
@@ -192,6 +195,7 @@ impl LicenseInfoSource {
                     DataType::String,
                     Value::Scalar(Scalar::String(human_readable_available_time)),
                 ),
+                BlockEntry::new(DataType::String, Value::Scalar(Scalar::String(feature_str))),
             ],
             1,
         ))

tests/fuse-compat/README.md

@@ -7,11 +7,15 @@ by a older version databend-query.
 
 ```shell
 tests/fuse-compat/test-fuse-compat.sh <old_ver>
+tests/fuse-compat/test-fuse-forward-compat.sh <old_ver>
 ```
 
 E.g. `tests/fuse-compat/test-fuse-compat.sh 0.7.151` tests if the fuse-table written
 by **databend-query-0.7.151** can be read by **current** version databend-query.
 
+`tests/fuse-compat/test-fuse-forward-compat.sh 1.2.307` tests if the fuse-table written
+by **current** can be read by **databend-query-0.7.151** version databend-query.
+
 ## Prerequisites
 
 - Current version of databend-query and databend-meta must reside in `./bins`:

tests/fuse-compat/test-fuse-compat.sh

@@ -53,7 +53,7 @@ download_query_config "$old_query_ver" old_config
 download_binary "$old_query_ver"
 
 old_config_path="old_config/$query_config_path"
-run_test $old_query_ver $old_config_path $logictest_path
+run_test $old_query_ver $old_config_path $logictest_path "backward"
 
 if [ -n "$stateless_test_path" ];
 then

tests/fuse-compat/test-fuse-forward-compat.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -o errexit
+SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
+echo " === SCRIPT_PATH: $SCRIPT_PATH"
+# go to work tree root
+cd "$SCRIPT_PATH/../../"
+ROOT="$(pwd)"
+pwd
+
+export RUST_BACKTRACE=full
+
+BUILD_PROFILE="${BUILD_PROFILE:-debug}"
+
+query_config_path="scripts/ci/deploy/config/databend-query-node-1.toml"
+
+usage() {
+    echo " === Assert that an old version query being compatible with lastest version query on fuse-table format"
+    echo " === Expect ./bins/current contains current version binaries"
+    echo " === Usage: $0 <old_version> <logictest_path> <supplementray_statless_test_path>"
+}
+
+source "${SCRIPT_PATH}/util.sh"
+
+
+# -- main --
+
+# The previous version to assert compatibility with
+# e.g. old_query_ver="0.7.151"
+old_query_ver="$1"
+
+# default sqllogic test suite is "testlogictest_path=${2:-"./base"}s/fuse-forward-compat/compat-logictest/"
+logictest_path=${2:-"./base"}
+
+# supplementary stateless test suite if provided (optional), which will be searched under "tests/fuse-forward-compat/compat-stateless"
+stateless_test_path="$3"
+
+echo " === old query ver : ${old_query_ver}"
+echo " === sql logic test path: ${logictest_path}"
+echo " === supplementary stateless test path: ${stateless_test_path}"
+
+
+chmod +x ./bins/current/*
+
+echo " === current metasrv ver: $(./bins/current/databend-meta --single --cmd ver | tr '\n' ' ')"
+echo " === current   query ver: $(./bins/current/databend-query --cmd ver | tr '\n' ' ')"
+echo " === old query ver: $old_query_ver"
+
+
+mkdir -p ./target/${BUILD_PROFILE}/
+
+download_query_config "$old_query_ver" old_config
+download_binary "$old_query_ver"
+
+echo "=== Now test forward compat ==="
+
+old_config_path="old_config/$query_config_path"
+run_test $old_query_ver $old_config_path $logictest_path "forward"
+
+if [ -n "$stateless_test_path" ];
+then
+  echo "=== ruing supplementary stateless test: ${stateless_test_path}"
+  run_stateless $stateless_test_path
+fi