From 7af45965ccd8c391baeb26aa7c0ad84b360b1f9f Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Tue, 19 Mar 2024 00:20:23 +0900 Subject: [PATCH] Bumping object version to 0.8 (#4757) --- .../src/index_config/serialize.rs | 19 +- .../quickwit-config/src/index_template/mod.rs | 2 +- .../src/index_template/serialize.rs | 37 ++-- quickwit/quickwit-config/src/lib.rs | 8 +- .../src/node_config/serialize.rs | 7 +- .../src/backward_compatibility_tests/mod.rs | 2 +- quickwit/quickwit-metastore/src/lib.rs | 8 +- .../file_backed_index/serialize.rs | 17 +- .../src/metastore/file_backed/manifest.rs | 19 +- .../src/metastore/index_metadata/serialize.rs | 31 +-- .../src/split_metadata_version.rs | 47 ++--- .../file-backed-index/v0.6.expected.json | 8 +- .../file-backed-index/v0.7.expected.json | 8 +- .../file-backed-index/v0.8.expected.json | 192 ++++++++++++++++++ .../test-data/file-backed-index/v0.8.json | 192 ++++++++++++++++++ .../index-metadata/v0.4.expected.json | 4 +- .../index-metadata/v0.5.expected.json | 4 +- .../index-metadata/v0.6.expected.json | 4 +- .../index-metadata/v0.7.expected.json | 4 +- .../index-metadata/v0.8.expected.json | 132 ++++++++++++ .../test-data/index-metadata/v0.8.json | 132 ++++++++++++ .../test-data/manifest/v0.7.expected.json | 4 +- .../test-data/manifest/v0.8.expected.json | 88 ++++++++ .../test-data/manifest/v0.8.json | 88 ++++++++ .../split-metadata/v0.6.expected.json | 2 +- .../split-metadata/v0.7.expected.json | 2 +- .../split-metadata/v0.8.expected.json | 29 +++ .../test-data/split-metadata/v0.8.json | 29 +++ 28 files changed, 1004 insertions(+), 115 deletions(-) create mode 100644 quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.expected.json create mode 100644 quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.json create mode 100644 quickwit/quickwit-metastore/test-data/index-metadata/v0.8.expected.json create mode 100644 quickwit/quickwit-metastore/test-data/index-metadata/v0.8.json create mode 100644 quickwit/quickwit-metastore/test-data/manifest/v0.8.expected.json create mode 100644 quickwit/quickwit-metastore/test-data/manifest/v0.8.json create mode 100644 quickwit/quickwit-metastore/test-data/split-metadata/v0.8.expected.json create mode 100644 quickwit/quickwit-metastore/test-data/split-metadata/v0.8.json diff --git a/quickwit/quickwit-config/src/index_config/serialize.rs b/quickwit/quickwit-config/src/index_config/serialize.rs index 76d5c26ef9a..5fe4fb0e38e 100644 --- a/quickwit/quickwit-config/src/index_config/serialize.rs +++ b/quickwit/quickwit-config/src/index_config/serialize.rs @@ -29,23 +29,24 @@ use crate::{ }; /// Alias for the latest serialization format. -type IndexConfigForSerialization = IndexConfigV0_7; +type IndexConfigForSerialization = IndexConfigV0_8; #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "version")] pub(crate) enum VersionedIndexConfig { - #[serde(rename = "0.7")] + #[serde(rename = "0.8")] // Retro compatibility #[serde(alias = "0.4")] #[serde(alias = "0.5")] #[serde(alias = "0.6")] - V0_7(IndexConfigV0_7), + #[serde(alias = "0.7")] + V0_8(IndexConfigV0_8), } impl From for IndexConfigForSerialization { fn from(versioned_config: VersionedIndexConfig) -> IndexConfigForSerialization { match versioned_config { - VersionedIndexConfig::V0_7(v0_6) => v0_6, + VersionedIndexConfig::V0_8(v0_8) => v0_8, } } } @@ -109,7 +110,7 @@ impl IndexConfigForSerialization { impl From for VersionedIndexConfig { fn from(index_config: IndexConfig) -> Self { - VersionedIndexConfig::V0_7(index_config.into()) + VersionedIndexConfig::V0_8(index_config.into()) } } @@ -118,14 +119,14 @@ impl TryFrom for IndexConfig { fn try_from(versioned_index_config: VersionedIndexConfig) -> anyhow::Result { match versioned_index_config { - VersionedIndexConfig::V0_7(v0_6) => v0_6.build_and_validate(None), + VersionedIndexConfig::V0_8(v0_8) => v0_8.build_and_validate(None), } } } #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] -pub struct IndexConfigV0_7 { +pub struct IndexConfigV0_8 { pub index_id: String, #[schema(value_type = String)] #[serde(default)] @@ -140,9 +141,9 @@ pub struct IndexConfigV0_7 { pub retention_policy_opt: Option, } -impl From for IndexConfigV0_7 { +impl From for IndexConfigV0_8 { fn from(index_config: IndexConfig) -> Self { - IndexConfigV0_7 { + IndexConfigV0_8 { index_id: index_config.index_id, index_uri: Some(index_config.index_uri), doc_mapping: index_config.doc_mapping, diff --git a/quickwit/quickwit-config/src/index_template/mod.rs b/quickwit/quickwit-config/src/index_template/mod.rs index 3fbdaff5256..a4a7418ae71 100644 --- a/quickwit/quickwit-config/src/index_template/mod.rs +++ b/quickwit/quickwit-config/src/index_template/mod.rs @@ -23,7 +23,7 @@ use anyhow::ensure; use quickwit_common::uri::Uri; use quickwit_proto::types::IndexId; use serde::{Deserialize, Serialize}; -pub use serialize::{IndexTemplateV0_7, VersionedIndexTemplate}; +pub use serialize::{IndexTemplateV0_8, VersionedIndexTemplate}; use crate::index_config::validate_index_config; use crate::{ diff --git a/quickwit/quickwit-config/src/index_template/serialize.rs b/quickwit/quickwit-config/src/index_template/serialize.rs index 30f0eed0e5d..8bba1138fb4 100644 --- a/quickwit/quickwit-config/src/index_template/serialize.rs +++ b/quickwit/quickwit-config/src/index_template/serialize.rs @@ -26,13 +26,14 @@ use crate::{DocMapping, IndexingSettings, RetentionPolicy, SearchSettings}; #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "version")] pub enum VersionedIndexTemplate { - #[serde(rename = "0.7")] - V0_7(IndexTemplateV0_7), + #[serde(rename = "0.8")] + #[serde(alias = "0.7")] + V0_8(IndexTemplateV0_8), } #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] -pub struct IndexTemplateV0_7 { +pub struct IndexTemplateV0_8 { #[schema(value_type = String)] pub template_id: IndexTemplateId, #[schema(value_type = Vec)] @@ -56,36 +57,36 @@ pub struct IndexTemplateV0_7 { impl From for IndexTemplate { fn from(versioned_index_template: VersionedIndexTemplate) -> Self { match versioned_index_template { - VersionedIndexTemplate::V0_7(v0_7) => v0_7.into(), + VersionedIndexTemplate::V0_8(v0_8) => v0_8.into(), } } } impl From for VersionedIndexTemplate { fn from(index_template: IndexTemplate) -> Self { - VersionedIndexTemplate::V0_7(index_template.into()) + VersionedIndexTemplate::V0_8(index_template.into()) } } -impl From for IndexTemplate { - fn from(index_template_v0_7: IndexTemplateV0_7) -> Self { +impl From for IndexTemplate { + fn from(index_template_v0_8: IndexTemplateV0_8) -> Self { IndexTemplate { - template_id: index_template_v0_7.template_id, - index_id_patterns: index_template_v0_7.index_id_patterns, - index_root_uri: index_template_v0_7.index_root_uri, - priority: index_template_v0_7.priority, - description: index_template_v0_7.description, - doc_mapping: index_template_v0_7.doc_mapping, - indexing_settings: index_template_v0_7.indexing_settings, - search_settings: index_template_v0_7.search_settings, - retention_policy_opt: index_template_v0_7.retention, + template_id: index_template_v0_8.template_id, + index_id_patterns: index_template_v0_8.index_id_patterns, + index_root_uri: index_template_v0_8.index_root_uri, + priority: index_template_v0_8.priority, + description: index_template_v0_8.description, + doc_mapping: index_template_v0_8.doc_mapping, + indexing_settings: index_template_v0_8.indexing_settings, + search_settings: index_template_v0_8.search_settings, + retention_policy_opt: index_template_v0_8.retention, } } } -impl From for IndexTemplateV0_7 { +impl From for IndexTemplateV0_8 { fn from(index_template: IndexTemplate) -> Self { - IndexTemplateV0_7 { + IndexTemplateV0_8 { template_id: index_template.template_id, index_id_patterns: index_template.index_id_patterns, index_root_uri: index_template.index_root_uri, diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index 1cb8119ee8e..3b458819922 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -44,7 +44,7 @@ mod templating; pub use cluster_config::ClusterConfig; // We export that one for backward compatibility. // See #2048 -use index_config::serialize::{IndexConfigV0_7, VersionedIndexConfig}; +use index_config::serialize::{IndexConfigV0_8, VersionedIndexConfig}; pub use index_config::{ build_doc_mapper, load_index_config_from_user_config, DocMapping, IndexConfig, IndexingResources, IndexingSettings, RetentionPolicy, SearchSettings, @@ -60,7 +60,7 @@ pub use source_config::{ }; use tracing::warn; -use crate::index_template::IndexTemplateV0_7; +use crate::index_template::IndexTemplateV0_8; pub use crate::index_template::{IndexTemplate, IndexTemplateId, VersionedIndexTemplate}; use crate::merge_policy_config::{ ConstWriteAmplificationMergePolicyConfig, MergePolicyConfig, StableLogMergePolicyConfig, @@ -90,9 +90,9 @@ pub use crate::storage_config::{ SourceConfigV0_7, SourceConfigV0_8, VersionedIndexConfig, - IndexConfigV0_7, + IndexConfigV0_8, VersionedIndexTemplate, - IndexTemplateV0_7, + IndexTemplateV0_8, SourceInputFormat, SourceParams, FileSourceParams, diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs index 175309d0e76..d058e4d888a 100644 --- a/quickwit/quickwit-config/src/node_config/serialize.rs +++ b/quickwit/quickwit-config/src/node_config/serialize.rs @@ -149,18 +149,19 @@ pub async fn load_node_config_with_env( #[derive(Debug, Deserialize)] #[serde(tag = "version")] enum VersionedNodeConfig { - #[serde(rename = "0.7")] + #[serde(rename = "0.8")] // Retro compatibility. + #[serde(alias = "0.7")] #[serde(alias = "0.6")] #[serde(alias = "0.5")] #[serde(alias = "0.4")] - V0_7(NodeConfigBuilder), + V0_8(NodeConfigBuilder), } impl From for NodeConfigBuilder { fn from(versioned_node_config: VersionedNodeConfig) -> Self { match versioned_node_config { - VersionedNodeConfig::V0_7(node_config_builder) => node_config_builder, + VersionedNodeConfig::V0_8(node_config_builder) => node_config_builder, } } } diff --git a/quickwit/quickwit-metastore/src/backward_compatibility_tests/mod.rs b/quickwit/quickwit-metastore/src/backward_compatibility_tests/mod.rs index 9f0d52a5a0b..e707a145e1f 100644 --- a/quickwit/quickwit-metastore/src/backward_compatibility_tests/mod.rs +++ b/quickwit/quickwit-metastore/src/backward_compatibility_tests/mod.rs @@ -46,7 +46,7 @@ use crate::{IndexMetadata, SplitMetadata}; /// #[serde(rename="0.2")] /// V0_2(MyResourceV1) //< there was no change in this version. /// } -const GLOBAL_QUICKWIT_RESOURCE_VERSION: &str = "0.7"; +const GLOBAL_QUICKWIT_RESOURCE_VERSION: &str = "0.8"; /// This test makes sure that the resource is using the current `GLOBAL_QUICKWIT_RESOURCE_VERSION`. fn test_global_version(serializable: &T) -> anyhow::Result<()> { diff --git a/quickwit/quickwit-metastore/src/lib.rs b/quickwit/quickwit-metastore/src/lib.rs index 45b20864402..92bb05e1a2a 100644 --- a/quickwit/quickwit-metastore/src/lib.rs +++ b/quickwit/quickwit-metastore/src/lib.rs @@ -44,7 +44,7 @@ use std::ops::Range; pub use error::MetastoreResolverError; pub use metastore::control_plane_metastore::ControlPlaneMetastore; pub use metastore::file_backed::FileBackedMetastore; -pub(crate) use metastore::index_metadata::serialize::{IndexMetadataV0_7, VersionedIndexMetadata}; +pub(crate) use metastore::index_metadata::serialize::{IndexMetadataV0_8, VersionedIndexMetadata}; #[cfg(feature = "postgres")] pub use metastore::postgres::PostgresqlMetastore; pub use metastore::{ @@ -58,16 +58,16 @@ pub use metastore_resolver::MetastoreResolver; use quickwit_common::is_disjoint; use quickwit_doc_mapper::tag_pruning::TagFilterAst; pub use split_metadata::{Split, SplitInfo, SplitMaturity, SplitMetadata, SplitState}; -pub(crate) use split_metadata_version::{SplitMetadataV0_7, VersionedSplitMetadata}; +pub(crate) use split_metadata_version::{SplitMetadataV0_8, VersionedSplitMetadata}; #[derive(utoipa::OpenApi)] #[openapi(components(schemas( Split, SplitState, VersionedIndexMetadata, - IndexMetadataV0_7, + IndexMetadataV0_8, VersionedSplitMetadata, - SplitMetadataV0_7, + SplitMetadataV0_8, )))] /// Schema used for the OpenAPI generation which are apart of this crate. pub struct MetastoreApiSchemas; diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs index f59a99cbbff..f8ecf803200 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed/file_backed_index/serialize.rs @@ -34,30 +34,31 @@ use crate::{IndexMetadata, Split}; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "version")] pub(crate) enum VersionedFileBackedIndex { - #[serde(rename = "0.7")] + #[serde(rename = "0.8")] // Retro compatibility. + #[serde(alias = "0.7")] #[serde(alias = "0.6")] #[serde(alias = "0.5")] #[serde(alias = "0.4")] - V0_7(FileBackedIndexV0_7), + V0_8(FileBackedIndexV0_8), } impl From for VersionedFileBackedIndex { fn from(index: FileBackedIndex) -> Self { - VersionedFileBackedIndex::V0_7(index.into()) + VersionedFileBackedIndex::V0_8(index.into()) } } impl From for FileBackedIndex { fn from(index: VersionedFileBackedIndex) -> Self { match index { - VersionedFileBackedIndex::V0_7(v0_6) => v0_6.into(), + VersionedFileBackedIndex::V0_8(v0_8) => v0_8.into(), } } } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FileBackedIndexV0_7 { +pub(crate) struct FileBackedIndexV0_8 { #[serde(rename = "index")] metadata: IndexMetadata, splits: Vec, @@ -68,7 +69,7 @@ pub(crate) struct FileBackedIndexV0_7 { delete_tasks: Vec, } -impl From for FileBackedIndexV0_7 { +impl From for FileBackedIndexV0_8 { fn from(index: FileBackedIndex) -> Self { let splits = index .splits @@ -104,8 +105,8 @@ impl From for FileBackedIndexV0_7 { } } -impl From for FileBackedIndex { - fn from(mut index: FileBackedIndexV0_7) -> Self { +impl From for FileBackedIndex { + fn from(mut index: FileBackedIndexV0_8) -> Self { // if the index is otel-traces-v0_6, convert set bytes fields input and output format to hex // to be compatible with the v0_6 version. // TODO: remove after 0.8 release. diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/manifest.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/manifest.rs index 326b7c57f9a..a24dc571ed8 100644 --- a/quickwit/quickwit-metastore/src/metastore/file_backed/manifest.rs +++ b/quickwit/quickwit-metastore/src/metastore/file_backed/manifest.rs @@ -74,46 +74,47 @@ pub(crate) struct Manifest { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "version")] enum VersionedManifest { - #[serde(rename = "0.7")] - V0_7(ManifestV0_7), + #[serde(rename = "0.8")] + #[serde(alias = "0.7")] + V0_8(ManifestV0_8), } impl From for VersionedManifest { fn from(manifest: Manifest) -> Self { - VersionedManifest::V0_7(manifest.into()) + VersionedManifest::V0_8(manifest.into()) } } impl From for Manifest { fn from(versioned_manifest: VersionedManifest) -> Self { match versioned_manifest { - VersionedManifest::V0_7(manifest) => manifest.into(), + VersionedManifest::V0_8(manifest) => manifest.into(), } } } #[derive(Clone, Debug, Serialize, Deserialize)] -struct ManifestV0_7 { +struct ManifestV0_8 { indexes: BTreeMap, templates: Vec, } -impl From for ManifestV0_7 { +impl From for ManifestV0_8 { fn from(manifest: Manifest) -> Self { let templates = manifest .templates .into_values() .sorted_unstable_by(|left, right| left.template_id.cmp(&right.template_id)) .collect(); - ManifestV0_7 { + ManifestV0_8 { indexes: manifest.indexes, templates, } } } -impl From for Manifest { - fn from(manifest: ManifestV0_7) -> Self { +impl From for Manifest { + fn from(manifest: ManifestV0_8) -> Self { let indexes = manifest.indexes.into_iter().collect(); let templates = manifest .templates diff --git a/quickwit/quickwit-metastore/src/metastore/index_metadata/serialize.rs b/quickwit/quickwit-metastore/src/metastore/index_metadata/serialize.rs index d7292f8fe8c..88c89a85c39 100644 --- a/quickwit/quickwit-metastore/src/metastore/index_metadata/serialize.rs +++ b/quickwit/quickwit-metastore/src/metastore/index_metadata/serialize.rs @@ -30,17 +30,18 @@ use crate::IndexMetadata; #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "version")] pub(crate) enum VersionedIndexMetadata { - #[serde(rename = "0.7")] + #[serde(rename = "0.8")] // Retro compatibility. + #[serde(alias = "0.7")] #[serde(alias = "0.6")] #[serde(alias = "0.5")] #[serde(alias = "0.4")] - V0_7(IndexMetadataV0_7), + V0_8(IndexMetadataV0_8), } impl From for VersionedIndexMetadata { fn from(index_metadata: IndexMetadata) -> Self { - VersionedIndexMetadata::V0_7(index_metadata.into()) + VersionedIndexMetadata::V0_8(index_metadata.into()) } } @@ -51,12 +52,12 @@ impl TryFrom for IndexMetadata { match index_metadata { // When we have more than one version, you should chain version conversion. // ie. Implement conversion from V_k -> V_{k+1} - VersionedIndexMetadata::V0_7(v6) => v6.try_into(), + VersionedIndexMetadata::V0_8(v8) => v8.try_into(), } } } -impl From for IndexMetadataV0_7 { +impl From for IndexMetadataV0_8 { fn from(index_metadata: IndexMetadata) -> Self { let sources: Vec = index_metadata.sources.values().cloned().collect(); Self { @@ -70,7 +71,7 @@ impl From for IndexMetadataV0_7 { } #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] -pub(crate) struct IndexMetadataV0_7 { +pub(crate) struct IndexMetadataV0_8 { #[schema(value_type = String)] // Defaults to nil for backward compatibility. #[serde(default, alias = "index_id")] @@ -85,26 +86,26 @@ pub(crate) struct IndexMetadataV0_7 { pub sources: Vec, } -impl TryFrom for IndexMetadata { +impl TryFrom for IndexMetadata { type Error = anyhow::Error; - fn try_from(v0_6: IndexMetadataV0_7) -> anyhow::Result { + fn try_from(v0_8: IndexMetadataV0_8) -> anyhow::Result { let mut sources: HashMap = Default::default(); - for source in v0_6.sources { + for source in v0_8.sources { if sources.contains_key(&source.source_id) { anyhow::bail!("source `{}` is defined more than once", source.source_id); } sources.insert(source.source_id.clone(), source); } Ok(Self { - index_uid: if v0_6.index_uid.is_empty() { - IndexUid::from_parts(&v0_6.index_config.index_id, 0) + index_uid: if v0_8.index_uid.is_empty() { + IndexUid::from_parts(&v0_8.index_config.index_id, 0) } else { - v0_6.index_uid + v0_8.index_uid }, - index_config: v0_6.index_config, - checkpoint: v0_6.checkpoint, - create_timestamp: v0_6.create_timestamp, + index_config: v0_8.index_config, + checkpoint: v0_8.checkpoint, + create_timestamp: v0_8.create_timestamp, sources, }) } diff --git a/quickwit/quickwit-metastore/src/split_metadata_version.rs b/quickwit/quickwit-metastore/src/split_metadata_version.rs index e71fe9e2824..476eabb95fc 100644 --- a/quickwit/quickwit-metastore/src/split_metadata_version.rs +++ b/quickwit/quickwit-metastore/src/split_metadata_version.rs @@ -27,7 +27,7 @@ use crate::split_metadata::{utc_now_timestamp, SplitMaturity}; use crate::SplitMetadata; #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] -pub(crate) struct SplitMetadataV0_7 { +pub(crate) struct SplitMetadataV0_8 { /// Split ID. Joined with the index URI (/), this ID /// should be enough to uniquely identify a split. /// In reality, some information may be implicitly configured @@ -93,11 +93,11 @@ pub(crate) struct SplitMetadataV0_7 { num_merge_ops: usize, } -impl From for SplitMetadata { - fn from(v6: SplitMetadataV0_7) -> Self { - let source_id = v6.source_id.unwrap_or_else(|| "unknown".to_string()); +impl From for SplitMetadata { + fn from(v8: SplitMetadataV0_8) -> Self { + let source_id = v8.source_id.unwrap_or_else(|| "unknown".to_string()); - let node_id = if let Some(node_id) = v6.node_id { + let node_id = if let Some(node_id) = v8.node_id { // The previous version encoded `v1.node_id` as `{node_id}/{pipeline_ord}`. // Since pipeline_ord is no longer needed, we only extract the `node_id` portion // to keep backward compatibility. This has the advantage of avoiding a @@ -112,27 +112,27 @@ impl From for SplitMetadata { }; SplitMetadata { - split_id: v6.split_id, - index_uid: v6.index_uid, - partition_id: v6.partition_id, + split_id: v8.split_id, + index_uid: v8.index_uid, + partition_id: v8.partition_id, source_id, node_id, - delete_opstamp: v6.delete_opstamp, - num_docs: v6.num_docs, - uncompressed_docs_size_in_bytes: v6.uncompressed_docs_size_in_bytes, - time_range: v6.time_range, - create_timestamp: v6.create_timestamp, - maturity: v6.maturity, - tags: v6.tags, - footer_offsets: v6.footer_offsets, - num_merge_ops: v6.num_merge_ops, + delete_opstamp: v8.delete_opstamp, + num_docs: v8.num_docs, + uncompressed_docs_size_in_bytes: v8.uncompressed_docs_size_in_bytes, + time_range: v8.time_range, + create_timestamp: v8.create_timestamp, + maturity: v8.maturity, + tags: v8.tags, + footer_offsets: v8.footer_offsets, + num_merge_ops: v8.num_merge_ops, } } } -impl From for SplitMetadataV0_7 { +impl From for SplitMetadataV0_8 { fn from(split: SplitMetadata) -> Self { - SplitMetadataV0_7 { + SplitMetadataV0_8 { split_id: split.split_id, index_uid: split.index_uid, partition_id: split.partition_id, @@ -154,24 +154,25 @@ impl From for SplitMetadataV0_7 { #[derive(Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "version")] pub(crate) enum VersionedSplitMetadata { - #[serde(rename = "0.7")] + #[serde(rename = "0.8")] // Retro compatibility. + #[serde(alias = "0.7")] #[serde(alias = "0.6")] #[serde(alias = "0.5")] #[serde(alias = "0.4")] - V0_7(SplitMetadataV0_7), + V0_8(SplitMetadataV0_8), } impl From for SplitMetadata { fn from(versioned_helper: VersionedSplitMetadata) -> Self { match versioned_helper { - VersionedSplitMetadata::V0_7(v0_6) => v0_6.into(), + VersionedSplitMetadata::V0_8(v0_8) => v0_8.into(), } } } impl From for VersionedSplitMetadata { fn from(split_metadata: SplitMetadata) -> Self { - VersionedSplitMetadata::V0_7(split_metadata.into()) + VersionedSplitMetadata::V0_8(split_metadata.into()) } } diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json index c37cacd4811..30dff33588b 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json @@ -118,7 +118,7 @@ "message" ] }, - "version": "0.7" + "version": "0.8" }, "index_uid": "my-index:00000000000000000000000000", "sources": [ @@ -139,7 +139,7 @@ "version": "0.8" } ], - "version": "0.7" + "version": "0.8" }, "splits": [ { @@ -172,8 +172,8 @@ }, "uncompressed_docs_size_in_bytes": 234234, "update_timestamp": 1789, - "version": "0.7" + "version": "0.8" } ], - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.7.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.7.expected.json index 09c32160fd6..8af520d4800 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.7.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.7.expected.json @@ -118,7 +118,7 @@ "message" ] }, - "version": "0.7" + "version": "0.8" }, "index_uid": "my-index:00000000000000000000000000", "sources": [ @@ -139,7 +139,7 @@ "version": "0.8" } ], - "version": "0.7" + "version": "0.8" }, "shards": { "_ingest-source": [ @@ -185,8 +185,8 @@ }, "uncompressed_docs_size_in_bytes": 234234, "update_timestamp": 1789, - "version": "0.7" + "version": "0.8" } ], - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.expected.json new file mode 100644 index 00000000000..8af520d4800 --- /dev/null +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.expected.json @@ -0,0 +1,192 @@ +{ + "delete_tasks": [ + { + "create_timestamp": 0, + "delete_query": { + "index_uid": "my-index:00000000000000000000000000", + "query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}}]}" + }, + "opstamp": 10 + } + ], + "index": { + "checkpoint": { + "kafka-source": { + "00000000000000000000": "00000000000000000042" + } + }, + "create_timestamp": 1789, + "index_config": { + "doc_mapping": { + "dynamic_mapping": { + "expand_dots": true, + "fast": { + "normalizer": "raw" + }, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "raw" + }, + "field_mappings": [ + { + "coerce": true, + "fast": true, + "indexed": true, + "name": "tenant_id", + "output_format": "number", + "stored": true, + "type": "u64" + }, + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "timestamp", + "output_format": "rfc3339", + "stored": true, + "type": "datetime" + }, + { + "fast": false, + "fieldnorms": false, + "indexed": true, + "name": "log_level", + "record": "basic", + "stored": true, + "tokenizer": "raw", + "type": "text" + }, + { + "fast": false, + "fieldnorms": false, + "indexed": true, + "name": "message", + "record": "position", + "stored": true, + "tokenizer": "default", + "type": "text" + } + ], + "index_field_presence": true, + "max_num_partitions": 100, + "mode": "dynamic", + "partition_key": "tenant_id", + "store_source": true, + "tag_fields": [ + "log_level", + "tenant_id" + ], + "timestamp_field": "timestamp", + "tokenizers": [ + { + "filters": [], + "name": "custom_tokenizer", + "pattern": "[^\\p{L}\\p{N}]+", + "type": "regex" + } + ] + }, + "index_id": "my-index", + "index_uri": "s3://quickwit-indexes/my-index", + "indexing_settings": { + "commit_timeout_secs": 301, + "docstore_blocksize": 1000000, + "docstore_compression_level": 8, + "merge_policy": { + "maturation_period": "2days", + "max_merge_factor": 11, + "merge_factor": 9, + "min_level_num_docs": 100000, + "type": "stable_log" + }, + "resources": { + "heap_size": "50.0 MB" + }, + "split_num_docs_target": 10000001 + }, + "retention": { + "period": "90 days", + "schedule": "daily" + }, + "search_settings": { + "default_search_fields": [ + "message" + ] + }, + "version": "0.8" + }, + "index_uid": "my-index:00000000000000000000000000", + "sources": [ + { + "enabled": true, + "input_format": "json", + "num_pipelines": 2, + "params": { + "client_params": {}, + "topic": "kafka-topic" + }, + "source_id": "kafka-source", + "source_type": "kafka", + "transform": { + "script": ".message = downcase(string!(.message))", + "timezone": "UTC" + }, + "version": "0.8" + } + ], + "version": "0.8" + }, + "shards": { + "_ingest-source": [ + { + "follower_id": "follower-ingester", + "index_uid": "my-index:00000000000000000000000000", + "leader_id": "leader-ingester", + "publish_position_inclusive": "", + "shard_id": "00000000000000000001", + "shard_state": 1, + "source_id": "_ingest-source" + } + ] + }, + "splits": [ + { + "create_timestamp": 3, + "delete_opstamp": 10, + "footer_offsets": { + "end": 2000, + "start": 1000 + }, + "index_uid": "my-index:00000000000000000000000000", + "maturity": { + "maturation_period_millis": 4000, + "type": "immature" + }, + "node_id": "node", + "num_docs": 12303, + "num_merge_ops": 3, + "partition_id": 7, + "publish_timestamp": 1789, + "source_id": "source", + "split_id": "split", + "split_state": "Published", + "tags": [ + "234", + "aaa" + ], + "time_range": { + "end": 130198, + "start": 121000 + }, + "uncompressed_docs_size_in_bytes": 234234, + "update_timestamp": 1789, + "version": "0.8" + } + ], + "version": "0.8" +} diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.json new file mode 100644 index 00000000000..8af520d4800 --- /dev/null +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.8.json @@ -0,0 +1,192 @@ +{ + "delete_tasks": [ + { + "create_timestamp": 0, + "delete_query": { + "index_uid": "my-index:00000000000000000000000000", + "query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}}]}" + }, + "opstamp": 10 + } + ], + "index": { + "checkpoint": { + "kafka-source": { + "00000000000000000000": "00000000000000000042" + } + }, + "create_timestamp": 1789, + "index_config": { + "doc_mapping": { + "dynamic_mapping": { + "expand_dots": true, + "fast": { + "normalizer": "raw" + }, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "raw" + }, + "field_mappings": [ + { + "coerce": true, + "fast": true, + "indexed": true, + "name": "tenant_id", + "output_format": "number", + "stored": true, + "type": "u64" + }, + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "timestamp", + "output_format": "rfc3339", + "stored": true, + "type": "datetime" + }, + { + "fast": false, + "fieldnorms": false, + "indexed": true, + "name": "log_level", + "record": "basic", + "stored": true, + "tokenizer": "raw", + "type": "text" + }, + { + "fast": false, + "fieldnorms": false, + "indexed": true, + "name": "message", + "record": "position", + "stored": true, + "tokenizer": "default", + "type": "text" + } + ], + "index_field_presence": true, + "max_num_partitions": 100, + "mode": "dynamic", + "partition_key": "tenant_id", + "store_source": true, + "tag_fields": [ + "log_level", + "tenant_id" + ], + "timestamp_field": "timestamp", + "tokenizers": [ + { + "filters": [], + "name": "custom_tokenizer", + "pattern": "[^\\p{L}\\p{N}]+", + "type": "regex" + } + ] + }, + "index_id": "my-index", + "index_uri": "s3://quickwit-indexes/my-index", + "indexing_settings": { + "commit_timeout_secs": 301, + "docstore_blocksize": 1000000, + "docstore_compression_level": 8, + "merge_policy": { + "maturation_period": "2days", + "max_merge_factor": 11, + "merge_factor": 9, + "min_level_num_docs": 100000, + "type": "stable_log" + }, + "resources": { + "heap_size": "50.0 MB" + }, + "split_num_docs_target": 10000001 + }, + "retention": { + "period": "90 days", + "schedule": "daily" + }, + "search_settings": { + "default_search_fields": [ + "message" + ] + }, + "version": "0.8" + }, + "index_uid": "my-index:00000000000000000000000000", + "sources": [ + { + "enabled": true, + "input_format": "json", + "num_pipelines": 2, + "params": { + "client_params": {}, + "topic": "kafka-topic" + }, + "source_id": "kafka-source", + "source_type": "kafka", + "transform": { + "script": ".message = downcase(string!(.message))", + "timezone": "UTC" + }, + "version": "0.8" + } + ], + "version": "0.8" + }, + "shards": { + "_ingest-source": [ + { + "follower_id": "follower-ingester", + "index_uid": "my-index:00000000000000000000000000", + "leader_id": "leader-ingester", + "publish_position_inclusive": "", + "shard_id": "00000000000000000001", + "shard_state": 1, + "source_id": "_ingest-source" + } + ] + }, + "splits": [ + { + "create_timestamp": 3, + "delete_opstamp": 10, + "footer_offsets": { + "end": 2000, + "start": 1000 + }, + "index_uid": "my-index:00000000000000000000000000", + "maturity": { + "maturation_period_millis": 4000, + "type": "immature" + }, + "node_id": "node", + "num_docs": 12303, + "num_merge_ops": 3, + "partition_id": 7, + "publish_timestamp": 1789, + "source_id": "source", + "split_id": "split", + "split_state": "Published", + "tags": [ + "234", + "aaa" + ], + "time_range": { + "end": 130198, + "start": 121000 + }, + "uncompressed_docs_size_in_bytes": 234234, + "update_timestamp": 1789, + "version": "0.8" + } + ], + "version": "0.8" +} diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json index a3ebd13ffcb..9411c58dde3 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json @@ -100,7 +100,7 @@ "message" ] }, - "version": "0.7" + "version": "0.8" }, "index_uid": "my-index:00000000000000000000000000", "sources": [ @@ -121,5 +121,5 @@ "version": "0.8" } ], - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json index a3ebd13ffcb..9411c58dde3 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json @@ -100,7 +100,7 @@ "message" ] }, - "version": "0.7" + "version": "0.8" }, "index_uid": "my-index:00000000000000000000000000", "sources": [ @@ -121,5 +121,5 @@ "version": "0.8" } ], - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json index b29bd173750..bd3c52e5e91 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json @@ -107,7 +107,7 @@ "message" ] }, - "version": "0.7" + "version": "0.8" }, "index_uid": "my-index:00000000000000000000000000", "sources": [ @@ -128,5 +128,5 @@ "version": "0.8" } ], - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.7.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.7.expected.json index b29bd173750..bd3c52e5e91 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.7.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.7.expected.json @@ -107,7 +107,7 @@ "message" ] }, - "version": "0.7" + "version": "0.8" }, "index_uid": "my-index:00000000000000000000000000", "sources": [ @@ -128,5 +128,5 @@ "version": "0.8" } ], - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.expected.json new file mode 100644 index 00000000000..bd3c52e5e91 --- /dev/null +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.expected.json @@ -0,0 +1,132 @@ +{ + "checkpoint": { + "kafka-source": { + "00000000000000000000": "00000000000000000042" + } + }, + "create_timestamp": 1789, + "index_config": { + "doc_mapping": { + "dynamic_mapping": { + "expand_dots": true, + "fast": { + "normalizer": "raw" + }, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "raw" + }, + "field_mappings": [ + { + "coerce": true, + "fast": true, + "indexed": true, + "name": "tenant_id", + "output_format": "number", + "stored": true, + "type": "u64" + }, + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "timestamp", + "output_format": "rfc3339", + "stored": true, + "type": "datetime" + }, + { + "fast": false, + "fieldnorms": false, + "indexed": true, + "name": "log_level", + "record": "basic", + "stored": true, + "tokenizer": "raw", + "type": "text" + }, + { + "fast": false, + "fieldnorms": false, + "indexed": true, + "name": "message", + "record": "position", + "stored": true, + "tokenizer": "default", + "type": "text" + } + ], + "index_field_presence": true, + "max_num_partitions": 100, + "mode": "dynamic", + "partition_key": "tenant_id", + "store_source": true, + "tag_fields": [ + "log_level", + "tenant_id" + ], + "timestamp_field": "timestamp", + "tokenizers": [ + { + "filters": [], + "name": "custom_tokenizer", + "pattern": "[^\\p{L}\\p{N}]+", + "type": "regex" + } + ] + }, + "index_id": "my-index", + "index_uri": "s3://quickwit-indexes/my-index", + "indexing_settings": { + "commit_timeout_secs": 301, + "docstore_blocksize": 1000000, + "docstore_compression_level": 8, + "merge_policy": { + "maturation_period": "2days", + "max_merge_factor": 11, + "merge_factor": 9, + "min_level_num_docs": 100000, + "type": "stable_log" + }, + "resources": { + "heap_size": "50.0 MB" + }, + "split_num_docs_target": 10000001 + }, + "retention": { + "period": "90 days", + "schedule": "daily" + }, + "search_settings": { + "default_search_fields": [ + "message" + ] + }, + "version": "0.8" + }, + "index_uid": "my-index:00000000000000000000000000", + "sources": [ + { + "enabled": true, + "input_format": "json", + "num_pipelines": 2, + "params": { + "client_params": {}, + "topic": "kafka-topic" + }, + "source_id": "kafka-source", + "source_type": "kafka", + "transform": { + "script": ".message = downcase(string!(.message))", + "timezone": "UTC" + }, + "version": "0.8" + } + ], + "version": "0.8" +} diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.json new file mode 100644 index 00000000000..bd3c52e5e91 --- /dev/null +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.8.json @@ -0,0 +1,132 @@ +{ + "checkpoint": { + "kafka-source": { + "00000000000000000000": "00000000000000000042" + } + }, + "create_timestamp": 1789, + "index_config": { + "doc_mapping": { + "dynamic_mapping": { + "expand_dots": true, + "fast": { + "normalizer": "raw" + }, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "raw" + }, + "field_mappings": [ + { + "coerce": true, + "fast": true, + "indexed": true, + "name": "tenant_id", + "output_format": "number", + "stored": true, + "type": "u64" + }, + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "timestamp", + "output_format": "rfc3339", + "stored": true, + "type": "datetime" + }, + { + "fast": false, + "fieldnorms": false, + "indexed": true, + "name": "log_level", + "record": "basic", + "stored": true, + "tokenizer": "raw", + "type": "text" + }, + { + "fast": false, + "fieldnorms": false, + "indexed": true, + "name": "message", + "record": "position", + "stored": true, + "tokenizer": "default", + "type": "text" + } + ], + "index_field_presence": true, + "max_num_partitions": 100, + "mode": "dynamic", + "partition_key": "tenant_id", + "store_source": true, + "tag_fields": [ + "log_level", + "tenant_id" + ], + "timestamp_field": "timestamp", + "tokenizers": [ + { + "filters": [], + "name": "custom_tokenizer", + "pattern": "[^\\p{L}\\p{N}]+", + "type": "regex" + } + ] + }, + "index_id": "my-index", + "index_uri": "s3://quickwit-indexes/my-index", + "indexing_settings": { + "commit_timeout_secs": 301, + "docstore_blocksize": 1000000, + "docstore_compression_level": 8, + "merge_policy": { + "maturation_period": "2days", + "max_merge_factor": 11, + "merge_factor": 9, + "min_level_num_docs": 100000, + "type": "stable_log" + }, + "resources": { + "heap_size": "50.0 MB" + }, + "split_num_docs_target": 10000001 + }, + "retention": { + "period": "90 days", + "schedule": "daily" + }, + "search_settings": { + "default_search_fields": [ + "message" + ] + }, + "version": "0.8" + }, + "index_uid": "my-index:00000000000000000000000000", + "sources": [ + { + "enabled": true, + "input_format": "json", + "num_pipelines": 2, + "params": { + "client_params": {}, + "topic": "kafka-topic" + }, + "source_id": "kafka-source", + "source_type": "kafka", + "transform": { + "script": ".message = downcase(string!(.message))", + "timezone": "UTC" + }, + "version": "0.8" + } + ], + "version": "0.8" +} diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.7.expected.json b/quickwit/quickwit-metastore/test-data/manifest/v0.7.expected.json index 73c01cd19b1..2f1a2f647ac 100644 --- a/quickwit/quickwit-metastore/test-data/manifest/v0.7.expected.json +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.7.expected.json @@ -81,8 +81,8 @@ "default_search_fields": [] }, "template_id": "test-template", - "version": "0.7" + "version": "0.8" } ], - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.8.expected.json b/quickwit/quickwit-metastore/test-data/manifest/v0.8.expected.json new file mode 100644 index 00000000000..2f1a2f647ac --- /dev/null +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.8.expected.json @@ -0,0 +1,88 @@ +{ + "indexes": { + "test-index-1": "creating", + "test-index-2": "active", + "test-index-3": "deleting" + }, + "templates": [ + { + "description": "Test description.", + "doc_mapping": { + "dynamic_mapping": { + "expand_dots": true, + "fast": { + "normalizer": "raw" + }, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "raw" + }, + "field_mappings": [ + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "ts", + "output_format": "rfc3339", + "stored": true, + "type": "datetime" + }, + { + "expand_dots": true, + "fast": false, + "indexed": true, + "name": "message", + "record": "basic", + "stored": true, + "tokenizer": "raw", + "type": "json" + } + ], + "index_field_presence": false, + "max_num_partitions": 200, + "mode": "dynamic", + "store_source": false, + "tag_fields": [], + "timestamp_field": "ts", + "tokenizers": [] + }, + "index_id_patterns": [ + "test-index-foo*", + "-test-index-foobar" + ], + "index_root_uri": "ram:///indexes", + "indexing_settings": { + "commit_timeout_secs": 60, + "docstore_blocksize": 1000000, + "docstore_compression_level": 8, + "merge_policy": { + "maturation_period": "2days", + "max_merge_factor": 12, + "merge_factor": 10, + "min_level_num_docs": 100000, + "type": "stable_log" + }, + "resources": { + "heap_size": "2.0 GB" + }, + "split_num_docs_target": 10000000 + }, + "priority": 100, + "retention": { + "period": "42 days", + "schedule": "daily" + }, + "search_settings": { + "default_search_fields": [] + }, + "template_id": "test-template", + "version": "0.8" + } + ], + "version": "0.8" +} diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.8.json b/quickwit/quickwit-metastore/test-data/manifest/v0.8.json new file mode 100644 index 00000000000..2f1a2f647ac --- /dev/null +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.8.json @@ -0,0 +1,88 @@ +{ + "indexes": { + "test-index-1": "creating", + "test-index-2": "active", + "test-index-3": "deleting" + }, + "templates": [ + { + "description": "Test description.", + "doc_mapping": { + "dynamic_mapping": { + "expand_dots": true, + "fast": { + "normalizer": "raw" + }, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "raw" + }, + "field_mappings": [ + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "ts", + "output_format": "rfc3339", + "stored": true, + "type": "datetime" + }, + { + "expand_dots": true, + "fast": false, + "indexed": true, + "name": "message", + "record": "basic", + "stored": true, + "tokenizer": "raw", + "type": "json" + } + ], + "index_field_presence": false, + "max_num_partitions": 200, + "mode": "dynamic", + "store_source": false, + "tag_fields": [], + "timestamp_field": "ts", + "tokenizers": [] + }, + "index_id_patterns": [ + "test-index-foo*", + "-test-index-foobar" + ], + "index_root_uri": "ram:///indexes", + "indexing_settings": { + "commit_timeout_secs": 60, + "docstore_blocksize": 1000000, + "docstore_compression_level": 8, + "merge_policy": { + "maturation_period": "2days", + "max_merge_factor": 12, + "merge_factor": 10, + "min_level_num_docs": 100000, + "type": "stable_log" + }, + "resources": { + "heap_size": "2.0 GB" + }, + "split_num_docs_target": 10000000 + }, + "priority": 100, + "retention": { + "period": "42 days", + "schedule": "daily" + }, + "search_settings": { + "default_search_fields": [] + }, + "template_id": "test-template", + "version": "0.8" + } + ], + "version": "0.8" +} diff --git a/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.expected.json index d83ecc012d9..d6475bd3525 100644 --- a/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.expected.json +++ b/quickwit/quickwit-metastore/test-data/split-metadata/v0.6.expected.json @@ -25,5 +25,5 @@ "start": 121000 }, "uncompressed_docs_size_in_bytes": 234234, - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/split-metadata/v0.7.expected.json b/quickwit/quickwit-metastore/test-data/split-metadata/v0.7.expected.json index d83ecc012d9..d6475bd3525 100644 --- a/quickwit/quickwit-metastore/test-data/split-metadata/v0.7.expected.json +++ b/quickwit/quickwit-metastore/test-data/split-metadata/v0.7.expected.json @@ -25,5 +25,5 @@ "start": 121000 }, "uncompressed_docs_size_in_bytes": 234234, - "version": "0.7" + "version": "0.8" } diff --git a/quickwit/quickwit-metastore/test-data/split-metadata/v0.8.expected.json b/quickwit/quickwit-metastore/test-data/split-metadata/v0.8.expected.json new file mode 100644 index 00000000000..d6475bd3525 --- /dev/null +++ b/quickwit/quickwit-metastore/test-data/split-metadata/v0.8.expected.json @@ -0,0 +1,29 @@ +{ + "create_timestamp": 3, + "delete_opstamp": 10, + "footer_offsets": { + "end": 2000, + "start": 1000 + }, + "index_uid": "my-index:00000000000000000000000000", + "maturity": { + "maturation_period_millis": 4000, + "type": "immature" + }, + "node_id": "node", + "num_docs": 12303, + "num_merge_ops": 3, + "partition_id": 7, + "source_id": "source", + "split_id": "split", + "tags": [ + "234", + "aaa" + ], + "time_range": { + "end": 130198, + "start": 121000 + }, + "uncompressed_docs_size_in_bytes": 234234, + "version": "0.8" +} diff --git a/quickwit/quickwit-metastore/test-data/split-metadata/v0.8.json b/quickwit/quickwit-metastore/test-data/split-metadata/v0.8.json new file mode 100644 index 00000000000..d6475bd3525 --- /dev/null +++ b/quickwit/quickwit-metastore/test-data/split-metadata/v0.8.json @@ -0,0 +1,29 @@ +{ + "create_timestamp": 3, + "delete_opstamp": 10, + "footer_offsets": { + "end": 2000, + "start": 1000 + }, + "index_uid": "my-index:00000000000000000000000000", + "maturity": { + "maturation_period_millis": 4000, + "type": "immature" + }, + "node_id": "node", + "num_docs": 12303, + "num_merge_ops": 3, + "partition_id": 7, + "source_id": "source", + "split_id": "split", + "tags": [ + "234", + "aaa" + ], + "time_range": { + "end": 130198, + "start": 121000 + }, + "uncompressed_docs_size_in_bytes": 234234, + "version": "0.8" +}