diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 28bec93593d..0db9bbed0de 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -6049,6 +6049,7 @@ dependencies = [ "hyper 0.14.29", "itertools 0.13.0", "quickwit-actors", + "quickwit-cli", "quickwit-common", "quickwit-config", "quickwit-metastore", diff --git a/quickwit/quickwit-integration-tests/Cargo.toml b/quickwit/quickwit-integration-tests/Cargo.toml index 269b8276ccf..7030d2ba004 100644 --- a/quickwit/quickwit-integration-tests/Cargo.toml +++ b/quickwit/quickwit-integration-tests/Cargo.toml @@ -25,6 +25,7 @@ tonic = { workspace = true } tracing = { workspace = true } quickwit-actors = { workspace = true, features = ["testsuite"] } +quickwit-cli = { workspace = true } quickwit-common = { workspace = true, features = ["testsuite"] } quickwit-config = { workspace = true, features = ["testsuite"] } quickwit-metastore = { workspace = true, features = ["testsuite"] } diff --git a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs index 5be71dc5620..824e24988e1 100644 --- a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs +++ b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs @@ -18,6 +18,7 @@ // along with this program. If not, see . use std::collections::{HashMap, HashSet}; +use std::io::Write; use std::net::SocketAddr; use std::path::PathBuf; use std::str::FromStr; @@ -26,6 +27,7 @@ use std::time::{Duration, Instant}; use futures_util::future; use itertools::Itertools; use quickwit_actors::ActorExitStatus; +use quickwit_cli::tool::{local_ingest_docs_cli, LocalIngestDocsArgs}; use quickwit_common::new_coolid; use quickwit_common::runtimes::RuntimesConfig; use quickwit_common::test_utils::{wait_for_server_ready, wait_until_predicate}; @@ -44,6 +46,7 @@ use quickwit_rest_client::rest_client::{ use quickwit_serve::{serve_quickwit, ListSplitsQueryParams}; use quickwit_storage::StorageResolver; use reqwest::Url; +use serde_json::Value; use tempfile::TempDir; use tokio::sync::watch::{self, Receiver, Sender}; use tokio::task::JoinHandle; @@ -383,6 +386,45 @@ impl ClusterSandbox { Ok(()) } + pub async fn local_ingest(&self, index_id: &str, json_data: &[Value]) -> anyhow::Result<()> { + let test_conf = self + .node_configs + .iter() + .find(|config| config.services.contains(&QuickwitService::Indexer)) + .ok_or(anyhow::anyhow!("No indexer node found"))?; + // NodeConfig cannot be serialized, we write our own simplified config + let mut tmp_config_file = tempfile::Builder::new().suffix(".yaml").tempfile().unwrap(); + let node_config = format!( + r#" + version: 0.8 + metastore_uri: {} + data_dir: {:?} + "#, + test_conf.node_config.metastore_uri, test_conf.node_config.data_dir_path + ); + tmp_config_file.write_all(node_config.as_bytes())?; + tmp_config_file.flush()?; + + let mut tmp_data_file = tempfile::NamedTempFile::new().unwrap(); + for line in json_data { + serde_json::to_writer(&mut tmp_data_file, line)?; + tmp_data_file.write_all(b"\n")?; + } + tmp_data_file.flush()?; + + local_ingest_docs_cli(LocalIngestDocsArgs { + clear_cache: false, + config_uri: QuickwitUri::from_str(tmp_config_file.path().to_str().unwrap())?, + index_id: index_id.to_string(), + input_format: quickwit_config::SourceInputFormat::Json, + overwrite: false, + vrl_script: None, + input_path_opt: Some(tmp_data_file.path().to_path_buf()), + }) + .await?; + Ok(()) + } + pub async fn shutdown(self) -> Result>, anyhow::Error> { // We need to drop rest clients first because reqwest can hold connections open // preventing rest server's graceful shutdown. diff --git a/quickwit/quickwit-integration-tests/src/tests/mod.rs b/quickwit/quickwit-integration-tests/src/tests/mod.rs index 76322012b30..9c50db45bbc 100644 --- a/quickwit/quickwit-integration-tests/src/tests/mod.rs +++ b/quickwit/quickwit-integration-tests/src/tests/mod.rs @@ -19,4 +19,4 @@ mod basic_tests; mod index_tests; -mod index_update_tests; +mod update_tests; diff --git a/quickwit/quickwit-integration-tests/src/tests/update_tests/doc_mapping_tests.rs b/quickwit/quickwit-integration-tests/src/tests/update_tests/doc_mapping_tests.rs new file mode 100644 index 00000000000..422ab6e850f --- /dev/null +++ b/quickwit/quickwit-integration-tests/src/tests/update_tests/doc_mapping_tests.rs @@ -0,0 +1,512 @@ +// Copyright (C) 2024 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::HashSet; +use std::time::Duration; + +use quickwit_config::service::QuickwitService; +use serde_json::{json, Value}; + +use super::assert_hits_unordered; +use crate::test_utils::ClusterSandbox; + +/// Update the doc mapping between 2 calls to local-ingest (forces separate indexing pipelines) and +/// assert the number of hits for the given query +async fn validate_search_across_doc_mapping_updates( + index_id: &str, + original_doc_mapping: Value, + ingest_before_update: &[Value], + updated_doc_mapping: Value, + ingest_after_update: &[Value], + query_and_expect: &[(&str, Result<&[Value], ()>)], +) { + quickwit_common::setup_logging_for_tests(); + let nodes_services = vec![HashSet::from_iter([ + QuickwitService::Searcher, + QuickwitService::Metastore, + QuickwitService::Indexer, + QuickwitService::ControlPlane, + QuickwitService::Janitor, + ])]; + let sandbox = ClusterSandbox::start_cluster_nodes(&nodes_services) + .await + .unwrap(); + sandbox.wait_for_cluster_num_ready_nodes(1).await.unwrap(); + + { + // Wait for indexer to fully start. + // The starting time is a bit long for a cluster. + tokio::time::sleep(Duration::from_secs(3)).await; + let indexing_service_counters = sandbox + .indexer_rest_client + .node_stats() + .indexing() + .await + .unwrap(); + assert_eq!(indexing_service_counters.num_running_pipelines, 0); + } + + // Create index + sandbox + .indexer_rest_client + .indexes() + .create( + json!({ + "version": "0.8", + "index_id": index_id, + "doc_mapping": original_doc_mapping, + "indexing_settings": { + "commit_timeout_secs": 1 + }, + }) + .to_string(), + quickwit_config::ConfigFormat::Json, + false, + ) + .await + .unwrap(); + + assert!(sandbox + .indexer_rest_client + .node_health() + .is_live() + .await + .unwrap()); + + // Wait until indexing pipelines are started. + sandbox.wait_for_indexing_pipelines(1).await.unwrap(); + + // We use local ingest to always pick up the latest doc mapping + sandbox + .local_ingest(index_id, ingest_before_update) + .await + .unwrap(); + + // Update index to also search "body" by default, search should now have 1 hit + sandbox + .searcher_rest_client + .indexes() + .update( + index_id, + json!({ + "version": "0.8", + "index_id": index_id, + "doc_mapping": updated_doc_mapping, + "indexing_settings": { + "commit_timeout_secs": 1, + }, + }) + .to_string(), + quickwit_config::ConfigFormat::Json, + ) + .await + .unwrap(); + + sandbox + .local_ingest(index_id, ingest_after_update) + .await + .unwrap(); + + for (query, expected_hits) in query_and_expect.iter().copied() { + assert_hits_unordered(&sandbox, index_id, query, expected_hits).await; + } + + sandbox.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_update_doc_mapping_text_to_u64() { + let index_id = "update-text-to-u64"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text"} + ] + }); + let ingest_before_update = &[json!({"body": "14"}), json!({"body": "15"})]; + let updated_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "u64"} + ] + }); + let ingest_after_update = &[json!({"body": 16}), json!({"body": 17})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:14", Ok(&[json!({"body": 14})])), + ("body:16", Ok(&[json!({"body": 16})])), + // error expected because the validation is performed + // by latest doc mapping + ("body:hello", Err(())), + ], + ) + .await; +} + +#[tokio::test] +async fn test_update_doc_mapping_u64_to_text() { + let index_id = "update-u64-to-text"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "u64"} + ], + "mode": "strict", + }); + let ingest_before_update = &[json!({"body": 14}), json!({"body": 15})]; + let updated_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text"}, + ], + "mode": "strict", + }); + let ingest_after_update = &[json!({"body": "16"}), json!({"body": "hello world"})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:14", Ok(&[json!({"body": "14"})])), + ("body:16", Ok(&[json!({"body": "16"})])), + ("body:hello", Ok(&[json!({"body": "hello world"})])), + ], + ) + .await; +} + +#[tokio::test] +async fn test_update_doc_mapping_json_to_text() { + let index_id = "update-json-to-text"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "json"} + ] + }); + let ingest_before_update = &[ + json!({"body": {"field1": "hello"}}), + json!({"body": {"field2": "world"}}), + ]; + let updated_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text"} + ] + }); + let ingest_after_update = &[json!({"body": "hello world"})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:hello", Ok(&[json!({"body": "hello world"})])), + // error expected because the validation is performed + // by latest doc mapping + ("body.field1:hello", Err(())), + ], + ) + .await; +} + +#[tokio::test] +#[ignore] +async fn test_update_doc_mapping_json_to_object() { + let index_id = "update-json-to-object"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "json"} + ] + }); + let ingest_before_update = &[ + json!({"body": {"field1": "hello"}}), + json!({"body": {"field2": "world"}}), + ]; + let updated_doc_mappings = json!({ + "field_mappings": [ + { + "name": "body", + "type": "object", + "field_mappings": [ + {"name": "field1", "type": "text"}, + {"name": "field2", "type": "text"}, + ] + } + ] + }); + let ingest_after_update = &[ + json!({"body": {"field1": "hola"}}), + json!({"body": {"field2": "mundo"}}), + ]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ( + "body.field1:hello", + Ok(&[json!({"body": {"field1": "hello"}})]), + ), + ( + "body.field1:hola", + Ok(&[json!({"body": {"field1": "hola"}})]), + ), + ], + ) + .await; +} + +// TODO expected to be fix as part of #5084 +#[tokio::test] +#[ignore] +async fn test_update_doc_mapping_tokenizer_default_to_raw() { + let index_id = "update-tokenizer-default-to-raw"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text", "tokenizer": "default"} + ] + }); + let ingest_before_update = &[json!({"body": "hello-world"})]; + let updated_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text", "tokenizer": "raw"} + ] + }); + let ingest_after_update = &[json!({"body": "bonjour-monde"})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:hello", Ok(&[json!({"body": "hello-world"})])), + ("body:world", Ok(&[json!({"body": "bonjour-monde"})])), + // phrases queries won't apply to older splits that didn't support them + ("body:\"hello world\"", Ok(&[])), + ("body:\"hello-world\"", Ok(&[])), + ("body:bonjour", Ok(&[])), + ("body:monde", Ok(&[])), + // the raw tokenizer only returns exact matches + ("body:\"bonjour monde\"", Ok(&[])), + ( + "body:\"bonjour-monde\"", + Ok(&[json!({"body": "bonjour-monde"})]), + ), + ], + ) + .await; +} + +// TODO expected to be fix as part of #5084 +#[tokio::test] +#[ignore] +async fn test_update_doc_mapping_tokenizer_add_position() { + let index_id = "update-tokenizer-add-position"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text", "tokenizer": "default"} + ] + }); + let ingest_before_update = &[json!({"body": "hello-world"})]; + let updated_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text", "tokenizer": "default", "record": "position"} + ] + }); + let ingest_after_update = &[json!({"body": "bonjour-monde"})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:hello", Ok(&[json!({"body": "hello-world"})])), + ("body:world", Ok(&[json!({"body": "hello-world"})])), + // phrases queries don't apply to older splits that didn't support them + ("body:\"hello-world\"", Ok(&[])), + ("body:\"hello world\"", Ok(&[])), + ("body:bonjour", Ok(&[json!({"body": "bonjour-monde"})])), + ("body:monde", Ok(&[json!({"body": "bonjour-monde"})])), + ( + "body:\"bonjour-monde\"", + Ok(&[json!({"body": "bonjour-monde"})]), + ), + ( + "body:\"bonjour monde\"", + Ok(&[json!({"body": "bonjour-monde"})]), + ), + ], + ) + .await; +} + +#[tokio::test] +async fn test_update_doc_mapping_tokenizer_raw_to_phrase() { + let index_id = "update-tokenizer-raw-to-phrase"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text", "tokenizer": "raw"} + ] + }); + let ingest_before_update = &[json!({"body": "hello-world"})]; + let updated_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text", "tokenizer": "default", "record": "position"} + ] + }); + let ingest_after_update = &[json!({"body": "bonjour-monde"})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:hello", Ok(&[])), + ("body:world", Ok(&[])), + // raw tokenizer used here, only exact matches returned + ( + "body:\"hello-world\"", + Ok(&[json!({"body": "hello-world"})]), + ), + ("body:\"hello world\"", Ok(&[])), + ("body:bonjour", Ok(&[json!({"body": "bonjour-monde"})])), + ("body:monde", Ok(&[json!({"body": "bonjour-monde"})])), + ( + "body:\"bonjour-monde\"", + Ok(&[json!({"body": "bonjour-monde"})]), + ), + ( + "body:\"bonjour monde\"", + Ok(&[json!({"body": "bonjour-monde"})]), + ), + ], + ) + .await; +} + +#[tokio::test] +#[ignore] +async fn test_update_doc_mapping_strict_to_dynamic() { + let index_id = "update-strict-to-dynamic"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text"} + ], + "mode": "strict", + }); + let ingest_before_update = &[json!({"body": "hello"})]; + let updated_doc_mappings = json!({ + "mode": "dynamic", + }); + let ingest_after_update = &[json!({"body": "world", "title": "salutations"})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:hello", Ok(&[json!({"body": "hello"})])), + ( + "body:world", + Ok(&[json!({"body": "world", "title": "salutations"})]), + ), + ( + "title:salutations", + Ok(&[json!({"body": "world", "title": "salutations"})]), + ), + ], + ) + .await; +} + +#[tokio::test] +async fn test_update_doc_mapping_dynamic_to_strict() { + let index_id = "update-dynamic-to-strict"; + let original_doc_mappings = json!({ + "mode": "dynamic", + }); + let ingest_before_update = &[json!({"body": "hello"})]; + let updated_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text"} + ], + "mode": "strict", + }); + let ingest_after_update = &[json!({"body": "world"})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:hello", Ok(&[json!({"body": "hello"})])), + ("body:world", Ok(&[json!({"body": "world"})])), + ], + ) + .await; +} + +#[tokio::test] +async fn test_update_doc_mapping_add_field_on_strict() { + let index_id = "update-add-field-on-strict"; + let original_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text"}, + ], + "mode": "strict", + }); + let ingest_before_update = &[json!({"body": "hello"})]; + let updated_doc_mappings = json!({ + "field_mappings": [ + {"name": "body", "type": "text"}, + {"name": "title", "type": "text"}, + ], + "mode": "strict", + }); + let ingest_after_update = &[json!({"body": "world", "title": "salutations"})]; + validate_search_across_doc_mapping_updates( + index_id, + original_doc_mappings, + ingest_before_update, + updated_doc_mappings, + ingest_after_update, + &[ + ("body:hello", Ok(&[json!({"body": "hello"})])), + ( + "body:world", + Ok(&[json!({"body": "world", "title": "salutations"})]), + ), + ( + "title:salutations", + Ok(&[json!({"body": "world", "title": "salutations"})]), + ), + ], + ) + .await; +} diff --git a/quickwit/quickwit-integration-tests/src/tests/update_tests/mod.rs b/quickwit/quickwit-integration-tests/src/tests/update_tests/mod.rs new file mode 100644 index 00000000000..4871be449ef --- /dev/null +++ b/quickwit/quickwit-integration-tests/src/tests/update_tests/mod.rs @@ -0,0 +1,67 @@ +// Copyright (C) 2024 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use quickwit_serve::SearchRequestQueryString; +use serde_json::Value; + +use crate::test_utils::ClusterSandbox; + +/// Checks that the result of the given query matches the expected values +async fn assert_hits_unordered( + sandbox: &ClusterSandbox, + index_id: &str, + query: &str, + expected_result: Result<&[Value], ()>, +) { + let search_res = sandbox + .searcher_rest_client + .search( + index_id, + SearchRequestQueryString { + query: query.to_string(), + max_hits: expected_result.map(|hits| hits.len() as u64).unwrap_or(1), + ..Default::default() + }, + ) + .await; + if let Ok(expected_hits) = expected_result { + let resp = search_res.unwrap_or_else(|_| panic!("query: {}", query)); + assert_eq!(resp.errors.len(), 0, "query: {}", query); + assert_eq!( + resp.num_hits, + expected_hits.len() as u64, + "query: {}", + query + ); + for expected_hit in expected_hits { + assert!( + resp.hits.contains(expected_hit), + "query: {} -> expected hits: {:?}, got: {:?}", + query, + expected_hits, + resp.hits + ); + } + } else if let Ok(search_response) = search_res { + assert!(!search_response.errors.is_empty(), "query: {}", query); + } +} + +mod doc_mapping_tests; +mod search_settings_tests; diff --git a/quickwit/quickwit-integration-tests/src/tests/index_update_tests.rs b/quickwit/quickwit-integration-tests/src/tests/update_tests/search_settings_tests.rs similarity index 75% rename from quickwit/quickwit-integration-tests/src/tests/index_update_tests.rs rename to quickwit/quickwit-integration-tests/src/tests/update_tests/search_settings_tests.rs index 18e7d3f80a6..52d43627f22 100644 --- a/quickwit/quickwit-integration-tests/src/tests/index_update_tests.rs +++ b/quickwit/quickwit-integration-tests/src/tests/update_tests/search_settings_tests.rs @@ -22,14 +22,14 @@ use std::time::Duration; use quickwit_config::service::QuickwitService; use quickwit_rest_client::rest_client::CommitType; -use quickwit_serve::SearchRequestQueryString; use serde_json::json; +use super::assert_hits_unordered; use crate::ingest_json; use crate::test_utils::{ingest_with_retry, ClusterSandbox}; #[tokio::test] -async fn test_update_on_multi_nodes_cluster() { +async fn test_update_search_settings_on_multi_nodes_cluster() { quickwit_common::setup_logging_for_tests(); let nodes_services = vec![ HashSet::from_iter([QuickwitService::Searcher]), @@ -56,7 +56,7 @@ async fn test_update_on_multi_nodes_cluster() { assert_eq!(indexing_service_counters.num_running_pipelines, 0); } - // Create index + // Create an index sandbox .indexer_rest_client .indexes() @@ -87,36 +87,28 @@ async fn test_update_on_multi_nodes_cluster() { .await .unwrap()); - // Wait until indexing pipelines are started. + // Wait until indexing pipelines are started sandbox.wait_for_indexing_pipelines(1).await.unwrap(); - // Check that ingest request send to searcher is forwarded to indexer and thus indexed. ingest_with_retry( - &sandbox.searcher_rest_client, + &sandbox.indexer_rest_client, "my-updatable-index", ingest_json!({"title": "first", "body": "first record"}), CommitType::Auto, ) .await .unwrap(); - // Wait until split is committed and search. + + // Wait until split is committed tokio::time::sleep(Duration::from_secs(4)).await; - // No hit because default_search_fields covers "title" only - let search_response_no_hit = sandbox - .searcher_rest_client - .search( - "my-updatable-index", - SearchRequestQueryString { - query: "record".to_string(), - ..Default::default() - }, - ) - .await - .unwrap(); - assert_eq!(search_response_no_hit.num_hits, 0); - // Update index to also search "body" by default, search should now have 1 hit + + // No hit because `default_search_fields`` only covers the `title` field + assert_hits_unordered(&sandbox, "my-updatable-index", "record", Ok(&[])).await; + + // Update the index to also search `body` by default, the same search should + // now have 1 hit sandbox - .searcher_rest_client + .indexer_rest_client .indexes() .update( "my-updatable-index", @@ -138,17 +130,14 @@ async fn test_update_on_multi_nodes_cluster() { ) .await .unwrap(); - let search_response_no_hit = sandbox - .searcher_rest_client - .search( - "my-updatable-index", - SearchRequestQueryString { - query: "record".to_string(), - ..Default::default() - }, - ) - .await - .unwrap(); - assert_eq!(search_response_no_hit.num_hits, 1); + + assert_hits_unordered( + &sandbox, + "my-updatable-index", + "record", + Ok(&[json!({"title": "first", "body": "first record"})]), + ) + .await; + sandbox.shutdown().await.unwrap(); }