Skip to content
This repository was archived by the owner on Apr 4, 2023. It is now read-only.

Commit 65df5fb

Browse files
MarinPostmaMarinPostma
committed
fix tests
1 parent 0177030 commit 65df5fb

File tree

8 files changed

+245
-264
lines changed

8 files changed

+245
-264
lines changed

‎milli/src/index.rs‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -808,7 +808,7 @@ pub(crate) mod tests {
808808
use maplit::btreemap;
809809
use tempfile::TempDir;
810810

811-
use crate::update::{IndexDocuments, UpdateFormat};
811+
use crate::update::{IndexDocuments};
812812
use crate::Index;
813813

814814
pub(crate) struct TempIndex {
@@ -844,13 +844,12 @@ pub(crate) mod tests {
844844
let index = Index::new(options, &path).unwrap();
845845

846846
let mut wtxn = index.write_txn().unwrap();
847-
let content = &br#"[
847+
let content = documents!([
848848
{ "id": 1, "name": "kevin" },
849849
{ "id": 2, "name": "bob", "age": 20 },
850850
{ "id": 2, "name": "bob", "age": 20 }
851-
]"#[..];
852-
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
853-
builder.update_format(UpdateFormat::Json);
851+
]);
852+
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
854853
builder.execute(content, |_, _| ()).unwrap();
855854
wtxn.commit().unwrap();
856855

@@ -869,8 +868,12 @@ pub(crate) mod tests {
869868
// we add all the documents a second time. we are supposed to get the same
870869
// field_distribution in the end
871870
let mut wtxn = index.write_txn().unwrap();
872-
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
873-
builder.update_format(UpdateFormat::Json);
871+
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
872+
let content = documents!([
873+
{ "id": 1, "name": "kevin" },
874+
{ "id": 2, "name": "bob", "age": 20 },
875+
{ "id": 2, "name": "bob", "age": 20 }
876+
]);
874877
builder.execute(content, |_, _| ()).unwrap();
875878
wtxn.commit().unwrap();
876879

@@ -887,13 +890,12 @@ pub(crate) mod tests {
887890
);
888891

889892
// then we update a document by removing one field and another by adding one field
890-
let content = &br#"[
893+
let content = documents!([
891894
{ "id": 1, "name": "kevin", "has_dog": true },
892895
{ "id": 2, "name": "bob" }
893-
]"#[..];
896+
]);
894897
let mut wtxn = index.write_txn().unwrap();
895-
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
896-
builder.update_format(UpdateFormat::Json);
898+
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
897899
builder.execute(content, |_, _| ()).unwrap();
898900
wtxn.commit().unwrap();
899901

‎milli/src/lib.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
#[macro_use]
22
extern crate pest_derive;
33

4+
#[macro_use]
5+
pub mod documents;
6+
47
mod criterion;
58
mod error;
69
mod external_documents_ids;
@@ -12,7 +15,6 @@ pub mod proximity;
1215
mod search;
1316
pub mod tree_level;
1417
pub mod update;
15-
pub mod documents;
1618

1719
use std::borrow::Cow;
1820
use std::collections::{BTreeMap, HashMap};

‎milli/src/search/distinct/mod.rs‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,21 +27,24 @@ pub trait Distinct {
2727
#[cfg(test)]
2828
mod test {
2929
use std::collections::HashSet;
30+
use std::io::Cursor;
3031

32+
use bimap::BiHashMap;
3133
use once_cell::sync::Lazy;
3234
use rand::seq::SliceRandom;
3335
use rand::Rng;
3436
use roaring::RoaringBitmap;
3537
use serde_json::{json, Value};
3638

39+
use crate::documents::{DocumentsBuilder, DocumentsReader};
3740
use crate::index::tests::TempIndex;
3841
use crate::index::Index;
39-
use crate::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat};
42+
use crate::update::{IndexDocumentsMethod, UpdateBuilder};
4043
use crate::{DocumentId, FieldId, BEU32};
4144

42-
static JSON: Lazy<Value> = Lazy::new(generate_json);
45+
static JSON: Lazy<Vec<u8>> = Lazy::new(generate_json);
4346

44-
fn generate_json() -> Value {
47+
fn generate_json() -> Vec<u8> {
4548
let mut rng = rand::thread_rng();
4649
let num_docs = rng.gen_range(10..30);
4750

@@ -69,7 +72,13 @@ mod test {
6972
documents.push(doc);
7073
}
7174

72-
Value::Array(documents)
75+
let mut cursor = Cursor::new(Vec::new());
76+
let mut builder = DocumentsBuilder::new(&mut cursor, BiHashMap::new()).unwrap();
77+
78+
builder.add_documents(documents).unwrap();
79+
builder.finish().unwrap();
80+
81+
cursor.into_inner()
7382
}
7483

7584
/// Returns a temporary index populated with random test documents, the FieldId for the
@@ -89,13 +98,14 @@ mod test {
8998
let mut addition = builder.index_documents(&mut txn, &index);
9099

91100
addition.index_documents_method(IndexDocumentsMethod::ReplaceDocuments);
92-
addition.update_format(UpdateFormat::Json);
93-
addition.execute(JSON.to_string().as_bytes(), |_, _| ()).unwrap();
101+
let reader = crate::documents::DocumentsReader::from_reader(Cursor::new(&*JSON)).unwrap();
102+
addition.execute(reader, |_, _| ()).unwrap();
94103

95104
let fields_map = index.fields_ids_map(&txn).unwrap();
96105
let fid = fields_map.id(&distinct).unwrap();
97106

98-
let map = (0..JSON.as_array().unwrap().len() as u32).collect();
107+
let documents = DocumentsReader::from_reader(Cursor::new(&*JSON)).unwrap();
108+
let map = (0..documents.len() as u32).collect();
99109

100110
txn.commit().unwrap();
101111

‎milli/src/update/clear_documents.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ mod tests {
8080
use heed::EnvOpenOptions;
8181

8282
use super::*;
83-
use crate::update::{IndexDocuments, UpdateFormat};
83+
use crate::update::{IndexDocuments};
8484

8585
#[test]
8686
fn clear_documents() {
@@ -90,13 +90,12 @@ mod tests {
9090
let index = Index::new(options, &path).unwrap();
9191

9292
let mut wtxn = index.write_txn().unwrap();
93-
let content = &br#"[
93+
let content = documents!([
9494
{ "id": 0, "name": "kevin", "age": 20 },
9595
{ "id": 1, "name": "kevina" },
9696
{ "id": 2, "name": "benoit", "country": "France" }
97-
]"#[..];
98-
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
99-
builder.update_format(UpdateFormat::Json);
97+
]);
98+
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
10099
builder.execute(content, |_, _| ()).unwrap();
101100

102101
// Clear all documents from the database.

‎milli/src/update/delete_documents.rs‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -548,7 +548,7 @@ mod tests {
548548
use maplit::hashset;
549549

550550
use super::*;
551-
use crate::update::{IndexDocuments, Settings, UpdateFormat};
551+
use crate::update::{IndexDocuments, Settings};
552552
use crate::FilterCondition;
553553

554554
#[test]
@@ -559,13 +559,12 @@ mod tests {
559559
let index = Index::new(options, &path).unwrap();
560560

561561
let mut wtxn = index.write_txn().unwrap();
562-
let content = &br#"[
562+
let content = documents!([
563563
{ "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } },
564564
{ "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
565565
{ "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
566-
]"#[..];
567-
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
568-
builder.update_format(UpdateFormat::Json);
566+
]);
567+
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
569568
builder.execute(content, |_, _| ()).unwrap();
570569

571570
// delete those documents, ids are synchronous therefore 0, 1, and 2.
@@ -590,13 +589,12 @@ mod tests {
590589
let index = Index::new(options, &path).unwrap();
591590

592591
let mut wtxn = index.write_txn().unwrap();
593-
let content = &br#"[
592+
let content = documents!([
594593
{ "mysuperid": 0, "name": "kevin" },
595594
{ "mysuperid": 1, "name": "kevina" },
596595
{ "mysuperid": 2, "name": "benoit" }
597-
]"#[..];
598-
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
599-
builder.update_format(UpdateFormat::Json);
596+
]);
597+
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
600598
builder.execute(content, |_, _| ()).unwrap();
601599

602600
// Delete not all of the documents but some of them.
@@ -621,7 +619,7 @@ mod tests {
621619
builder.set_filterable_fields(hashset! { S("label") });
622620
builder.execute(|_, _| ()).unwrap();
623621

624-
let content = &br#"[
622+
let content = documents!([
625623
{"docid":"1_4","label":"sign"},
626624
{"docid":"1_5","label":"letter"},
627625
{"docid":"1_7","label":"abstract,cartoon,design,pattern"},
@@ -642,9 +640,8 @@ mod tests {
642640
{"docid":"1_58","label":"abstract,art,cartoon"},
643641
{"docid":"1_68","label":"design"},
644642
{"docid":"1_69","label":"geometry"}
645-
]"#[..];
646-
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
647-
builder.update_format(UpdateFormat::Json);
643+
]);
644+
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
648645
builder.execute(content, |_, _| ()).unwrap();
649646

650647
// Delete not all of the documents but some of them.

0 commit comments

Comments
 (0)