Skip to content

Latest commit

 

History

History
266 lines (249 loc) · 7.48 KB

2024_11_02_release.md

File metadata and controls

266 lines (249 loc) · 7.48 KB

TEST Code

Create Data

embeddings.InitEmbeddings()
	dataString := []string{
		"When I'm hungry, I usually have a sandwich.",
		"I tend to eat porridge whenever I feel hungry.",
		"For a typical meal, I often enjoy a bowl of salad.",
	}

	query := "I'm hungry, what should I eat?"

	addString := []string{
		"Sometimes, I grab a quick snack like nuts or fruit.",
		"On most days, I prefer having pasta for dinner.",
	}
	type InsertRecord struct {
		Metadata   map[string]interface{}
		Id         string
		BucketName string
		Vector     []float32
	}
	var uid string
	firstRecord := make([]InsertRecord, 0)
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[0],
			"type":        12,
			"gender":      "boy",
			"name":        "John",
		},
	})
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[1],
			"type":        12632,
			"gender":      "girl",
			"name":        "anna",
		},
	})
	uid = uuid.New().String()
	firstRecord = append(firstRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": dataString[2],
			"type":        6302,
			"gender":      "boy",
			"name":        "Mark",
		},
	})
	// create dataset
	originDataset := make([]*dataCoordinatorV1.ModifyDataset, 0, 3)
	for i, data := range firstRecord {
		vec, err := embeddings.TextEmbedding(dataString[i])
		if err != nil {
			log.Fatal(err)
		}
		metas, _ := msgpack.Marshal(data.Metadata)
		originDataset = append(originDataset, &dataCoordinatorV1.ModifyDataset{
			BucketName: "tbucket",
			Id:         data.Metadata["_id"].(string),
			Vector:     vec,
			Metadata:   metas,
		})
	}
	laterRecord := make([]InsertRecord, 0)
	uid = uuid.New().String()
	laterRecord = append(laterRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": addString[0],
			"type":        142124,
			"gender":      "girl",
			"name":        "Ann",
		},
	})
	uid = uuid.New().String()
	laterRecord = append(laterRecord, InsertRecord{
		Metadata: map[string]interface{}{
			"_id":         uid,
			"description": addString[1],
			"type":        122,
			"gender":      "girl",
			"name":        "swan",
		},
	})

	afterDataset := make([]*dataCoordinatorV1.ModifyDataset, 0, 3)
	for i, data := range laterRecord {
		vec, err := embeddings.TextEmbedding(addString[i])
		if err != nil {
			log.Fatal(err)
		}
		metas, _ := msgpack.Marshal(data.Metadata)
		afterDataset = append(afterDataset, &dataCoordinatorV1.ModifyDataset{
			BucketName: "tbucket",
			Id:         data.Metadata["_id"].(string),
			Vector:     vec,
			Metadata:   metas,
		})
	}
	conn, err := grpc.Dial(":50051", grpc.WithTransportCredentials(insecure.NewCredentials()))
	fmt.Println(err)
	rclient := resourceCoordinatorV1.NewResourceCoordinatorClient(conn)

	res, err := rclient.CreateBucket(context.Background(), &resourceCoordinatorV1.Bucket{
		BucketName: "tbucket",
		Dim:        384,
		Space:      resourceCoordinatorV1.Space_Cosine,
	})
	fmt.Println(err, res.Status)

	dclient := dataCoordinatorV1.NewDatasetCoordinatorClient(conn)
	for _, data := range originDataset {
		res, err := dclient.Insert(context.Background(), data)
		fmt.Println(res.Status, res.Error, err)
	}
	// first search
	searchVec, _ := embeddings.TextEmbedding(query)
	fmt.Println("old data search")
	resp, err := dclient.VectorSearch(context.Background(), &dataCoordinatorV1.SearchReq{
		BucketName: "tbucket",
		Vector:     searchVec,
		TopK:       3,
		EfSearch:   16,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			meta := make(map[string]interface{})
			msgpack.Unmarshal(dd.GetMetadata(), &meta)
			fmt.Println(dd.Id, dd.Score, meta)
		}
	} else {
		log.Fatal(res.Error.ErrorMessage)
	}
	//add after data
	for _, data := range afterDataset {
		res, err := dclient.Insert(context.Background(), data)
		fmt.Println(res.Status, res.Error, err)
	}
	// second search
	fmt.Println("old+new data search")
	resp, err = dclient.VectorSearch(context.Background(), &dataCoordinatorV1.SearchReq{
		BucketName: "tbucket",
		Vector:     searchVec,
		TopK:       5,
		EfSearch:   16,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			meta := make(map[string]interface{})
			msgpack.Unmarshal(dd.GetMetadata(), &meta)
			fmt.Println(dd.Id, dd.Score, meta)
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}

VectorSearch

resp, err := dclient.VectorSearch(context.Background(), &dataCoordinatorV1.SearchReq{
		BucketName: "tbucket",
		Vector:     searchVec,
		TopK:       5,
		EfSearch:   16,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			meta := make(map[string]interface{})
			msgpack.Unmarshal(dd.GetMetadata(), &meta)
			fmt.Println(dd.Id, dd.Score, meta)
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}

--ouput
 65 map[_id:d318e9ce-aef1-421b-81f3-c94da21d17d5 description:When I'm hungry, I usually have a sandwich. gender:boy name:John type:12]
 56.3 map[_id:64d17a7f-c2c5-418f-95f5-088cd869d5d9 description:I tend to eat porridge whenever I feel hungry. gender:girl name:anna type:12632]
 50.6 map[_id:0043efbb-aa8e-47eb-89a4-1154b67b3e90 description:Sometimes, I grab a quick snack like nuts or fruit. gender:girl name:Ann type:142124]
 41.2 map[_id:eeb04548-e04c-4d6f-b22b-af088e8d75b3 description:On most days, I prefer having pasta for dinner. gender:girl name:swan type:122]
 33.3 map[_id:d788135e-0054-42d3-a851-85d69945184b description:For a typical meal, I often enjoy a bowl of salad. gender:boy name:Mark type:6302]

FilterSearch

fmt.Println("pure search")
	resp, err = dclient.FilterSearch(context.Background(), &dataCoordinatorV1.SearchReq{
		BucketName: "tbucket",
		Filter: map[string]string{
			"type": "12",
		},
		TopK:     5,
		EfSearch: 16,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			meta := make(map[string]interface{})
			msgpack.Unmarshal(dd.GetMetadata(), &meta)
			fmt.Println(dd.Id, dd.Score, meta)
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}
--output
 100 map[_id:d318e9ce-aef1-421b-81f3-c94da21d17d5 description:When I'm hungry, I usually have a sandwich. gender:boy name:John type:12]

HybridSearch

fmt.Println("hybrid search")
	resp, err = dclient.HybridSearch(context.Background(), &dataCoordinatorV1.SearchReq{
		BucketName: "tbucket",
		Vector:     searchVec,
		Filter: map[string]string{
			"gender": "girl",
		},
		TopK:     5,
		EfSearch: 16,
	})
	if err != nil {
		log.Fatal(err)
	}
	if resp.Status {
		for _, dd := range resp.GetCandidates() {
			meta := make(map[string]interface{})
			msgpack.Unmarshal(dd.GetMetadata(), &meta)
			fmt.Println(dd.Id, dd.Score, meta)
		}
	} else {
		log.Fatal(resp.Error.ErrorMessage)
	}
--output
 56.3 map[_id:64d17a7f-c2c5-418f-95f5-088cd869d5d9 description:I tend to eat porridge whenever I feel hungry. gender:girl name:anna type:12632]
 50.6 map[_id:0043efbb-aa8e-47eb-89a4-1154b67b3e90 description:Sometimes, I grab a quick snack like nuts or fruit. gender:girl name:Ann type:142124]
 41.2 map[_id:eeb04548-e04c-4d6f-b22b-af088e8d75b3 description:On most days, I prefer having pasta for dinner. gender:girl name:swan type:122]