Merge pull request #45 from bgokden/refactor

Refactor

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: circleci/golang:1.16.2-buster
+      - image: circleci/golang:1.16.5-buster
     steps:
       - checkout
       - run: mkdir -p ./artifacts
@@ -12,7 +12,7 @@ jobs:
       - run: $GOPATH/bin/goveralls -coverprofile=coverage.out -service=circle-ci -repotoken $COVERALLS_TOKEN
   publish-and-release:
     docker:
-      - image: circleci/golang:1.16.2-buster
+      - image: circleci/golang:1.16.5-buster
     steps:
       - checkout
       - add_ssh_keys

Dockerfile

@@ -1,7 +1,7 @@
 # build stage
 #FROM golang:1.15.6-alpine AS build-env
 #FROM golang:1.16.0-alpine3.12 AS build-env
-FROM golang:1.16.0-buster AS build-env
+FROM golang:1.16.5-buster AS build-env
 RUN apt-get update && apt-get install -y git bash curl build-essential
 
 ENV USER_ID 0

cmd/root.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 
-	"github.com/magneticio/go-common/logging"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -14,6 +14,7 @@ import (
 )
 
 var cfgFile string
+var Verbose bool
 
 // Version should be in format vd.d.d where d is a decimal number
 const Version string = semver.Version
@@ -43,16 +44,14 @@ func Execute() {
 
 func init() {
 
-	logging.Init(os.Stdout, os.Stderr)
-
 	cobra.OnInitialize(initConfig)
 
 	// Here you will define your flags and configuration settings.
 	// Cobra supports persistent flags, which, if defined here,
 	// will be global for your application.
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.veri/config.yaml")
-	rootCmd.PersistentFlags().BoolVarP(&logging.Verbose, "verbose", "v", false, "Verbose output")
+	rootCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "Verbose output")
 
 	viper.BindEnv("config", "VERICONFIG")
 
@@ -64,7 +63,7 @@ func initConfig() {
 	if cfgFile == "" {
 		cfgFile = viper.GetString("config")
 	}
-	logging.Info("Using Config file path: %v\n", cfgFile)
+	log.Printf("Using Config file path: %v\n", cfgFile)
 
 	if cfgFile != "" {
 		// Use config file from the flag.
@@ -73,7 +72,7 @@ func initConfig() {
 		// Find home directory.
 		home, homeDirError := homedir.Dir()
 		if homeDirError != nil {
-			logging.Error("Can not find home Directory: %v\n", homeDirError)
+			log.Printf("Can not find home Directory: %v\n", homeDirError)
 			os.Exit(1)
 		}
 		// Search config in home directory with name ".veri" (without extension).
@@ -83,8 +82,8 @@ func initConfig() {
 	}
 	// If a config file is found, read it in.
 	if err := viper.ReadInConfig(); err == nil {
-		logging.Info("Using config file: %v\n", viper.ConfigFileUsed())
+		log.Printf("Using config file: %v\n", viper.ConfigFileUsed())
 	} else {
-		logging.Error("Config can not be read due to error: %v\n", err)
+		log.Printf("Config can not be read due to error: %v\n", err)
 	}
 }

data/data.go

@@ -26,7 +26,7 @@ type DataSource interface {
 type Annoyer struct {
 	sync.RWMutex
 	DataIndex  *[]*pb.Datum
-	AnnoyIndex annoyindex.AnnoyIndex
+	AnnoyIndex annoyindex.AnnoyIndexEuclidean
 }
 
 // Data represents a dataset with similar struture
@@ -156,7 +156,7 @@ func (dt *Data) Process(force bool) error {
 		}
 		histUnit := 1 / nFloat
 		newDataIndex := make([]*pb.Datum, max(1000, int(dt.N)))
-		var newAnnoyIndex annoyindex.AnnoyIndex
+		var newAnnoyIndex annoyindex.AnnoyIndexEuclidean
 		err := dt.DB.View(func(txn *badger.Txn) error {
 			opts := badger.DefaultIteratorOptions
 			opts.PrefetchValues = false
@@ -222,6 +222,9 @@ func (dt *Data) Process(force bool) error {
 			newAnnoyIndex.Build(10)
 			// log.Printf("Updating index. len: %v\n", len(newDataIndex))
 			dt.Annoyer.Lock()
+			if dt.Annoyer.DataIndex != nil {
+				annoyindex.DeleteAnnoyIndexEuclidean(dt.Annoyer.AnnoyIndex)
+			}
 			dt.Annoyer.AnnoyIndex = newAnnoyIndex
 			dt.Annoyer.DataIndex = &newDataIndex
 			dt.Annoyer.Unlock()

data/search.go

@@ -150,7 +150,7 @@ func (c *Collector) Send(buf *z.Buffer) error {
 func (c *Collector) PassesFilters(datum *pb.Datum) bool {
 	if len(c.GroupFilters) > 0 {
 		jsonLabel := string(datum.Key.GroupLabel)
-		for _, filter := range c.Filters {
+		for _, filter := range c.GroupFilters {
 			value := gjson.Get(jsonLabel, filter)
 			if !value.Exists() {
 				return false
@@ -198,7 +198,7 @@ func (c *Collector) ToList(key []byte, itr *badger.Iterator) (*bpb.KVList, error
 		if len(c.GroupFilters) > 0 {
 			filterFailed := false
 			jsonLabel := string(datumKey.GroupLabel)
-			for _, filter := range c.Filters {
+			for _, filter := range c.GroupFilters {
 				value := gjson.Get(jsonLabel, filter)
 				if !value.Exists() {
 					filterFailed = true
@@ -247,6 +247,8 @@ var vectorComparisonFuncs = map[string]func(arr1 []float32, arr2 []float32) floa
 	"VectorMultiplication":  VectorMultiplication,
 	"CosineSimilarity":      CosineSimilarity,
 	"QuickVectorDistance":   QuickVectorDistance,
+	"AngularDistance":       AngularDistance,
+	"AnnoyAngularDistance":  AngularDistance,
 }
 
 func GetVectorComparisonFunction(name string) func(arr1 []float32, arr2 []float32) float64 {

data/util.go

@@ -52,6 +52,11 @@ func VectorMultiplication(arr1 []float32, arr2 []float32) float64 {
 	return float64(ret)
 }
 
+// AngularDistance sim(u.v) = (1 - arccos(cosine_similarity(u, v)) / pi)
+func AngularDistance(a []float32, b []float32) float64 {
+	return 1.0 - (math.Acos(CosineSimilarity(a, b)) / math.Pi)
+}
+
 // CosineSimilarity for vector similarity
 func CosineSimilarity(a []float32, b []float32) float64 {
 	count := 0

go.mod

@@ -1,19 +1,18 @@
 module github.com/bgokden/veri
 
-go 1.15
+go 1.16
 
 require (
 	github.com/bgokden/go-cache v2.1.1+incompatible
 	github.com/chewxy/math32 v1.0.8
-	github.com/dgraph-io/badger/v3 v3.2011.1
-	github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d
+	github.com/dgraph-io/badger/v3 v3.2103.0
+	github.com/dgraph-io/ristretto v0.0.4-0.20210309073149-3836124cdc5a
 	github.com/goburrow/cache v0.1.3
 	github.com/golang/protobuf v1.4.1
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.2
 	github.com/jinzhu/copier v0.0.0-20201025035756-632e723a6687
 	github.com/kr/pretty v0.2.0 // indirect
-	github.com/magneticio/go-common v0.0.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.1.3