Merge pull request #215 from weaviate/add-support-for-voyageai-module

Add support for text2vec-voyageai module

Author: antas-marcin

.cicd/test.sh

@@ -146,6 +146,7 @@ function check_creates_template() {
   check_modules "--set modules.text2vec-aws.enabled=true --set modules.generative-aws.enabled=true" "value: generative-aws,text2vec-aws"
   check_modules "--set modules.generative-anyscale.enabled=true" "value: generative-anyscale"
   check_modules "--set modules.generative-anyscale.enabled=true --set modules.generative-anyscale.apiKey=apiKey" "value: generative-anyscale"
+  check_modules "--set modules.text2vec-voyageai.enabled=true" "value: text2vec-voyageai"
 
   check_modules "--set modules.text2vec-openai.enabled=true --set modules.text2vec-openai.azureApiKey=azureApiKey" "value: text2vec-openai"
   check_modules "--set modules.qna-openai.enabled=true --set modules.qna-openai.azureApiKey=azureApiKey" "value: qna-openai"
@@ -168,6 +169,7 @@ function check_creates_template() {
   check_string_existence "--set modules.text2vec-aws.enabled=true --set modules.text2vec-aws.envSecrets.AWS_ACCESS_KEY_ID=key --set modules.text2vec-aws.envSecrets.AWS_SECRET_ACCESS_KEY=secret" "name: AWS_ACCESS_KEY_ID"
   check_string_existence "--set modules.text2vec-aws.enabled=true --set modules.text2vec-aws.envSecrets.AWS_ACCESS_KEY_ID=key --set modules.text2vec-aws.envSecrets.AWS_SECRET_ACCESS_KEY=secret" "name: AWS_SECRET_ACCESS_KEY"
   check_string_existence "--set modules.generative-anyscale.enabled=true --set modules.generative-anyscale.apiKey=apiKey" "name: ANYSCALE_APIKEY"
+  check_string_existence "--set modules.text2vec-voyageai.enabled=true --set modules.text2vec-voyageai.apiKey=voyageaiApiKey" "name: VOYAGEAI_APIKEY"
 
   _settingPassageQueryOn="--set modules.text2vec-contextionary.enabled=false --set modules.text2vec-transformers.passageQueryServices.passage.enabled=true --set modules.text2vec-transformers.passageQueryServices.query.enabled=true"
   check_setting_has_value "$_settingPassageQueryOn" "name: TRANSFORMERS_PASSAGE_INFERENCE_API" "value: http://transformers-inference-passage.default.svc.cluster.local.:8080"

weaviate/templates/_helpers.tpl

@@ -71,6 +71,9 @@
   {{- if (index .Values "modules" "text2vec-aws" "enabled") -}}
     {{ $modules = append $modules "text2vec-aws" }}
   {{- end -}}
+  {{- if (index .Values "modules" "text2vec-voyageai" "enabled") -}}
+    {{ $modules = append $modules "text2vec-voyageai" }}
+  {{- end -}}
   {{- if (index .Values "modules" "ref2vec-centroid" "enabled") -}}
     {{ $modules = append $modules "ref2vec-centroid" }}
   {{- end -}}

weaviate/templates/apiKeyVoyageAISecret.yaml

@@ -0,0 +1,12 @@
+{{ if and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey") }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: weaviate-voyageai
+  labels:
+    app.kubernetes.io/name: weaviate
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+type: Opaque
+data:
+  apiKey: {{ index .Values "modules" "text2vec-voyageai" "apiKey" | b64enc }}
+{{ end }}

weaviate/templates/weaviateStatefulset.yaml

@@ -225,6 +225,13 @@ spec:
                 name: weaviate-jinaai
                 key: apiKey
           {{- end  }}
+          {{- if and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey") }}
+          - name: VOYAGEAI_APIKEY
+            valueFrom:
+              secretKeyRef:
+                name: weaviate-voyageai
+                key: apiKey
+          {{- end  }}
           {{- if and (index .Values "modules" "generative-anyscale" "enabled") (index .Values "modules" "generative-anyscale" "apiKey") }}
           - name: ANYSCALE_APIKEY
             valueFrom:

weaviate/values.yaml

@@ -848,6 +848,20 @@ modules:
     #   AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id
     #   AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key
 
+  # The text2vec-voyageai module uses VoaygeAI Embeddings API
+  # to dynamically compute vector embeddings based on the
+  # sentence's context.
+  # More information about VoaygeAI Embeddings API can be found here:
+  # https://www.voyageai.com/
+  text2vec-voyageai:
+
+    # enable if you want to use VoaygeAI module
+    enabled: false
+
+    # Set your VoaygeAI API Key to be passed to Weaviate pod as
+    # an environment variable
+    apiKey: ''
+
   # The multi2vec-clip modules uses CLIP transformers to vectorize both images
   # and text in the same vector space. It is typically slow(er) on CPUs and should
   # run with CUDA-enabled GPUs for optimal performance.