From 99ef026ff17497849f34303e9758c3c470436b9c Mon Sep 17 00:00:00 2001 From: Brian Vander Schaaf Date: Fri, 8 Nov 2024 13:19:33 -0500 Subject: [PATCH 1/7] feat: enable Azure blob storage --- charts/langsmith/templates/_helpers.tpl | 34 +++++++++++++++++++++++++ charts/langsmith/templates/secrets.yaml | 3 +++ charts/langsmith/values.yaml | 12 ++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/charts/langsmith/templates/_helpers.tpl b/charts/langsmith/templates/_helpers.tpl index 50ba1db..c963aa1 100644 --- a/charts/langsmith/templates/_helpers.tpl +++ b/charts/langsmith/templates/_helpers.tpl @@ -272,6 +272,13 @@ Template containing common environment variables that are used by several servic {{- if .Values.config.blobStorage.enabled }} - name: FF_S3_STORAGE_ENABLED value: {{ .Values.config.blobStorage.enabled | quote }} +- name: FF_BLOB_STORAGE_ENABLED + value: {{ .Values.config.blobStorage.enabled | quote }} +- name: BLOB_STORAGE_ENGINE + value: {{ .Values.config.blobStorage.engine | quote }} +- name: MIN_BLOB_STORAGE_SIZE_KB + value: {{ .Values.config.blobStorage.minBlobStorageSizeKb | quote }} +{{- if eq .Values.config.blobStorage.engine "S3" }} - name: S3_BUCKET_NAME value: {{ .Values.config.blobStorage.bucketName | quote }} - name: S3_RUN_MANIFEST_BUCKET_NAME @@ -291,6 +298,33 @@ Template containing common environment variables that are used by several servic key: blob_storage_access_key_secret optional: true {{- end }} +{{- if eq .Values.config.blobStorage.engine "Azure" }} +- name: AZURE_STORAGE_ACCOUNT_NAME + value: {{ .Values.config.blobStorage.azureStorageAccountName | quote }} +- name: AZURE_STORAGE_CONTAINER_NAME + value: {{ .Values.config.blobStorage.azureStorageContainerName | quote }} +- name: AZURE_STORAGE_SERVICE_URL_OVERRIDE + value: {{ .Values.config.blobStorage.azureStorageServiceUrlOverride | quote }} +- name: AZURE_STORAGE_ACCOUNT_KEY + valueFrom: + secretKeyRef: + name: {{ include "langsmith.secretsName" . }} + key: azure_storage_account_key + optional: true +- name: AZURE_STORAGE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{ include "langsmith.secretsName" . }} + key: azure_storage_connection_string + optional: true +- name: AZURE_STORAGE_SAS_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "langsmith.secretsName" . }} + key: azure_storage_sas_token + optional: true +{{- end }} +{{- end }} - name: FF_CH_SEARCH_ENABLED value: {{ .Values.config.blobStorage.chSearchEnabled | quote }} {{ include "langsmith.conditionalEnvVarsResolved" . }} diff --git a/charts/langsmith/templates/secrets.yaml b/charts/langsmith/templates/secrets.yaml index 74f3a9d..3b55735 100644 --- a/charts/langsmith/templates/secrets.yaml +++ b/charts/langsmith/templates/secrets.yaml @@ -17,4 +17,7 @@ data: initial_org_admin_password: {{ .Values.config.basicAuth.initialOrgAdminPassword | b64enc | quote }} blob_storage_access_key: {{ .Values.config.blobStorage.accessKey | b64enc | quote }} blob_storage_access_key_secret: {{ .Values.config.blobStorage.accessKeySecret | b64enc | quote }} + azure_storage_account_key: {{ .Values.config.blobStorage.azureStorageAccountKey | b64enc | quote }} + azure_storage_connection_string: {{ .Values.config.blobStorage.azureStorageConnectionString | b64enc | quote }} + azure_storage_sas_token: {{ .Values.config.blobStorage.azureStorageSASToken | b64enc | quote }} {{- end }} diff --git a/charts/langsmith/values.yaml b/charts/langsmith/values.yaml index c99b8cb..04d35e8 100644 --- a/charts/langsmith/values.yaml +++ b/charts/langsmith/values.yaml @@ -107,18 +107,28 @@ config: shortlived: "1209600" # -- Blob storage configuration # Optional. Used to store inputs, outputs, and errors in Blob Storage. - # We currently support S3, GCS, and Minio as Blob Storage providers. + # We currently support S3, GCS, Minio, and Azure as Blob Storage providers. blobStorage: enabled: false + engine: "S3" # If you are using langsmith-managed-clickhouse, you may not want inputs to be stored in clickhouse for search. # Set this as false to ensure that inputs/outputs/errors are not stored in clickhouse. chSearchEnabled: true + # Set this to change the threshold for payloads to be stored in blob storage + minBlobStorageSizeKb: "20" # If you are using workload identity, you may not need to store the S3 credentials in the secrets. # Instead, you will need to add the workload identity annotation to the backend and queue service accounts. accessKey: "" accessKeySecret: "" bucketName: "" apiURL: "https://s3.us-west-2.amazonaws.com" + # The following blob storage configuration values are for Azure and require blobStorage.engine = "Azure" + azureStorageAccountName: "" + azureStorageAccountKey: "" + azureStorageContainerName: "" + azureStorageConnectionString: "" + azureStorageSASToken: "" + azureStorageServiceUrlOverride: "" aceBackend: name: "ace-backend" From 7cd07a6639e3dce29b5900fc9e7876d8edc7b920 Mon Sep 17 00:00:00 2001 From: Brian Vander Schaaf Date: Fri, 8 Nov 2024 13:20:02 -0500 Subject: [PATCH 2/7] docs --- charts/langsmith/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/langsmith/README.md b/charts/langsmith/README.md index 778fd01..9f700f2 100644 --- a/charts/langsmith/README.md +++ b/charts/langsmith/README.md @@ -190,7 +190,7 @@ For information on how to use this chart, up-to-date release notes, and other gu | config.basicAuth.initialOrgAdminEmail | string | `""` | | | config.basicAuth.initialOrgAdminPassword | string | `""` | | | config.basicAuth.jwtSecret | string | `""` | | -| config.blobStorage | object | `{"accessKey":"","accessKeySecret":"","apiURL":"https://s3.us-west-2.amazonaws.com","bucketName":"","chSearchEnabled":true,"enabled":false}` | Blob storage configuration Optional. Used to store inputs, outputs, and errors in Blob Storage. We currently support S3, GCS, and Minio as Blob Storage providers. | +| config.blobStorage | object | `{"accessKey":"","accessKeySecret":"","apiURL":"https://s3.us-west-2.amazonaws.com","azureStorageAccountKey":"","azureStorageAccountName":"","azureStorageConnectionString":"","azureStorageContainerName":"","azureStorageSASToken":"","azureStorageServiceUrlOverride":"","bucketName":"","chSearchEnabled":true,"enabled":false,"engine":"S3","minBlobStorageSizeKb":"20"}` | Blob storage configuration Optional. Used to store inputs, outputs, and errors in Blob Storage. We currently support S3, GCS, Minio, and Azure as Blob Storage providers. | | config.existingSecretName | string | `""` | | | config.hostname | string | `""` | Base URL of the LangSmith installation. Used for redirects. | | config.langsmithLicenseKey | string | `""` | | @@ -775,6 +775,6 @@ For information on how to use this chart, up-to-date release notes, and other gu | Ankush | | | ---------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.11.3](https://github.com/norwoodj/helm-docs/releases/v1.11.3) +Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1) ## Docs Generated by [helm-docs](https://github.com/norwoodj/helm-docs) `helm-docs -t ./charts/langsmith/README.md.gotmpl` From 766315f5443f229be921075b29f56aa276d03769 Mon Sep 17 00:00:00 2001 From: Brian Vander Schaaf Date: Tue, 12 Nov 2024 15:48:28 -0500 Subject: [PATCH 3/7] bump version --- charts/langsmith/Chart.yaml | 4 ++-- charts/langsmith/README.md | 12 ++++++------ charts/langsmith/values.yaml | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/charts/langsmith/Chart.yaml b/charts/langsmith/Chart.yaml index 2447c09..c7680a6 100644 --- a/charts/langsmith/Chart.yaml +++ b/charts/langsmith/Chart.yaml @@ -5,5 +5,5 @@ maintainers: email: ankush@langchain.dev description: Helm chart to deploy the langsmith application and all services it depends on. type: application -version: 0.8.8 -appVersion: "0.8.27" +version: 0.8.9 +appVersion: "0.8.29" diff --git a/charts/langsmith/README.md b/charts/langsmith/README.md index b8b4b3b..eb743c2 100644 --- a/charts/langsmith/README.md +++ b/charts/langsmith/README.md @@ -1,6 +1,6 @@ # langsmith -![Version: 0.8.8](https://img.shields.io/badge/Version-0.8.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.8.27](https://img.shields.io/badge/AppVersion-0.8.27-informational?style=flat-square) +![Version: 0.8.9](https://img.shields.io/badge/Version-0.8.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.8.29](https://img.shields.io/badge/AppVersion-0.8.29-informational?style=flat-square) Helm chart to deploy the langsmith application and all services it depends on. @@ -148,23 +148,23 @@ For information on how to use this chart, up-to-date release notes, and other gu | fullnameOverride | string | `""` | String to fully override `"langsmith.fullname"` | | images.aceBackendImage.pullPolicy | string | `"IfNotPresent"` | | | images.aceBackendImage.repository | string | `"docker.io/langchain/langsmith-ace-backend"` | | -| images.aceBackendImage.tag | string | `"0.8.27"` | | +| images.aceBackendImage.tag | string | `"0.8.29"` | | | images.backendImage.pullPolicy | string | `"IfNotPresent"` | | | images.backendImage.repository | string | `"docker.io/langchain/langsmith-backend"` | | -| images.backendImage.tag | string | `"0.8.27"` | | +| images.backendImage.tag | string | `"0.8.29"` | | | images.clickhouseImage.pullPolicy | string | `"Always"` | | | images.clickhouseImage.repository | string | `"docker.io/clickhouse/clickhouse-server"` | | | images.clickhouseImage.tag | string | `"24.5"` | | | images.frontendImage.pullPolicy | string | `"IfNotPresent"` | | | images.frontendImage.repository | string | `"docker.io/langchain/langsmith-frontend"` | | -| images.frontendImage.tag | string | `"0.8.27"` | | +| images.frontendImage.tag | string | `"0.8.29"` | | | images.imagePullSecrets | list | `[]` | Secrets with credentials to pull images from a private registry. Specified as name: value. | | images.platformBackendImage.pullPolicy | string | `"IfNotPresent"` | | | images.platformBackendImage.repository | string | `"docker.io/langchain/langsmith-go-backend"` | | -| images.platformBackendImage.tag | string | `"0.8.27"` | | +| images.platformBackendImage.tag | string | `"0.8.29"` | | | images.playgroundImage.pullPolicy | string | `"IfNotPresent"` | | | images.playgroundImage.repository | string | `"docker.io/langchain/langsmith-playground"` | | -| images.playgroundImage.tag | string | `"0.8.27"` | | +| images.playgroundImage.tag | string | `"0.8.29"` | | | images.postgresImage.pullPolicy | string | `"IfNotPresent"` | | | images.postgresImage.repository | string | `"docker.io/postgres"` | | | images.postgresImage.tag | string | `"14.7"` | | diff --git a/charts/langsmith/values.yaml b/charts/langsmith/values.yaml index facf46a..ec3b593 100644 --- a/charts/langsmith/values.yaml +++ b/charts/langsmith/values.yaml @@ -17,23 +17,23 @@ images: aceBackendImage: repository: "docker.io/langchain/langsmith-ace-backend" pullPolicy: IfNotPresent - tag: "0.8.27" + tag: "0.8.29" backendImage: repository: "docker.io/langchain/langsmith-backend" pullPolicy: IfNotPresent - tag: "0.8.27" + tag: "0.8.29" frontendImage: repository: "docker.io/langchain/langsmith-frontend" pullPolicy: IfNotPresent - tag: "0.8.27" + tag: "0.8.29" platformBackendImage: repository: "docker.io/langchain/langsmith-go-backend" pullPolicy: IfNotPresent - tag: "0.8.27" + tag: "0.8.29" playgroundImage: repository: "docker.io/langchain/langsmith-playground" pullPolicy: IfNotPresent - tag: "0.8.27" + tag: "0.8.29" postgresImage: repository: "docker.io/postgres" pullPolicy: IfNotPresent From a9fd29b8f1f6d85f899514d0d97a69a096570818 Mon Sep 17 00:00:00 2001 From: Brian Vander Schaaf Date: Tue, 12 Nov 2024 16:10:39 -0500 Subject: [PATCH 4/7] add required param --- charts/langsmith/templates/_helpers.tpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/langsmith/templates/_helpers.tpl b/charts/langsmith/templates/_helpers.tpl index c963aa1..5dbf8ea 100644 --- a/charts/langsmith/templates/_helpers.tpl +++ b/charts/langsmith/templates/_helpers.tpl @@ -299,6 +299,9 @@ Template containing common environment variables that are used by several servic optional: true {{- end }} {{- if eq .Values.config.blobStorage.engine "Azure" }} +{{/* TODO: remove S3_API_URL once not required by config checks */}} +- name: S3_API_URL + value: {{ .Values.config.blobStorage.apiURL | quote }} - name: AZURE_STORAGE_ACCOUNT_NAME value: {{ .Values.config.blobStorage.azureStorageAccountName | quote }} - name: AZURE_STORAGE_CONTAINER_NAME From e57a392b3f72775cb74469d75b018495a82f1968 Mon Sep 17 00:00:00 2001 From: Brian Vander Schaaf Date: Tue, 12 Nov 2024 16:36:16 -0500 Subject: [PATCH 5/7] remove --- charts/langsmith/templates/_helpers.tpl | 3 --- 1 file changed, 3 deletions(-) diff --git a/charts/langsmith/templates/_helpers.tpl b/charts/langsmith/templates/_helpers.tpl index 5dbf8ea..c963aa1 100644 --- a/charts/langsmith/templates/_helpers.tpl +++ b/charts/langsmith/templates/_helpers.tpl @@ -299,9 +299,6 @@ Template containing common environment variables that are used by several servic optional: true {{- end }} {{- if eq .Values.config.blobStorage.engine "Azure" }} -{{/* TODO: remove S3_API_URL once not required by config checks */}} -- name: S3_API_URL - value: {{ .Values.config.blobStorage.apiURL | quote }} - name: AZURE_STORAGE_ACCOUNT_NAME value: {{ .Values.config.blobStorage.azureStorageAccountName | quote }} - name: AZURE_STORAGE_CONTAINER_NAME From 79eb763c21a93d3f1a8b0a9e04dc3aebb87e0d0f Mon Sep 17 00:00:00 2001 From: Brian Vander Schaaf Date: Tue, 12 Nov 2024 16:55:43 -0500 Subject: [PATCH 6/7] remove sas token --- charts/langsmith/README.md | 2 +- charts/langsmith/templates/_helpers.tpl | 6 ------ charts/langsmith/templates/secrets.yaml | 1 - charts/langsmith/values.yaml | 1 - 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/charts/langsmith/README.md b/charts/langsmith/README.md index eb743c2..eed1d34 100644 --- a/charts/langsmith/README.md +++ b/charts/langsmith/README.md @@ -190,7 +190,7 @@ For information on how to use this chart, up-to-date release notes, and other gu | config.basicAuth.initialOrgAdminEmail | string | `""` | | | config.basicAuth.initialOrgAdminPassword | string | `""` | | | config.basicAuth.jwtSecret | string | `""` | | -| config.blobStorage | object | `{"accessKey":"","accessKeySecret":"","apiURL":"https://s3.us-west-2.amazonaws.com","azureStorageAccountKey":"","azureStorageAccountName":"","azureStorageConnectionString":"","azureStorageContainerName":"","azureStorageSASToken":"","azureStorageServiceUrlOverride":"","bucketName":"","chSearchEnabled":true,"enabled":false,"engine":"S3","minBlobStorageSizeKb":"20"}` | Blob storage configuration Optional. Used to store inputs, outputs, and errors in Blob Storage. We currently support S3, GCS, Minio, and Azure as Blob Storage providers. | +| config.blobStorage | object | `{"accessKey":"","accessKeySecret":"","apiURL":"https://s3.us-west-2.amazonaws.com","azureStorageAccountKey":"","azureStorageAccountName":"","azureStorageConnectionString":"","azureStorageContainerName":"","azureStorageServiceUrlOverride":"","bucketName":"","chSearchEnabled":true,"enabled":false,"engine":"S3","minBlobStorageSizeKb":"20"}` | Blob storage configuration Optional. Used to store inputs, outputs, and errors in Blob Storage. We currently support S3, GCS, Minio, and Azure as Blob Storage providers. | | config.existingSecretName | string | `""` | | | config.hostname | string | `""` | Base URL of the LangSmith installation. Used for redirects. | | config.langsmithLicenseKey | string | `""` | | diff --git a/charts/langsmith/templates/_helpers.tpl b/charts/langsmith/templates/_helpers.tpl index c963aa1..3b4d6ed 100644 --- a/charts/langsmith/templates/_helpers.tpl +++ b/charts/langsmith/templates/_helpers.tpl @@ -317,12 +317,6 @@ Template containing common environment variables that are used by several servic name: {{ include "langsmith.secretsName" . }} key: azure_storage_connection_string optional: true -- name: AZURE_STORAGE_SAS_TOKEN - valueFrom: - secretKeyRef: - name: {{ include "langsmith.secretsName" . }} - key: azure_storage_sas_token - optional: true {{- end }} {{- end }} - name: FF_CH_SEARCH_ENABLED diff --git a/charts/langsmith/templates/secrets.yaml b/charts/langsmith/templates/secrets.yaml index 3b55735..b962949 100644 --- a/charts/langsmith/templates/secrets.yaml +++ b/charts/langsmith/templates/secrets.yaml @@ -19,5 +19,4 @@ data: blob_storage_access_key_secret: {{ .Values.config.blobStorage.accessKeySecret | b64enc | quote }} azure_storage_account_key: {{ .Values.config.blobStorage.azureStorageAccountKey | b64enc | quote }} azure_storage_connection_string: {{ .Values.config.blobStorage.azureStorageConnectionString | b64enc | quote }} - azure_storage_sas_token: {{ .Values.config.blobStorage.azureStorageSASToken | b64enc | quote }} {{- end }} diff --git a/charts/langsmith/values.yaml b/charts/langsmith/values.yaml index ec3b593..f08c3e2 100644 --- a/charts/langsmith/values.yaml +++ b/charts/langsmith/values.yaml @@ -127,7 +127,6 @@ config: azureStorageAccountKey: "" azureStorageContainerName: "" azureStorageConnectionString: "" - azureStorageSASToken: "" azureStorageServiceUrlOverride: "" aceBackend: From 137b7718bc2fbdbd35e98ded5a2814734fb35e5a Mon Sep 17 00:00:00 2001 From: Brian Vander Schaaf Date: Tue, 12 Nov 2024 17:45:04 -0500 Subject: [PATCH 7/7] add comments --- charts/langsmith/README.md | 4 ++++ charts/langsmith/values.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/charts/langsmith/README.md b/charts/langsmith/README.md index eed1d34..7aef794 100644 --- a/charts/langsmith/README.md +++ b/charts/langsmith/README.md @@ -191,6 +191,10 @@ For information on how to use this chart, up-to-date release notes, and other gu | config.basicAuth.initialOrgAdminPassword | string | `""` | | | config.basicAuth.jwtSecret | string | `""` | | | config.blobStorage | object | `{"accessKey":"","accessKeySecret":"","apiURL":"https://s3.us-west-2.amazonaws.com","azureStorageAccountKey":"","azureStorageAccountName":"","azureStorageConnectionString":"","azureStorageContainerName":"","azureStorageServiceUrlOverride":"","bucketName":"","chSearchEnabled":true,"enabled":false,"engine":"S3","minBlobStorageSizeKb":"20"}` | Blob storage configuration Optional. Used to store inputs, outputs, and errors in Blob Storage. We currently support S3, GCS, Minio, and Azure as Blob Storage providers. | +| config.blobStorage.azureStorageAccountName | string | `""` | Optional. Set this along with azureStorageAccountKey to use a storage account and access key. Higher precedence than azureStorageConnectionString. | +| config.blobStorage.azureStorageConnectionString | string | `""` | Optional. Use this to specify the full connection string including any authentication params. | +| config.blobStorage.azureStorageContainerName | string | `""` | Required if using Azure blob storage | +| config.blobStorage.azureStorageServiceUrlOverride | string | `""` | Optional. Use this to customize the service URL, which by default is 'https://.blob.core.windows.net/' | | config.existingSecretName | string | `""` | | | config.hostname | string | `""` | Base URL of the LangSmith installation. Used for redirects. | | config.langsmithLicenseKey | string | `""` | | diff --git a/charts/langsmith/values.yaml b/charts/langsmith/values.yaml index f08c3e2..0b32c50 100644 --- a/charts/langsmith/values.yaml +++ b/charts/langsmith/values.yaml @@ -123,10 +123,14 @@ config: bucketName: "" apiURL: "https://s3.us-west-2.amazonaws.com" # The following blob storage configuration values are for Azure and require blobStorage.engine = "Azure" + # -- Optional. Set this along with azureStorageAccountKey to use a storage account and access key. Higher precedence than azureStorageConnectionString. azureStorageAccountName: "" azureStorageAccountKey: "" + # -- Required if using Azure blob storage azureStorageContainerName: "" + # -- Optional. Use this to specify the full connection string including any authentication params. azureStorageConnectionString: "" + # -- Optional. Use this to customize the service URL, which by default is 'https://.blob.core.windows.net/' azureStorageServiceUrlOverride: "" aceBackend: