diff --git a/charts/graphscope-interactive/README.md b/charts/graphscope-interactive/README.md index 5a7fbdc5c6c7..b1f81daa5880 100644 --- a/charts/graphscope-interactive/README.md +++ b/charts/graphscope-interactive/README.md @@ -13,6 +13,9 @@ $ kubectl describe svc {your-release-name} -graphscope-interactive-frontend | gr #192.168.0.44:7687 # the first is the gremlin endpoint(currently not supported) # the second is the cypher endpoint +$ kubectl describe svc {your-release-name} -graphscope-interactive-engine | grep "Endpoints:" | awk -F' ' '{print $2}' +# the first is the admin port +# the second is the query port ``` Delete the deployment via @@ -105,3 +108,88 @@ hiactorWorkerNum: 1 # currently only support 1. hiactorTimeout: 240000 ``` + + +## TODO + +- TODO: Support cypher/gremlin queries. + +## Installtion + +```bash +helm install lei-test -f settings.yaml . --set odps.access.id="",odps.access.key="",odps.endpoint="" +export NODE_IP=$(ktl -n kubetask get pod lei-test-graphscope-interactive-primary-0 -o jsonpath="{.status.podIP}") +export ADMIN_PORT=$(ktl get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[0].ports[0].containerPort}') +export QUERY_PORT=$(ktl get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}') +export ADMIN_ENDPOINT=${NODE_IP}:${ADMIN_PORT} +export QUERY_ENDPOINT=${NODE_IP}:${QUERY_PORT} +echo "ADMIN_ENDPOINT: ${ADMIN_ENDPOINT}" +echo "QUERY_ENDPOINT: ${QUERY_ENDPOINT}" +``` + +```bash +export NODE_IP=$(118f -n kubetask get pod lei-test-graphscope-interactive-primary-0 -o jsonpath="{.status.podIP}") +export ADMIN_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[0].ports[0].containerPort}') +export QUERY_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}') +``` + +```bash +# to verify the helm char +helm install lei-test --dry-run +``` + + +## add resty.http to nginx images + +A customized nginx image + + +## nginx conf + + + # - name: admin-nginx + # image: {{ include "graphscope-interactive.nginx.image" . }} + # imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }} + # # command: ["sleep", "infinity"] + # ports: + # - name: admin-port + # containerPort: {{ .Values.frontend.service.adminPort }} + # {{- if .Values.resources.frontend }} + # resources: {{- toYaml .Values.resources.frontend | nindent 12 }} + # {{- end }} + # volumeMounts: + # - name: workspace + # mountPath: {{ .Values.workspace }} + # - name: config + # mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + # subPath: engine_config.yaml + # - name: admin-nginx-config + # mountPath: /etc/nginx/nginx.conf + # subPath: nginx.conf \ No newline at end of file diff --git a/charts/graphscope-interactive/templates/_helpers.tpl b/charts/graphscope-interactive/templates/_helpers.tpl index a77e42303686..dc3222c57ddd 100644 --- a/charts/graphscope-interactive/templates/_helpers.tpl +++ b/charts/graphscope-interactive/templates/_helpers.tpl @@ -27,8 +27,12 @@ If release name contains chart name it will be used as a full name. {{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "frontend" | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{- define "graphscope-interactive.engine.fullname" -}} -{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "engine" | trunc 63 | trimSuffix "-" -}} +{{- define "graphscope-interactive.primary.fullname" -}} +{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "primary" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "graphscope-interactive.secondary.fullname" -}} +{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "secondary" | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -77,11 +81,46 @@ Return the proper graphscope-interactive frontend image name {{- end -}} {{/* -Return the proper graphscope-interactive engine image name +Return the proper graphscope-interactive primary image name +*/}} +{{- define "graphscope-interactive.primary.image" -}} +{{- $tag := .Chart.AppVersion | toString -}} +{{- with .Values.primary.image -}} +{{- if .tag -}} +{{- $tag = .tag | toString -}} +{{- end -}} +{{- if .registry -}} +{{- printf "%s/%s:%s" .registry .repository $tag -}} +{{- else -}} +{{- printf "%s:%s" .repository $tag -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper graphscope-interactive primary image name +*/}} +{{- define "graphscope-interactive.nginx.image" -}} +{{- $tag := .Chart.AppVersion | toString -}} +{{- with .Values.nginx.image -}} +{{- if .tag -}} +{{- $tag = .tag | toString -}} +{{- end -}} +{{- if .registry -}} +{{- printf "%s/%s:%s" .registry .repository $tag -}} +{{- else -}} +{{- printf "%s:%s" .repository $tag -}} +{{- end -}} +{{- end -}} +{{- end -}} + + +{{/* +Return the proper graphscope-interactive secondary image name */}} -{{- define "graphscope-interactive.engine.image" -}} +{{- define "graphscope-interactive.secondary.image" -}} {{- $tag := .Chart.AppVersion | toString -}} -{{- with .Values.engine.image -}} +{{- with .Values.secondary.image -}} {{- if .tag -}} {{- $tag = .tag | toString -}} {{- end -}} @@ -159,6 +198,21 @@ Return the engineConfigPath with the graphscope configuration /etc/interactive/interactive_config.yaml {{- end -}} + +{{/* +Primary service name. +*/}} +{{- define "graphscope-interactive.primary.serviceName" -}} +{{- printf "%s.%s.svc.%s" (include "graphscope-interactive.primary.fullname" .) .Release.Namespace .Values.clusterDomain }} +{{- end -}} + +{{/* +From where a http client could fetch the schema uri +*/}} +{{- define "graphscope-interactive.graphSchemaUri" -}} +{{- printf "http://%s-0.%s:%d" (include "graphscope-interactive.fullname" .) (include "graphscope-interactive.primary.serviceName" .) .Values.primary.service.adminPort | trimSuffix "-" -}} +{{- end -}} + {{/* Return the realEngineConfigPath with the graphscope configuration, templated by frontend */}} diff --git a/charts/graphscope-interactive/templates/admin_nginx_conf.yaml b/charts/graphscope-interactive/templates/admin_nginx_conf.yaml new file mode 100644 index 000000000000..b48fe303a926 --- /dev/null +++ b/charts/graphscope-interactive/templates/admin_nginx_conf.yaml @@ -0,0 +1,159 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-admin-nginx-config + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: configmap + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + nginx.conf: | + events {} + http { + resolver local=on valid=5s; + server { + {{- $adminPort := .Values.primary.service.adminPort | int }} + listen {{ $adminPort }}; + client_body_buffer_size 10M; + client_max_body_size 10M; + location / { + {{- $primaryBaseName := include "graphscope-interactive.primary.fullname" . }} + {{- $secondaryBaseName := include "graphscope-interactive.secondary.fullname" . }} + {{- $replicaCount := .Values.secondary.replicaCount | int }} + {{- $primaryServiceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.primary.fullname" .) .Release.Namespace .Values.clusterDomain }} + {{- $secondaryServiceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }} + {{- $port := .Values.secondary.service.adminPort | int }} + proxy_pass {{ printf "http://%s-0.%s:%d" $primaryBaseName $primaryServiceName $port | quote }}; + content_by_lua_block { + function arrayToString(arr, separator) + separator = separator or ", " -- Default separator if not provided + return table.concat(arr, separator) + end + function send_request(http, full_uri, method, body_data, headers) + local httpc = http.new() + if method == "GET" or method == "DELETE" then + return httpc:request_uri(full_uri, { + method = method, + }) + elseif method == "POST" or method == "PUT" then + return httpc:request_uri(full_uri, { + method = method, + body = body_data, + headers = headers + }) + else + ngx.log(ngx.ERR, " not recognized method ", method) + end + end + local http = require "resty.http" + local res = {} + local status_codes = {} + local error_message = nil -- Initialize a variable to capture error messages + + local urls = { + {{ printf "http://%s-0.%s:%d" $primaryBaseName $primaryServiceName $port | quote }}; + {{- if eq $replicaCount 1 }} + {{ printf "http://%s-0.%s:%d" $secondaryBaseName $secondaryServiceName $port | quote }} + {{- else }} + {{- range $i := until (sub $replicaCount 1 | int ) }} + {{ printf "\"http://%s-%d.%s:%d\"," $secondaryBaseName $i $secondaryServiceName $port }} + {{- end }} + {{ printf "http://%s-%d.%s:%d" $secondaryBaseName (sub $replicaCount 1) $secondaryServiceName $port | quote }} + {{- end }} + } + + local original_headers = ngx.req.get_headers() + local request_uri=ngx.var.request_uri + local method = ngx.req.get_method() + + -- Create a table for modified headers + local backend_headers = {} + + -- Copy the relevant headers, if needed, or modify them + for key, value in pairs(original_headers) do + -- You can filter headers if needed (e.g., skip "host" or "authorization") + if key ~= "Host" and key ~= "User-Agent" and key ~= "Content-Length" then + backend_headers[key] = value + end + end + + + ngx.req.read_body() -- Read the request body + + -- resize status_codes to the number of replicas + for i = 1, #urls do + status_codes[i] = 0 + res[i] = "" + end + + local threads = {} + local body_data = ngx.req.get_body_data() + for index, backend in ipairs(urls) do + -- full_uri is backend + request_uri + local full_uri = backend .. request_uri + threads[index] = ngx.thread.spawn(function() + local response, err = send_request(http, full_uri, method, body_data, backend_headers) + local status_code = 0 + if response ~= nil then + status_code = response.status + res[index] = response.body + if response.status < 200 or response.status >= 300 then + if not error_message then -- Capture the error message from the first failed request + error_message = response.body or "Failed request without a body." + end + end + else + status_code = 500 + if err ~= nil then + ngx.log(ngx.ERR, "Failed to request: ", err) + if not error_message then -- Capture error when no response + error_message = "Error: " .. err + end + else + error_message = "Not found" + end + end + status_codes[index] = status_code + end) + end + + for _, thread in ipairs(threads) do + coroutine.resume(thread) + end + + for _, thread in ipairs(threads) do + ngx.thread.wait(thread) + end + + local success = true + local final_status_code = 200 + for i = 1, #urls do + if status_codes[i] < 200 or status_codes[i] >= 300 then + ngx.log(ngx.ERR, "Failed to request: ", urls[i], " with status code: ", status_codes[i], " and response: ", res[i], " index: ", i) + success = false + final_status_code = status_codes[i] + break + end + end + + ngx.header.content_type = 'application/json' + if success then + ngx.status = final_status_code + ngx.log(ngx.INFO, "Success: ", arrayToString(res, ", "), " with status code: ", final_status_code) + ngx.say(res[1]) + ngx.exit(final_status_code) + else + ngx.status = final_status_code + ngx.log(ngx.ERR, "Failed to request: ", error_message, " with status code: ", final_status_code) + ngx.say(error_message) + ngx.exit(final_status_code) + end + } + } + } + } \ No newline at end of file diff --git a/charts/graphscope-interactive/templates/configmap.yaml b/charts/graphscope-interactive/templates/configmap.yaml index 248ea2f14990..ebc471763774 100644 --- a/charts/graphscope-interactive/templates/configmap.yaml +++ b/charts/graphscope-interactive/templates/configmap.yaml @@ -13,13 +13,13 @@ metadata: {{- end }} data: interactive_config.yaml: |- - log_level: {{ .Values.engine.logLevel }} + log_level: {{ .Values.logLevel }} default_graph: {{ .Values.defaultGraph }} compute_engine: type: hiactor workers: - - ENGINE_SERVICE_HOST:10000 - thread_num_per_worker: {{ .Values.engine.threadNumPerWorker }} + - localhost:10000 + thread_num_per_worker: {{ .Values.primary.threadNumPerWorker }} compiler: planner: is_on: true @@ -29,7 +29,7 @@ data: - FilterMatchRule - NotMatchToAntiJoinRule endpoint: - default_listen_address: ENGINE_SERVICE_HOST + default_listen_address: localhost bolt_connector: disabled: false port: {{ .Values.frontend.service.cypherPort }} @@ -38,10 +38,6 @@ data: port: {{ .Values.frontend.service.gremlinPort }} query_timeout: {{ .Values.frontend.service.queryTimeout }} http_service: - default_listen_address: ENGINE_SERVICE_HOST - admin_port: {{ .Values.engine.service.adminPort }} - query_port: {{ .Values.engine.service.queryPort }} - setup.sh: |- - #!/bin/bash - sudo sed -e "s/ENGINE_SERVICE_HOST/${ENGINE_SERVICE_HOST}/g" ${ENGINE_CONFIG_PATH} > ${REAL_ENGINE_CONFIG_PATH} - echo "Finish set ENGINE_SERVICE_HOST to ${ENGINE_SERVICE_HOST}" \ No newline at end of file + default_listen_address: localhost + admin_port: {{ .Values.primary.service.adminPort }} + query_port: {{ .Values.primary.service.queryPort }} \ No newline at end of file diff --git a/charts/graphscope-interactive/templates/engine/statefulset.yaml b/charts/graphscope-interactive/templates/engine/statefulset.yaml deleted file mode 100644 index bf322194b0fa..000000000000 --- a/charts/graphscope-interactive/templates/engine/statefulset.yaml +++ /dev/null @@ -1,167 +0,0 @@ -{{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }} -{{- $engineFullName := include "graphscope-interactive.engine.fullname" . }} -{{- $releaseNamespace := .Release.Namespace }} -{{- $clusterDomain := .Values.clusterDomain }} - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "graphscope-interactive.engine.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} - app.kubernetes.io/component: engine - {{- if .Values.commonLabels }} - {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -spec: - replicas: {{ .Values.engine.replicaCount }} - selector: - matchLabels: {{ include "graphscope-interactive.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: engine - serviceName: {{ include "graphscope-interactive.engine.fullname" . }}-headless - updateStrategy: - type: {{ .Values.engine.updateStrategy }} - {{- if (eq "Recreate" .Values.engine.updateStrategy) }} - rollingUpdate: null - {{- end }} - template: - metadata: - annotations: - {{- if .Values.engine.podAnnotations }} - {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.podAnnotations "context" $) | nindent 8 }} - {{- end }} - labels: {{- include "graphscope-interactive.labels" . | nindent 8 }} - app.kubernetes.io/component: engine - {{- if .Values.commonLabels }} - {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "graphscope-interactive.serviceAccountName" . }} - {{- if .Values.engine.affinity }} - affinity: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.affinity "context" $) | nindent 8 }} - {{- end }} - initContainers: - {{- if .Values.engine.initContainers }} - {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.initContainers "context" $) | nindent 8 }} - {{- end }} - containers: - - name: engine - image: {{ include "graphscope-interactive.engine.image" . }} - imagePullPolicy: {{ .Values.engine.image.pullPolicy | quote }} - command: - - /bin/bash - - -c - - | - echo "Starting engine..." - # first check interactive_config.yaml exists - if [ ! -f ${ENGINE_CONFIG_PATH} ]; then - #error exit - echo "${ENGINE_CONFIG_PATH} not found, exiting..." - exit 1 - fi - # then check interactive_server binary exists and executable - if [ ! -x ${ENGINE_BINARY_PATH} ]; then - #error exit - echo "${ENGINE_BINARY_PATH} binary not found or not executable, exiting..." - exit 1 - fi - # always try to load the built-in graph: gs_interactive_default_graph - # for case CURRENT_GRAPH is not the default_graph, we assume the data is already loaded. - # TODO. - builtin_graph_schema_path="${INTERACTIVE_WORKSPACE}/data/${DEFAULT_GRAPH_NAME}/graph.yaml" - builtin_graph_data_path="${INTERACTIVE_WORKSPACE}/data/${DEFAULT_GRAPH_NAME}/indices/" - builtin_graph_import_path="${INTERACTIVE_WORKSPACE}/data/${DEFAULT_GRAPH_NAME}/bulk_load.yaml" - # if builtin_graph_data_path exists, skip - if [ ! -d ${builtin_graph_data_path} ]; then - mkdir -p ${INTERACTIVE_WORKSPACE}/data/${DEFAULT_GRAPH_NAME} - echo "Loading builtin graph: ${DEFAULT_GRAPH_NAME} with command: $builtin_graph_loader_cmd" - cp /opt/flex/share/gs_interactive_default_graph/graph.yaml ${builtin_graph_schema_path} - cp /opt/flex/share/gs_interactive_default_graph/bulk_load.yaml ${builtin_graph_import_path} - export FLEX_DATA_DIR=/opt/flex/share/gs_interactive_default_graph/ - - builtin_graph_loader_cmd="${BULK_LOADER_BINARY_PATH} -g ${builtin_graph_schema_path} -d ${builtin_graph_data_path} -l ${builtin_graph_import_path}" - echo "Loading builtin graph: ${DEFAULT_GRAPH_NAME} with command: $builtin_graph_loader_cmd" - eval $builtin_graph_loader_cmd - fi - - bash /etc/interactive/setup.sh - cmd="GLOG_v=10 ${ENGINE_BINARY_PATH} -c ${REAL_ENGINE_CONFIG_PATH}" - #cmd="${cmd} --enable-admin-service false -w ${INTERACTIVE_WORKSPACE}" - cmd="${cmd} -g ${builtin_graph_schema_path} --data-path ${builtin_graph_data_path}" - echo "Starting engine with command: $cmd" - eval $cmd - env: - - name: INTERACTIVE_WORKSPACE - value: {{ .Values.workspace | quote }} - - name: ENGINE_SERVICE_HOST - value: {{ $engineFullName }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} - - name: ENGINE_CONFIG_PATH - value: {{ include "graphscope-interactive.engineConfigPath" . }} - - name: REAL_ENGINE_CONFIG_PATH - value: {{ include "graphscope-interactive.realEngineConfigPath" . }} - - name: ENGINE_BINARY_PATH - value: {{ include "graphscope-interactive.engineBinaryPath" . }} - - name: ENGINE_SHARD_NUM - value: {{ .Values.engine.threadNumPerWorker | quote }} - - name: BULK_LOADER_BINARY_PATH - value: /opt/flex/bin/bulk_loader - - name: DEFAULT_GRAPH_NAME - value: {{ .Values.defaultGraph }} - ports: - - name: admin-port - containerPort: {{ .Values.engine.service.adminPort }} - - name: query-port - containerPort: {{ .Values.engine.service.queryPort }} - {{- if .Values.engine.resources }} - resources: {{- toYaml .Values.engine.resources | nindent 12 }} - {{- end }} - volumeMounts: - - name: workspace - mountPath: {{ .Values.workspace }} - - name: config - mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} - subPath: interactive_config.yaml - - name: config - mountPath: /etc/interactive/setup.sh - subPath: setup.sh - volumes: - - name: config - configMap: - name: {{ include "graphscope-interactive.configmapName" . }} - defaultMode: 0755 - {{- if and .Values.engine.persistence.enabled .Values.engine.persistence.existingClaim }} - - name: workspace - persistentVolumeClaim: - claimName: {{ tpl .Values.engine.persistence.existingClaim . }} - {{- else if not .Values.engine.persistence.enabled }} - - name: workspace - emptyDir: {} - {{- else if and .Values.engine.persistence.enabled (not .Values.engine.persistence.existingClaim) }} - volumeClaimTemplates: - - metadata: - name: workspace - {{- if .Values.persistence.annotations }} - annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} - {{- end }} - {{- if .Values.persistence.labels }} - labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }} - {{- end }} - spec: - accessModes: - {{- range .Values.persistence.accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.engine.persistence.size | quote }} - {{ include "graphscope-interactive.storageClass" . | nindent 8 }} - {{- if .Values.engine.persistence.selector }} - selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.persistence.selector "context" $) | nindent 10 }} - {{- end -}} - {{- end }} diff --git a/charts/graphscope-interactive/templates/engine/svc-headless.yaml b/charts/graphscope-interactive/templates/engine/svc-headless.yaml deleted file mode 100644 index a49094e692d2..000000000000 --- a/charts/graphscope-interactive/templates/engine/svc-headless.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "graphscope-interactive.engine.fullname" . }}-headless - namespace: {{ .Release.Namespace }} - labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} - app.kubernetes.io/component: engine - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - annotations: - {{- if .Values.commonAnnotations }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.engine.service.type }} - {{- if and (eq .Values.engine.service.type "ClusterIP") .Values.engine.service.clusterIP }} - clusterIP: {{ .Values.engine.service.clusterIP }} - {{- end }} - {{- if and .Values.engine.service.loadBalancerIP (eq .Values.engine.service.type "LoadBalancer") }} - loadBalancerIP: {{ .Values.engine.service.loadBalancerIP }} - externalTrafficPolicy: {{ .Values.engine.service.externalTrafficPolicy | quote }} - {{- end }} - {{- if and (eq .Values.engine.service.type "LoadBalancer") .Values.engine.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: {{- toYaml .Values.engine.service.loadBalancerSourceRanges | nindent 4 }} - {{- end }} - ports: - - name: admin-port - port: {{ .Values.engine.service.adminPort }} - protocol: TCP - targetPort: admin-port - - name: query-port - port: {{ .Values.engine.service.queryPort }} - protocol: TCP - targetPort: query-port - selector: {{- include "graphscope-interactive.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: engine diff --git a/charts/graphscope-interactive/templates/frontend/statefulset.yaml b/charts/graphscope-interactive/templates/frontend/statefulset.yaml index ef4ebd7f3c05..06862eb6b2de 100644 --- a/charts/graphscope-interactive/templates/frontend/statefulset.yaml +++ b/charts/graphscope-interactive/templates/frontend/statefulset.yaml @@ -1,5 +1,4 @@ {{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }} -{{- $engineFullName := include "graphscope-interactive.engine.fullname" . }} {{- $releaseNamespace := .Release.Namespace }} {{- $clusterDomain := .Values.clusterDomain }} @@ -59,48 +58,22 @@ spec: - -c - | echo "Starting frontend..." - - # first check interactive_config.yaml exists - if [ ! -f ${ENGINE_CONFIG_PATH} ]; then - #error exit - echo "${ENGINE_CONFIG_PATH} not found, exiting..." - exit 1 - fi - # check lib path has more than 1 file - if [ ! "$(ls -A ${COMPILER_CLASS_PATH})" ]; then - #error exit - echo "class path: ${COMPILER_CLASS_PATH} is empty, exiting..." - exit 1 - fi - # setup the template interactive_config.yaml - bash /etc/interactive/setup.sh - if [ ! -f ${REAL_ENGINE_CONFIG_PATH} ]; then - #error exit - echo "${REAL_ENGINE_CONFIG_PATH} not found, exiting..." - exit 1 - fi # get graph schema file - cmd="java -cp \"${COMPILER_CLASS_PATH}\" -Djna.library.path=${COMPILER_LIBRARY_PATH} " - cmd="${cmd} -Dgraph.schema=${GRAPH_SCHEMA_PATH} " - cmd="${cmd} com.alibaba.graphscope.GraphServer ${REAL_ENGINE_CONFIG_PATH}" - echo "Start compiler with command: ${cmd}" - eval ${cmd} + cmd="java -cp /opt/graphscope/interactive_engine/compiler/target/libs/:/opt/graphscope/interactive_engine/compiler/target/compiler-1.0-SNAPSHOT.jar" + cmd="$cmd -Djava.library.path=/opt/graphscope/interactive_engine/executor/ir/target/release/" + cmd="$cmd -Dgraph.schema=$GRAPH_SCHEMA_URI" + cmd="$cmd com.alibaba.graphscope.GraphServer" + cmd="$cmd $INTERACTIVE_CONFIG_PATH" + echo "Starting frontend with command: $cmd" + sleep infinity env: - - name: ENGINE_SERVICE_HOST - value: {{ $engineFullName }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} - name: TIMEOUT value: {{ .Values.hiactorTimeout | quote }} - - name: ENGINE_CONFIG_PATH + - name: INTERACTIVE_CONFIG_PATH value: {{ include "graphscope-interactive.engineConfigPath" . }} - - name: REAL_ENGINE_CONFIG_PATH - value: {{ include "graphscope-interactive.realEngineConfigPath" . }} - - name: COMPILER_CLASS_PATH - value: {{ include "graphscope-interactive.classPath" . }} - - name: COMPILER_LIBRARY_PATH - value: {{ include "graphscope-interactive.libraryPath" . }} - - name: GRAPH_SCHEMA_PATH - value: {{ include "graphscope-interactive.graphSchemaPath" . }} + - name: GRAPH_SCHEMA_URI + value: {{ include "graphscope-interactive.graphSchemaUri" . }} ports: - name: gremlin containerPort: {{ .Values.frontend.service.gremlinPort }} @@ -116,50 +89,58 @@ spec: periodSeconds: {{ .Values.frontend.readinessProbe.periodSeconds }} successThreshold: {{ .Values.frontend.readinessProbe.successThreshold }} {{- end }} - {{- if .Values.frontend.resources }} - resources: {{- toYaml .Values.frontend.resources | nindent 12 }} + {{- if .Values.resources.frontend }} + resources: {{- toYaml .Values.resources.frontend | nindent 12 }} {{- end }} volumeMounts: - - name: workspace - mountPath: {{ .Values.workspace }} - name: config mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} subPath: interactive_config.yaml - name: config mountPath: /etc/interactive/setup.sh subPath: setup.sh + - name: query-nginx + image: {{ include "graphscope-interactive.nginx.image" . }} + imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }} + # command: ["sleep", "infinity"] + ports: + - name: query-port + containerPort: {{ .Values.frontend.service.queryPort }} + {{- if .Values.resources.frontend }} + resources: {{- toYaml .Values.resources.frontend | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + - name: query-nginx-config + mountPath: /usr/local/openresty/nginx/conf/nginx.conf + subPath: nginx.conf + - name: admin-nginx + image: {{ include "graphscope-interactive.nginx.image" . }} + imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }} + # command: ["sleep", "infinity"] + ports: + - name: admin-port + containerPort: {{ .Values.frontend.service.adminPort }} + {{- if .Values.resources.frontend }} + resources: {{- toYaml .Values.resources.frontend | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + - name: admin-nginx-config + mountPath: /usr/local/openresty/nginx/conf/nginx.conf + subPath: nginx.conf volumes: + - name: admin-nginx-config + configMap: + name: {{ .Release.Name }}-admin-nginx-config + - name: query-nginx-config + configMap: + name: {{ .Release.Name }}-query-nginx-config - name: config configMap: name: {{ include "graphscope-interactive.configmapName" . }} defaultMode: 0755 - {{- if and .Values.engine.persistence.enabled .Values.engine.persistence.existingClaim }} - - name: workspace - persistentVolumeClaim: - claimName: {{ tpl .Values.engine.persistence.existingClaim . }} - {{- else if not .Values.engine.persistence.enabled }} - - name: workspace - emptyDir: {} - {{- else if and .Values.engine.persistence.enabled (not .Values.engine.persistence.existingClaim) }} - volumeClaimTemplates: - - metadata: - name: workspace - {{- if .Values.persistence.annotations }} - annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} - {{- end }} - {{- if .Values.persistence.labels }} - labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }} - {{- end }} - spec: - accessModes: - {{- range .Values.engine.persistence.accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.engine.persistence.size | quote }} - {{ include "graphscope-interactive.storageClass" . | nindent 8 }} - {{- if .Values.engine.persistence.selector }} - selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.persistence.selector "context" $) | nindent 10 }} - {{- end -}} - {{- end }} diff --git a/charts/graphscope-interactive/templates/frontend/svc.yaml b/charts/graphscope-interactive/templates/frontend/svc-headless.yaml similarity index 63% rename from charts/graphscope-interactive/templates/frontend/svc.yaml rename to charts/graphscope-interactive/templates/frontend/svc-headless.yaml index 829bf49a3f59..505a1b4328bb 100644 --- a/charts/graphscope-interactive/templates/frontend/svc.yaml +++ b/charts/graphscope-interactive/templates/frontend/svc-headless.yaml @@ -6,11 +6,11 @@ metadata: labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} app.kubernetes.io/component: frontend {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} {{- end }} annotations: {{- if .Values.commonAnnotations }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} spec: type: {{ .Values.frontend.service.type }} @@ -47,5 +47,27 @@ spec: nodePort: null {{- end }} {{- end }} + - name: admin + port: {{ .Values.frontend.service.adminPort }} + protocol: TCP + targetPort: admin + {{- if and (or (eq .Values.frontend.service.type "NodePort") (eq .Values.frontend.service.type "LoadBalancer")) (not (empty .Values.frontend.service.nodePorts.admin)) }} + {{- if (not (empty .Values.frontend.service.nodePorts.admin)) }} + nodePort: {{ .Values.frontend.service.nodePorts.admin }} + {{- else if eq .Values.frontend.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} + - name: query + port: {{ .Values.frontend.service.queryPort }} + protocol: TCP + targetPort: query + {{- if and (or (eq .Values.frontend.service.type "NodePort") (eq .Values.frontend.service.type "LoadBalancer")) (not (empty .Values.frontend.service.nodePorts.query)) }} + {{- if (not (empty .Values.frontend.service.nodePorts.query)) }} + nodePort: {{ .Values.frontend.service.nodePorts.query }} + {{- else if eq .Values.frontend.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} selector: {{ include "graphscope-interactive.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: frontend \ No newline at end of file diff --git a/charts/graphscope-interactive/templates/primary/statefulset.yaml b/charts/graphscope-interactive/templates/primary/statefulset.yaml new file mode 100644 index 000000000000..7d0db3a1c4f5 --- /dev/null +++ b/charts/graphscope-interactive/templates/primary/statefulset.yaml @@ -0,0 +1,153 @@ +{{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }} +{{- $primaryFullName := include "graphscope-interactive.primary.fullname" . }} +{{- $secondaryFullName := include "graphscope-interactive.secondary.fullname" . }} +{{- $releaseNamespace := .Release.Namespace }} +{{- $clusterDomain := .Values.clusterDomain }} + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "graphscope-interactive.primary.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: primary + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.primary.replicaCount }} + selector: + matchLabels: {{ include "graphscope-interactive.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: primary + serviceName: {{ include "graphscope-interactive.primary.fullname" . }} + updateStrategy: + type: {{ .Values.primary.updateStrategy }} + {{- if (eq "Recreate" .Values.primary.updateStrategy) }} + rollingUpdate: null + {{- end }} + template: + metadata: + annotations: + {{- if .Values.primary.podAnnotations }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "graphscope-interactive.labels" . | nindent 8 }} + app.kubernetes.io/component: primary + {{- if .Values.primary.podLabels }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.podLabels "context" $) | nindent 8 }} + {{- end }} + # alibabacloud.com/custom-cni-plugin-type: "nimitz" + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.primary.hostAliases }} + hostAliases: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.hostAliases "context" $) | nindent 8 }} + {{- end }} + hostNetwork: {{ .Values.primary.hostNetwork }} + hostIPC: {{ .Values.primary.hostIPC }} + {{- if .Values.primary.schedulerName }} + schedulerName: {{ .Values.primary.schedulerName | quote }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy | quote }} + {{- end }} + {{- if .Values.dnsConfig }} + dnsConfig: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.dnsConfig "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "graphscope-interactive.serviceAccountName" . }} + {{- if .Values.primary.affinity }} + affinity: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.affinity "context" $) | nindent 8 }} + {{- end }} + initContainers: + {{- if .Values.primary.initContainers }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.initContainers "context" $) | nindent 8 }} + {{- end }} + containers: + - name: master + image: {{ include "graphscope-interactive.primary.image" . }} + imagePullPolicy: {{ .Values.primary.image.pullPolicy | quote }} + # command: ["sleep", "infinity"] + command: + - /bin/bash + - -c + - | + POD_NAME=$MY_POD_NAME + if [ -z "$POD_NAME" ]; then + POD_NAME=$(hostname) + fi + echo "POD_NAME: $POD_NAME" + sudo chown -R graphscope:graphscope $INTERACTIVE_WORKSPACE + cmd="/opt/flex/bin/entrypoint.sh -w $INTERACTIVE_WORKSPACE" + echo "CMD: $cmd" + eval $cmd + # sleep infinity + env: + - name: INSTANCE_NAME + value: {{ .Release.Name | quote }} + - name: INTERACTIVE_WORKSPACE + value: {{ .Values.workspace | quote }} + - name: HANG_UNTIL_SUCCESS + value: {{ .Values.primary.hangUntilSuccess | quote }} + - name: ODPS_ACCESS_ID + value: {{ .Values.odps.access.id | quote }} + - name: ODPS_ACCESS_KEY + value: {{ .Values.odps.access.key | quote }} + - name: ODPS_ENDPOINT + value: {{ .Values.odps.endpoint | quote }} + ports: + - name: admin-port + containerPort: {{ .Values.primary.service.adminPort }} + {{- if .Values.resources.primary }} + resources: {{- toYaml .Values.resources.primary | nindent 12 }} + {{- end }} + volumeMounts: + - name: workspace + mountPath: {{ .Values.workspace }} + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + volumes: + - name: config + configMap: + name: {{ include "graphscope-interactive.configmapName" . }} + defaultMode: 0755 + {{- if and .Values.primary.persistence.enabled .Values.primary.persistence.existingClaim }} + - name: workspace + persistentVolumeClaim: + claimName: {{ tpl .Values.primary.persistence.existingClaim . }} + {{- else if not .Values.primary.persistence.enabled }} + - name: workspace + emptyDir: {} + {{- else if and .Values.primary.persistence.enabled (not .Values.primary.persistence.existingClaim) }} + volumeClaimTemplates: + - metadata: + name: workspace + {{- if .Values.persistence.annotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.persistence.labels }} + labels: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.primary.persistence.size | quote }} + {{ include "graphscope-interactive.storageClass" . | nindent 8 }} + {{- if .Values.primary.persistence.selector }} + selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.persistence.selector "context" $) | nindent 10 }} + {{- end -}} + {{- end }} diff --git a/charts/graphscope-interactive/templates/primary/svc.yaml b/charts/graphscope-interactive/templates/primary/svc.yaml new file mode 100644 index 000000000000..63783c1d2292 --- /dev/null +++ b/charts/graphscope-interactive/templates/primary/svc.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "graphscope-interactive.primary.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: primary + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.primary.service.type }} + {{- if and (eq .Values.primary.service.type "ClusterIP") .Values.primary.service.clusterIP }} + clusterIP: {{ .Values.primary.service.clusterIP }} + {{- end }} + {{- if and .Values.primary.service.loadBalancerIP (eq .Values.primary.service.type "LoadBalancer") }} + loadBalancerIP: {{ .Values.primary.service.loadBalancerIP }} + externalTrafficPolicy: {{ .Values.primary.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.primary.service.type "LoadBalancer") .Values.primary.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.primary.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + ports: + - name: admin-port + port: {{ .Values.primary.service.adminPort }} + protocol: TCP + targetPort: {{ .Values.primary.service.adminPort }} + - name: query-port + port: {{ .Values.primary.service.queryPort }} + protocol: TCP + targetPort: {{ .Values.primary.service.queryPort }} + selector: {{- include "graphscope-interactive.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: primary diff --git a/charts/graphscope-interactive/templates/query_nginx_conf.yaml b/charts/graphscope-interactive/templates/query_nginx_conf.yaml new file mode 100644 index 000000000000..461e1e59b2c8 --- /dev/null +++ b/charts/graphscope-interactive/templates/query_nginx_conf.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-query-nginx-config + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: configmap + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + nginx.conf: | + events {} + http { + resolver local=on valid=5s; + upstream my_service_1 { + {{- $baseName := include "graphscope-interactive.secondary.fullname" . }} + {{- $replicaCount := .Values.secondary.replicaCount | int }} + {{- $serviceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }} + {{- $port := .Values.secondary.service.queryPort | int }} + {{- if eq $replicaCount 1 }} + server {{ printf "%s-0.%s:%d;" $baseName $serviceName $port | quote }}; + {{- else }} + {{- range $i := until (sub $replicaCount 1 | int ) }} + server {{ printf "%s-%d.%s:%d" $baseName $i $serviceName $port | quote }}; + {{- end }} + server {{ printf "%s-%d.%s:%d" $baseName (sub $replicaCount 1) $serviceName $port | quote }}; + {{- end }} + } + + server { + listen 10000; + server_name localhost; + + location / { + proxy_pass http://my_service_1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + } diff --git a/charts/graphscope-interactive/templates/secondary/statefulset.yaml b/charts/graphscope-interactive/templates/secondary/statefulset.yaml new file mode 100644 index 000000000000..a8655f19c10c --- /dev/null +++ b/charts/graphscope-interactive/templates/secondary/statefulset.yaml @@ -0,0 +1,170 @@ +{{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }} +{{- $primaryFullName := include "graphscope-interactive.secondary.fullname" . }} +{{- $secondaryFullName := include "graphscope-interactive.secondary.fullname" . }} +{{- $releaseNamespace := .Release.Namespace }} +{{- $clusterDomain := .Values.clusterDomain }} + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "graphscope-interactive.secondary.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: secondary + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.secondary.replicaCount }} + selector: + matchLabels: {{ include "graphscope-interactive.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: secondary + serviceName: {{ include "graphscope-interactive.secondary.fullname" . }} + updateStrategy: + type: {{ .Values.secondary.updateStrategy }} + {{- if (eq "Recreate" .Values.secondary.updateStrategy) }} + rollingUpdate: null + {{- end }} + template: + metadata: + annotations: + {{- if .Values.secondary.podAnnotations }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "graphscope-interactive.labels" . | nindent 8 }} + app.kubernetes.io/component: secondary + {{- if .Values.secondary.podLabels }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.podLabels "context" $) | nindent 8 }} + {{- end }} + # alibabacloud.com/custom-cni-plugin-type: "nimitz" + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.secondary.hostAliases }} + hostAliases: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.hostAliases "context" $) | nindent 8 }} + {{- end }} + hostNetwork: {{ .Values.secondary.hostNetwork }} + hostIPC: {{ .Values.secondary.hostIPC }} + {{- if .Values.secondary.schedulerName }} + schedulerName: {{ .Values.secondary.schedulerName | quote }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy | quote }} + {{- end }} + {{- if .Values.dnsConfig }} + dnsConfig: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.dnsConfig "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "graphscope-interactive.serviceAccountName" . }} + {{- if .Values.secondary.affinity }} + affinity: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.affinity "context" $) | nindent 8 }} + {{- end }} + initContainers: + {{- if .Values.secondary.initContainers }} + {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.initContainers "context" $) | nindent 8 }} + {{- end }} + containers: + - name: secondary + image: {{ include "graphscope-interactive.secondary.image" . }} + imagePullPolicy: {{ .Values.secondary.image.pullPolicy | quote }} + #command: ["sleep", "infinity"] + command: + - /bin/bash + - -c + - | + POD_NAME=$MY_POD_NAME + if [ -z "$POD_NAME" ]; then + POD_NAME=$(hostname) + fi + echo "POD_NAME: $POD_NAME" + sudo chown -R graphscope:graphscope $INTERACTIVE_WORKSPACE + cmd="/opt/flex/bin/entrypoint.sh -w $INTERACTIVE_WORKSPACE" + echo "CMD: $cmd" + eval $cmd + # sleep infinity + env: + - name: INTERACTIVE_WORKSPACE + value: {{ .Values.workspace | quote }} + - name: primary_SERVICE_HOST + value: {{ $primaryFullName }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} + - name: ENGINE_CONFIG_PATH + value: {{ include "graphscope-interactive.engineConfigPath" . }} + - name: REAL_ENGINE_CONFIG_PATH + value: {{ include "graphscope-interactive.realEngineConfigPath" . }} + - name: SECONDARY_QUERY_PORT + value: {{ .Values.secondary.service.queryPort | quote }} + - name: ENGINE_BINARY_PATH + value: {{ include "graphscope-interactive.engineBinaryPath" . }} + - name: ENGINE_SHARD_NUM + value: {{ .Values.secondary.threadNumPerWorker | quote }} + - name: BULK_LOADER_BINARY_PATH + value: /opt/flex/bin/bulk_loader + - name: DEFAULT_GRAPH_NAME + value: {{ .Values.defaultGraph }} + - name: ODPS_ACCESS_ID + value: {{ .Values.odps.access.id | quote}} + - name: ODPS_ACCESS_KEY + value: {{ .Values.odps.access.key | quote}} + - name: ODPS_ENDPOINT + value: {{ .Values.odps.endpoint | quote}} + ports: + - name: admin-port + containerPort: {{ .Values.secondary.service.adminPort }} + - name: query-port + containerPort: {{ .Values.secondary.service.queryPort }} + {{- if .Values.resources.secondary }} + resources: {{- toYaml .Values.resources.secondary | nindent 12 }} + {{- end }} + volumeMounts: + - name: workspace + mountPath: {{ .Values.workspace }} + - name: config + mountPath: {{ include "graphscope-interactive.engineConfigPath" . }} + subPath: engine_config.yaml + # - name: config + # mountPath: /etc/interactive/setup.sh + # subPath: setup.sh + volumes: + - name: config + configMap: + name: {{ include "graphscope-interactive.configmapName" . }} + defaultMode: 0755 + {{- if and .Values.secondary.persistence.enabled .Values.secondary.persistence.existingClaim }} + - name: workspace + persistentVolumeClaim: + claimName: {{ tpl .Values.secondary.persistence.existingClaim . }} + {{- else if not .Values.secondary.persistence.enabled }} + - name: workspace + emptyDir: {} + {{- else if and .Values.secondary.persistence.enabled (not .Values.secondary.persistence.existingClaim) }} + volumeClaimTemplates: + - metadata: + name: workspace + {{- if .Values.persistence.annotations }} + annotations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.persistence.labels }} + labels: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.secondary.persistence.size | quote }} + {{ include "graphscope-interactive.storageClass" . | nindent 8 }} + {{- if .Values.secondary.persistence.selector }} + selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.secondary.persistence.selector "context" $) | nindent 10 }} + {{- end -}} + {{- end }} diff --git a/charts/graphscope-interactive/templates/secondary/svc.yaml b/charts/graphscope-interactive/templates/secondary/svc.yaml new file mode 100644 index 000000000000..50fb13d53e50 --- /dev/null +++ b/charts/graphscope-interactive/templates/secondary/svc.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "graphscope-interactive.secondary.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "graphscope-interactive.labels" . | nindent 4 }} + app.kubernetes.io/component: secondary + {{- if .Values.commonLabels }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.secondary.service.type }} + {{- if and (eq .Values.secondary.service.type "ClusterIP") .Values.secondary.service.clusterIP }} + clusterIP: {{ .Values.secondary.service.clusterIP }} + {{- end }} + {{- if and .Values.secondary.service.loadBalancerIP (eq .Values.secondary.service.type "LoadBalancer") }} + loadBalancerIP: {{ .Values.secondary.service.loadBalancerIP }} + externalTrafficPolicy: {{ .Values.secondary.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.secondary.service.type "LoadBalancer") .Values.secondary.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.secondary.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + ports: + - name: admin-port + port: {{ .Values.secondary.service.adminPort }} + protocol: TCP + targetPort: {{ .Values.secondary.service.adminPort }} + - name: query-port + port: {{ .Values.secondary.service.queryPort }} + protocol: TCP + targetPort: {{ .Values.secondary.service.queryPort }} + selector: {{- include "graphscope-interactive.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: secondary diff --git a/charts/graphscope-interactive/values.yaml b/charts/graphscope-interactive/values.yaml index ef424b655da5..0dc6aaee3cb6 100644 --- a/charts/graphscope-interactive/values.yaml +++ b/charts/graphscope-interactive/values.yaml @@ -6,6 +6,45 @@ nameOverride: "" fullnameOverride: "" +logLevel : "INFO" + +## GraphScope Interactive container's resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + primary: + limits: + cpu: 2000m + memory: 1Gi + requests: + cpu: 2000m + memory: 1Gi + secondary: + limits: + cpu: 2000m + memory: 1Gi + requests: + cpu: 2000m + memory: 1Gi + frontend: + limits: + cpu: 2000m + memory: 1Gi + requests: + cpu: 2000m + memory: 1Gi + + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + ## Cluster domain ## clusterDomain: cluster.local @@ -18,6 +57,17 @@ commonAnnotations: {} ## commonLabels: {} +global: + #imageRegistry: registry.cn-hongkong.aliyuncs.com + storageClass: "" + +odps: + secretName: "odps-secret" + access: + id: "" + key: "" + endpoint: "" + ## javaOpts: "" @@ -30,6 +80,7 @@ workspace: "/tmp/interactive_workspace" ## default graph defaultGraph: modern_graph +nodeSelector: {} hiactorWorkerNum: 1 @@ -79,30 +130,47 @@ persistence: ## labels: {} +## Ingress configuration +ingress: + hostname: "interactive.example" + paths: + - path: / + +nginx: + image: + registry: registry.cn-hongkong.aliyuncs.com + repository: graphscope/interactive + #fabiocicerchia/nginx-lua + # tag: "nginx-debug" + tag: "openresty" + pullPolicy: IfNotPresent + ## GraphScope Interactive parameters ## -engine: +primary: image: registry: registry.cn-hongkong.aliyuncs.com + #registry: reg.docker.alibaba-inc.com repository: graphscope/interactive + #repository: 7brs/interactive # Overrides the image tag whose default is the chart appVersion. - tag: "v0.0.3" + tag: "debug" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images ## - pullPolicy: IfNotPresent + pullPolicy: Always ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## Example: ## pullSecrets: ## - myRegistryKeySecretName ## - pullSecrets: [ ] + pullSecrets: [] replicaCount: 1 - logLevel: INFO + hangUntilSuccess: false # Number of thread each worker will use threadNumPerWorker: 1 @@ -133,6 +201,7 @@ engine: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set ## + affinity: {} # affinity: # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: @@ -148,6 +217,10 @@ engine: ## nodeSelector: {} + hostAliases: {} + + hostIPC: false + ## Tolerations for GraphScope Interactive pods assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## @@ -167,21 +240,6 @@ engine: enabled: false runAsUser: 1001 - ## GraphScope Interactive container's resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - ## GraphScope Interactive container's liveness and readiness probes ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes ## @@ -210,8 +268,8 @@ engine: ## Name of existing PVC to hold GraphScope Interactive data ## NOTE: When it's set the rest of persistence parameters are ignored ## - existingClaim: "graphscope-interactive-pvc" - #existingClaim: "" + # existingClaim: "graphscope-interactive-pvc" + existingClaim: "" ## Persistent Volume Storage Class ## If defined, storageClassName: @@ -248,7 +306,11 @@ engine: type: ClusterIP ## Service port ## - servicePort: 55557 + ports: + - name: query_port + port: 10000 + targetPort: 10000 + protocol: TCP queryPort: 10000 @@ -296,8 +358,227 @@ engine: ## # maxUnavailable: 1 - ## GraphScope Interactive pod label. If labels are same as commonLabels , this will take precedence. + # ## GraphScope Interactive pod label. If labels are same as commonLabels , this will take precedence. + # ## + podLabels: {} + + +## GraphScope Interactive parameters +## +secondary: + image: + registry: registry.cn-hongkong.aliyuncs.com + #registry: reg.docker.alibaba-inc.com + repository: graphscope/interactive + #repository: 7brs/interactive + # Overrides the image tag whose default is the chart appVersion. + tag: "debug" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + replicaCount: 2 + + # Number of thread each worker will use + threadNumPerWorker: 1 + + ## updateStrategy for GraphScope Interactive statefulset + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies + ## + updateStrategy: RollingUpdate + + ## GraphScope Interactive pod annotations + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ ## + podAnnotations: {} + + ## GraphScope Interactive pod affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## Allowed values: soft, hard + ## + podAffinityPreset: "" + + ## GraphScope Interactive pod anti-affinity preset + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + ## Allowed values: soft, hard + ## + podAntiAffinityPreset: soft + + ## Affinity for GraphScope Interactive pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set + ## + affinity: {} + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: app + # operator: In + # values: + # - interactive_single_node + + ## Node labels for GraphScope Interactive pods assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + hostAliases: {} + + hostIPC: false + + ## Tolerations for GraphScope Interactive pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## GraphScope Interactive Pod security context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + ## + podSecurityContext: + enabled: false + fsGroup: 1001 + + ## GraphScope Interactive container security context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## + containerSecurityContext: + enabled: false + runAsUser: 1001 + + + ## GraphScope Interactive container's liveness and readiness probes + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes + ## + livenessProbe: + enabled: false + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + readinessProbe: + enabled: false + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + ## If true, use a Persistent Volume Claim, If false, use emptyDir + ## + enabled: true + ## Name of existing PVC to hold GraphScope Interactive data + ## NOTE: When it's set the rest of persistence parameters are ignored + ## + # existingClaim: "graphscope-interactive-pvc" + existingClaim: "" + + ## Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "manual" + ## Persistent Volume Claim annotations + ## + annotations: {} + ## Persistent Volume Access Mode + ## + accessModes: + - ReadWriteOnce # read and write by a single node. + ## Persistent Volume size + ## + size: 3Gi + ## selector can be used to match an existing PersistentVolume + ## selector: + ## matchLabels: + ## app: my-app + ## + selector: {} + + initContainers: [] + + ## GraphScope interactive Service parameters + ## + service: + ## Service type + ## + #type: NodePort + type: ClusterIP + # type: LoadBalancer + ## Service port + ## + ports: + - name: query_port + port: 10000 + targetPort: 10000 + protocol: TCP + + queryPort: 10000 + + adminPort: 7777 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + service: "" + query: "" + admin: "" + ## Service clusterIP + ## + clusterIP: None + #clusterIP: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + loadBalancerIP: "" + ## Enable client source IP preservation + ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + ## Load Balancer sources + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## E.g. + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Provide any additional annotations which may be required + ## + annotations: {} + + ## GraphScope Interactive Pod Disruption Budget configuration + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + ## + pdb: + enabled: false + ## Min number of pods that must still be available after the eviction + ## + minAvailable: 1 + ## Max number of pods that can be unavailable after the eviction + ## + # maxUnavailable: 1 + + # ## GraphScope Interactive pod label. If labels are same as commonLabels , this will take precedence. + # ## podLabels: {} ## GraphScope Frontend parameters @@ -305,14 +586,16 @@ engine: frontend: image: registry: registry.cn-hongkong.aliyuncs.com + #registry: reg.docker.alibaba-inc.com repository: graphscope/interactive + #repository: 7brs/interactive # Overrides the image tag whose default is the chart appVersion. - tag: "v0.0.3" + tag: "debug" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images ## - pullPolicy: IfNotPresent + pullPolicy: Always ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## Example: @@ -349,7 +632,7 @@ frontend: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set ## - ## affinity: {} + affinity: {} # affinity: # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: @@ -365,25 +648,14 @@ frontend: ## nodeSelector: {} + hostIPC: false + + hostAliases: {} + ## Tolerations for GraphScope Interactive pods assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## - tolerations: [] - - ## GraphScope Interactive container's resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # tolerations: [] ## GraphScope Interactive container's liveness and readiness probes ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes @@ -410,10 +682,7 @@ frontend: service: ## Service type ## - type: LoadBalancer - ## Service port - ## - servicePort: 55556 + type: NodePort ## Gremlin console port ## @@ -422,6 +691,10 @@ frontend: ## Cypher server port cypherPort: 7687 + adminPort: 7778 + + queryPort: 10001 + ## query timeout queryTimeout: 30000 @@ -429,9 +702,10 @@ frontend: ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport ## nodePorts: - service: "" - gremlin: "" - cypher: "" + admin: 30001 + query: 30002 + gremlin: 30003 + cypher: 30004 ## Service clusterIP ## # clusterIP: None @@ -471,5 +745,3 @@ frontend: ## podLabels: {} -global: - storageClass: "" diff --git a/k8s/dockerfiles/interactive-entrypoint.sh b/k8s/dockerfiles/interactive-entrypoint.sh index 74eb149b3118..1258e4603393 100644 --- a/k8s/dockerfiles/interactive-entrypoint.sh +++ b/k8s/dockerfiles/interactive-entrypoint.sh @@ -38,24 +38,40 @@ EOF function prepare_workspace() { #receive args local workspace=$1 + local engine_config_path="${workspace}/conf/interactive_config.yaml" if [ -z "${workspace}" ]; then workspace="/tmp/interactive_workspace" fi #if workspace is not exist, create it if [ ! -d "${workspace}" ]; then - mkdir -p ${workspace} - mkdir -p ${workspace}/conf/ - else - echo "Workspace ${workspace} already exists" + mkdir -p ${workspace} + fi + if [ ! -d "${workspace}/conf" ]; then + mkdir -p ${workspace}/conf + fi + if [ ! -d "${workspace}/data" ]; then + mkdir -p ${workspace}/data + fi + if [ -f "${engine_config_path}" ]; then + echo "Engine config file ${engine_config_path} already exists" + echo "Using existing engine config file" return 0 + else + echo "Engine config file ${engine_config_path} does not exist" + echo "Creating engine config file and prepare the workspace" + # prepare interactive_config.yaml + cp /opt/flex/share/interactive_config.yaml $engine_config_path + #make sure the line which start with default_graph is changed to default_graph: ${DEFAULT_GRAPH_NAME} + sed -i "s/default_graph:.*/default_graph: ${DEFAULT_GRAPH_NAME}/" $engine_config_path fi - # prepare interactive_config.yaml - engine_config_path="${workspace}/conf/interactive_config.yaml" - cp /opt/flex/share/interactive_config.yaml $engine_config_path - #make sure the line which start with default_graph is changed to default_graph: ${DEFAULT_GRAPH_NAME} - sed -i "s/default_graph:.*/default_graph: ${DEFAULT_GRAPH_NAME}/" $engine_config_path - echo "Using default graph: ${DEFAULT_GRAPH_NAME} to start the service" - + + if [ -d "${workspace}/data/${DEFAULT_GRAPH_NAME}" ]; then + echo "Graph data directory ${workspace}/data/${DEFAULT_GRAPH_NAME} already exists" + echo "Using existing graph data directory" + return 0 + fi + + echo "Using default graph: ${DEFAULT_GRAPH_NAME} to start the service" # copy the builtin graph builtin_graph_dir="${workspace}/data/${DEFAULT_GRAPH_NAME}" mkdir -p $builtin_graph_dir @@ -106,6 +122,22 @@ EOF fi } +function launch_compiler() { + #expect 1 arg + if [ $# -ne 1 ]; then + echo "Usage: launch_compiler " + exit 1 + fi + local endpoint=$1 + compiler_config_path="/opt/flex/share/interactive_config.yaml" + compiler_cmd="java -cp /opt/flex/lib/* -Djna.library.path=/opt/flex/lib" + compiler_cmd="${compiler_cmd} -Dgraph.schema=${endpoint}/v1/service/status" + compiler_cmd="${compiler_cmd} com.alibaba.graphscope.GraphServer" + compiler_cmd="${compiler_cmd} ${compiler_config_path}" + echo "Starting the compiler with command:${compiler_cmd}" + eval $compiler_cmd +} + #################### Entry #################### @@ -113,6 +145,15 @@ ENABLE_COORDINATOR=false WORKSPACE=/tmp/interactive_workspace while [[ $# -gt 0 ]]; do case $1 in + -e | --endpoint) + shift + if [[ $# -eq 0 || $1 == -* ]]; then + echo "Option -e requires an argument." >&2 + exit 1 + fi + ENDPOINT=$1 + shift + ;; -w | --workspace) shift if [[ $# -eq 0 || $1 == -* ]]; then @@ -138,7 +179,12 @@ while [[ $# -gt 0 ]]; do esac done +if [ ! -z "${ENDPOINT}" ]; then + echo "ENDPOINT is set to ${ENDPOINT}, start compiler with this endpoint" + launch_compiler $ENDPOINT +else + prepare_workspace $WORKSPACE + launch_service $WORKSPACE + launch_coordinator +fi -prepare_workspace $WORKSPACE -launch_service $WORKSPACE -launch_coordinator diff --git a/k8s/dockerfiles/openresty.Dockerfile b/k8s/dockerfiles/openresty.Dockerfile new file mode 100644 index 000000000000..e1d23a0c5a56 --- /dev/null +++ b/k8s/dockerfiles/openresty.Dockerfile @@ -0,0 +1,6 @@ +FROM openresty/openresty:jammy + +# Copy https://github.com/ledgetech/lua-resty-http/blob/master/lib/resty/http_connect.lua to /usr/local/openresty/lualib/resty/http_connect.lua +COPY ./dockerfiles/resty/http_connect.lua /usr/local/openresty/lualib/resty/http_connect.lua +COPY ./dockerfiles/resty/http_headers.lua /usr/local/openresty/lualib/resty/http_headers.lua +COPY ./dockerfiles/resty/http.lua /usr/local/openresty/lualib/resty/http.lua \ No newline at end of file diff --git a/k8s/dockerfiles/resty/http.lua b/k8s/dockerfiles/resty/http.lua new file mode 100644 index 000000000000..8feaf77fe19b --- /dev/null +++ b/k8s/dockerfiles/resty/http.lua @@ -0,0 +1,1185 @@ +local http_headers = require "resty.http_headers" + +local ngx = ngx +local ngx_socket_tcp = ngx.socket.tcp +local ngx_req = ngx.req +local ngx_req_socket = ngx_req.socket +local ngx_req_get_headers = ngx_req.get_headers +local ngx_req_get_method = ngx_req.get_method +local str_lower = string.lower +local str_upper = string.upper +local str_find = string.find +local str_sub = string.sub +local tbl_concat = table.concat +local tbl_insert = table.insert +local ngx_encode_args = ngx.encode_args +local ngx_re_match = ngx.re.match +local ngx_re_gmatch = ngx.re.gmatch +local ngx_re_sub = ngx.re.sub +local ngx_re_gsub = ngx.re.gsub +local ngx_re_find = ngx.re.find +local ngx_log = ngx.log +local ngx_DEBUG = ngx.DEBUG +local ngx_ERR = ngx.ERR +local ngx_var = ngx.var +local ngx_print = ngx.print +local ngx_header = ngx.header +local co_yield = coroutine.yield +local co_create = coroutine.create +local co_status = coroutine.status +local co_resume = coroutine.resume +local setmetatable = setmetatable +local tonumber = tonumber +local tostring = tostring +local unpack = unpack +local rawget = rawget +local select = select +local ipairs = ipairs +local pairs = pairs +local pcall = pcall +local type = type + + +-- http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1 +local HOP_BY_HOP_HEADERS = { + ["connection"] = true, + ["keep-alive"] = true, + ["proxy-authenticate"] = true, + ["proxy-authorization"] = true, + ["te"] = true, + ["trailers"] = true, + ["transfer-encoding"] = true, + ["upgrade"] = true, + ["content-length"] = true, -- Not strictly hop-by-hop, but Nginx will deal + -- with this (may send chunked for example). +} + + +local EXPECTING_BODY = { + POST = true, + PUT = true, + PATCH = true, +} + + +-- Reimplemented coroutine.wrap, returning "nil, err" if the coroutine cannot +-- be resumed. This protects user code from infinite loops when doing things like +-- repeat +-- local chunk, err = res.body_reader() +-- if chunk then -- <-- This could be a string msg in the core wrap function. +-- ... +-- end +-- until not chunk +local co_wrap = function(func) + local co = co_create(func) + if not co then + return nil, "could not create coroutine" + else + return function(...) + if co_status(co) == "suspended" then + return select(2, co_resume(co, ...)) + else + return nil, "can't resume a " .. co_status(co) .. " coroutine" + end + end + end +end + + +-- Returns a new table, recursively copied from the one given. +-- +-- @param table table to be copied +-- @return table +local function tbl_copy(orig) + local orig_type = type(orig) + local copy + if orig_type == "table" then + copy = {} + for orig_key, orig_value in next, orig, nil do + copy[tbl_copy(orig_key)] = tbl_copy(orig_value) + end + else -- number, string, boolean, etc + copy = orig + end + return copy +end + + +local _M = { + _VERSION = '0.17.2', +} +_M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version + +local mt = { __index = _M } + + +local HTTP = { + [1.0] = " HTTP/1.0\r\n", + [1.1] = " HTTP/1.1\r\n", +} + + +local DEFAULT_PARAMS = { + method = "GET", + path = "/", + version = 1.1, +} + + +local DEBUG = false + + +function _M.new(_) + local sock, err = ngx_socket_tcp() + if not sock then + return nil, err + end + return setmetatable({ sock = sock, keepalive = true }, mt) +end + + +function _M.debug(d) + DEBUG = (d == true) +end + + +function _M.set_timeout(self, timeout) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + return sock:settimeout(timeout) +end + + +function _M.set_timeouts(self, connect_timeout, send_timeout, read_timeout) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + return sock:settimeouts(connect_timeout, send_timeout, read_timeout) +end + +do + local aio_connect = require "resty.http_connect" + -- Function signatures to support: + -- ok, err, ssl_session = httpc:connect(options_table) + -- ok, err = httpc:connect(host, port, options_table?) + -- ok, err = httpc:connect("unix:/path/to/unix.sock", options_table?) + function _M.connect(self, options, ...) + if type(options) == "table" then + -- all-in-one interface + return aio_connect(self, options) + else + -- backward compatible + return self:tcp_only_connect(options, ...) + end + end +end + +function _M.tcp_only_connect(self, ...) + ngx_log(ngx_DEBUG, "Use of deprecated `connect` method signature") + + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + self.host = select(1, ...) + self.port = select(2, ...) + + -- If port is not a number, this is likely a unix domain socket connection. + if type(self.port) ~= "number" then + self.port = nil + end + + self.keepalive = true + self.ssl = false + + return sock:connect(...) +end + + +function _M.set_keepalive(self, ...) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + if self.keepalive == true then + return sock:setkeepalive(...) + else + -- The server said we must close the connection, so we cannot setkeepalive. + -- If close() succeeds we return 2 instead of 1, to differentiate between + -- a normal setkeepalive() failure and an intentional close(). + local res, err = sock:close() + if res then + return 2, "connection must be closed" + else + return res, err + end + end +end + + +function _M.get_reused_times(self) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + return sock:getreusedtimes() +end + + +function _M.close(self) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + return sock:close() +end + + +local function _should_receive_body(method, code) + if method == "HEAD" then return nil end + if code == 204 or code == 304 then return nil end + if code >= 100 and code < 200 then return nil end + return true +end + + +function _M.parse_uri(_, uri, query_in_path) + if query_in_path == nil then query_in_path = true end + + local m, err = ngx_re_match( + uri, + [[^(?:(http[s]?):)?//((?:[^\[\]:/\?]+)|(?:\[.+\]))(?::(\d+))?([^\?]*)\??(.*)]], + "jo" + ) + + if not m then + if err then + return nil, "failed to match the uri: " .. uri .. ", " .. err + end + + return nil, "bad uri: " .. uri + else + -- If the URI is schemaless (i.e. //example.com) try to use our current + -- request scheme. + if not m[1] then + -- Schema-less URIs can occur in client side code, implying "inherit + -- the schema from the current request". We support it for a fairly + -- specific case; if for example you are using the ESI parser in + -- ledge (https://github.com/ledgetech/ledge) to perform in-flight + -- sub requests on the edge based on instructions found in markup, + -- those URIs may also be schemaless with the intention that the + -- subrequest would inherit the schema just like JavaScript would. + local scheme = ngx_var.scheme + if scheme == "http" or scheme == "https" then + m[1] = scheme + else + return nil, "schemaless URIs require a request context: " .. uri + end + end + + if m[3] then + m[3] = tonumber(m[3]) + else + if m[1] == "https" then + m[3] = 443 + else + m[3] = 80 + end + end + if not m[4] or "" == m[4] then m[4] = "/" end + + if query_in_path and m[5] and m[5] ~= "" then + m[4] = m[4] .. "?" .. m[5] + m[5] = nil + end + + return m, nil + end +end + + +local function _format_request(self, params) + local version = params.version + local headers = params.headers or {} + + local query = params.query or "" + if type(query) == "table" then + query = ngx_encode_args(query) + end + + if query ~= "" and str_sub(query, 1, 1) ~= "?" then + query = "?" .. query + end + + -- Initialize request + local req = { + str_upper(params.method), + " ", + self.path_prefix or "", + params.path, + query, + HTTP[version], + -- Pre-allocate slots for minimum headers and carriage return. + true, + true, + true, + } + local c = 7 -- req table index it's faster to do this inline vs table.insert + + -- Append headers + for key, values in pairs(headers) do + key = tostring(key) + + if type(values) == "table" then + for _, value in pairs(values) do + req[c] = key .. ": " .. tostring(value) .. "\r\n" + c = c + 1 + end + + else + req[c] = key .. ": " .. tostring(values) .. "\r\n" + c = c + 1 + end + end + + -- Close headers + req[c] = "\r\n" + + return tbl_concat(req) +end + + +local function _receive_status(sock) + local line, err = sock:receive("*l") + if not line then + return nil, nil, nil, err + end + + local version = tonumber(str_sub(line, 6, 8)) + if not version then + return nil, nil, nil, + "couldn't parse HTTP version from response status line: " .. line + end + + local status = tonumber(str_sub(line, 10, 12)) + if not status then + return nil, nil, nil, + "couldn't parse status code from response status line: " .. line + end + + local reason = str_sub(line, 14) + + return status, version, reason +end + + +local function _receive_headers(sock) + local headers = http_headers.new() + + repeat + local line, err = sock:receive("*l") + if not line then + return nil, err + end + + local m, err = ngx_re_match(line, "([^:\\s]+):\\s*(.*)", "jo") + if err then ngx_log(ngx_ERR, err) end + + if not m then + break + end + + local key = m[1] + local val = m[2] + if headers[key] then + if type(headers[key]) ~= "table" then + headers[key] = { headers[key] } + end + tbl_insert(headers[key], tostring(val)) + else + headers[key] = tostring(val) + end + until ngx_re_find(line, "^\\s*$", "jo") + + return headers, nil +end + + +local function transfer_encoding_is_chunked(headers) + local te = headers["Transfer-Encoding"] + if not te then + return false + end + + -- Handle duplicate headers + -- This shouldn't happen but can in the real world + if type(te) ~= "string" then + te = tbl_concat(te, ",") + end + + return str_find(str_lower(te), "chunked", 1, true) ~= nil +end +_M.transfer_encoding_is_chunked = transfer_encoding_is_chunked + + +local function _chunked_body_reader(sock, default_chunk_size) + return co_wrap(function(max_chunk_size) + local remaining = 0 + local length + max_chunk_size = max_chunk_size or default_chunk_size + + repeat + -- If we still have data on this chunk + if max_chunk_size and remaining > 0 then + + if remaining > max_chunk_size then + -- Consume up to max_chunk_size + length = max_chunk_size + remaining = remaining - max_chunk_size + else + -- Consume all remaining + length = remaining + remaining = 0 + end + else -- This is a fresh chunk + + -- Receive the chunk size + local str, err = sock:receive("*l") + if not str then + co_yield(nil, err) + end + + length = tonumber(str, 16) + + if not length then + co_yield(nil, "unable to read chunksize") + end + + if max_chunk_size and length > max_chunk_size then + -- Consume up to max_chunk_size + remaining = length - max_chunk_size + length = max_chunk_size + end + end + + if length > 0 then + local str, err = sock:receive(length) + if not str then + co_yield(nil, err) + end + + max_chunk_size = co_yield(str) or default_chunk_size + + -- If we're finished with this chunk, read the carriage return. + if remaining == 0 then + sock:receive(2) -- read \r\n + end + else + -- Read the last (zero length) chunk's carriage return + sock:receive(2) -- read \r\n + end + + until length == 0 + end) +end + + +local function _body_reader(sock, content_length, default_chunk_size) + return co_wrap(function(max_chunk_size) + max_chunk_size = max_chunk_size or default_chunk_size + + if not content_length and max_chunk_size then + -- We have no length, but wish to stream. + -- HTTP 1.0 with no length will close connection, so read chunks to the end. + repeat + local str, err, partial = sock:receive(max_chunk_size) + if not str and err == "closed" then + co_yield(partial, err) + end + + max_chunk_size = tonumber(co_yield(str) or default_chunk_size) + if max_chunk_size and max_chunk_size < 0 then max_chunk_size = nil end + + if not max_chunk_size then + ngx_log(ngx_ERR, "Buffer size not specified, bailing") + break + end + until not str + + elseif not content_length then + -- We have no length but don't wish to stream. + -- HTTP 1.0 with no length will close connection, so read to the end. + co_yield(sock:receive("*a")) + + elseif not max_chunk_size then + -- We have a length and potentially keep-alive, but want everything. + co_yield(sock:receive(content_length)) + + else + -- We have a length and potentially a keep-alive, and wish to stream + -- the response. + local received = 0 + repeat + local length = max_chunk_size + if received + length > content_length then + length = content_length - received + end + + if length > 0 then + local str, err = sock:receive(length) + if not str then + co_yield(nil, err) + end + received = received + length + + max_chunk_size = tonumber(co_yield(str) or default_chunk_size) + if max_chunk_size and max_chunk_size < 0 then max_chunk_size = nil end + + if not max_chunk_size then + ngx_log(ngx_ERR, "Buffer size not specified, bailing") + break + end + end + + until length == 0 + end + end) +end + + +local function _no_body_reader() + return nil +end + + +local function _read_body(res) + local reader = res.body_reader + + if not reader then + -- Most likely HEAD or 304 etc. + return nil, "no body to be read" + end + + local chunks = {} + local c = 1 + + local chunk, err + repeat + chunk, err = reader() + + if err then + return nil, err, tbl_concat(chunks) -- Return any data so far. + end + if chunk then + chunks[c] = chunk + c = c + 1 + end + until not chunk + + return tbl_concat(chunks) +end + + +local function _trailer_reader(sock) + return co_wrap(function() + co_yield(_receive_headers(sock)) + end) +end + + +local function _read_trailers(res) + local reader = res.trailer_reader + if not reader then + return nil, "no trailers" + end + + local trailers = reader() + setmetatable(res.headers, { __index = trailers }) +end + + +local function _send_body(sock, body) + if type(body) == "function" then + repeat + local chunk, err, partial = body() + + if chunk then + local ok, err = sock:send(chunk) + + if not ok then + return nil, err + end + elseif err ~= nil then + return nil, err, partial + end + + until chunk == nil + elseif body ~= nil then + local bytes, err = sock:send(body) + + if not bytes then + return nil, err + end + end + return true, nil +end + + +local function _handle_continue(sock, body) + local status, version, reason, err = _receive_status(sock) --luacheck: no unused + if not status then + return nil, nil, nil, err + end + + -- Only send body if we receive a 100 Continue + if status == 100 then + -- Read headers + local headers, err = _receive_headers(sock) + if not headers then + return nil, nil, nil, err + end + + local ok, err = _send_body(sock, body) + if not ok then + return nil, nil, nil, err + end + end + return status, version, reason, err +end + + +function _M.send_request(self, params) + -- Apply defaults + setmetatable(params, { __index = DEFAULT_PARAMS }) + + local sock = self.sock + local body = params.body + local headers = http_headers.new() + + -- We assign one-by-one so that the metatable can handle case insensitivity + -- for us. You can blame the spec for this inefficiency. + local params_headers = params.headers or {} + for k, v in pairs(params_headers) do + headers[k] = v + end + + if not headers["Proxy-Authorization"] then + -- TODO: next major, change this to always override the provided + -- header. Can't do that yet because it would be breaking. + -- The connect method uses self.http_proxy_auth in the poolname so + -- that should be leading. + headers["Proxy-Authorization"] = self.http_proxy_auth + end + + -- Ensure we have appropriate message length or encoding. + do + local is_chunked = transfer_encoding_is_chunked(headers) + + if is_chunked then + -- If we have both Transfer-Encoding and Content-Length we MUST + -- drop the Content-Length, to help prevent request smuggling. + -- https://tools.ietf.org/html/rfc7230#section-3.3.3 + headers["Content-Length"] = nil + + elseif not headers["Content-Length"] then + -- A length was not given, try to calculate one. + + local body_type = type(body) + + if body_type == "function" then + return nil, "Request body is a function but a length or chunked encoding is not specified" + + elseif body_type == "table" then + local length = 0 + for _, v in ipairs(body) do + length = length + #tostring(v) + end + headers["Content-Length"] = length + + elseif body == nil and EXPECTING_BODY[str_upper(params.method)] then + headers["Content-Length"] = 0 + + elseif body ~= nil then + headers["Content-Length"] = #tostring(body) + end + end + end + + if not headers["Host"] then + if (str_sub(self.host, 1, 5) == "unix:") then + return nil, "Unable to generate a useful Host header for a unix domain socket. Please provide one." + end + -- If we have a port (i.e. not connected to a unix domain socket), and this + -- port is non-standard, append it to the Host header. + if self.port then + if self.ssl and self.port ~= 443 then + headers["Host"] = self.host .. ":" .. self.port + elseif not self.ssl and self.port ~= 80 then + headers["Host"] = self.host .. ":" .. self.port + else + headers["Host"] = self.host + end + else + headers["Host"] = self.host + end + end + if not headers["User-Agent"] then + headers["User-Agent"] = _M._USER_AGENT + end + if params.version == 1.0 and not headers["Connection"] then + headers["Connection"] = "Keep-Alive" + end + + params.headers = headers + + -- Format and send request + local req = _format_request(self, params) + if DEBUG then ngx_log(ngx_DEBUG, "\n", req) end + local bytes, err = sock:send(req) + + if not bytes then + return nil, err + end + + -- Send the request body, unless we expect: continue, in which case + -- we handle this as part of reading the response. + if headers["Expect"] ~= "100-continue" then + local ok, err, partial = _send_body(sock, body) + if not ok then + return nil, err, partial + end + end + + return true +end + + +function _M.read_response(self, params) + local sock = self.sock + + local status, version, reason, err + + -- If we expect: continue, we need to handle this, sending the body if allowed. + -- If we don't get 100 back, then status is the actual status. + if params.headers["Expect"] == "100-continue" then + local _status, _version, _reason, _err = _handle_continue(sock, params.body) + if not _status then + return nil, _err + elseif _status ~= 100 then + status, version, reason, err = _status, _version, _reason, _err -- luacheck: no unused + end + end + + -- Just read the status as normal. + if not status then + status, version, reason, err = _receive_status(sock) + if not status then + return nil, err + end + end + + + local res_headers, err = _receive_headers(sock) + if not res_headers then + return nil, err + end + + -- keepalive is true by default. Determine if this is correct or not. + local ok, connection = pcall(str_lower, res_headers["Connection"]) + if ok then + if (version == 1.1 and str_find(connection, "close", 1, true)) or + (version == 1.0 and not str_find(connection, "keep-alive", 1, true)) then + self.keepalive = false + end + else + -- no connection header + if version == 1.0 then + self.keepalive = false + end + end + + local body_reader = _no_body_reader + local trailer_reader, err + local has_body = false + + -- Receive the body_reader + if _should_receive_body(params.method, status) then + has_body = true + + if version == 1.1 and transfer_encoding_is_chunked(res_headers) then + body_reader, err = _chunked_body_reader(sock) + else + local ok, length = pcall(tonumber, res_headers["Content-Length"]) + if not ok then + -- No content-length header, read until connection is closed by server + length = nil + end + + body_reader, err = _body_reader(sock, length) + end + end + + if res_headers["Trailer"] then + trailer_reader, err = _trailer_reader(sock) + end + + if err then + return nil, err + else + return { + status = status, + reason = reason, + headers = res_headers, + has_body = has_body, + body_reader = body_reader, + read_body = _read_body, + trailer_reader = trailer_reader, + read_trailers = _read_trailers, + } + end +end + + +function _M.request(self, params) + params = tbl_copy(params) -- Take by value + local res, err = self:send_request(params) + if not res then + return res, err + else + return self:read_response(params) + end +end + + +function _M.request_pipeline(self, requests) + requests = tbl_copy(requests) -- Take by value + + for _, params in ipairs(requests) do + if params.headers and params.headers["Expect"] == "100-continue" then + return nil, "Cannot pipeline request specifying Expect: 100-continue" + end + + local res, err = self:send_request(params) + if not res then + return res, err + end + end + + local responses = {} + for i, params in ipairs(requests) do + responses[i] = setmetatable({ + params = params, + response_read = false, + }, { + -- Read each actual response lazily, at the point the user tries + -- to access any of the fields. + __index = function(t, k) + local res, err + if t.response_read == false then + res, err = _M.read_response(self, t.params) + t.response_read = true + + if not res then + ngx_log(ngx_ERR, err) + else + for rk, rv in pairs(res) do + t[rk] = rv + end + end + end + return rawget(t, k) + end, + }) + end + return responses +end + + +function _M.request_uri(self, uri, params) + params = tbl_copy(params or {}) -- Take by value + if self.proxy_opts then + params.proxy_opts = tbl_copy(self.proxy_opts or {}) + end + + do + local parsed_uri, err = self:parse_uri(uri, false) + if not parsed_uri then + return nil, err + end + + local path, query + params.scheme, params.host, params.port, path, query = unpack(parsed_uri) + params.path = params.path or path + params.query = params.query or query + params.ssl_server_name = params.ssl_server_name or params.host + end + + do + local proxy_auth = (params.headers or {})["Proxy-Authorization"] + if proxy_auth and params.proxy_opts then + params.proxy_opts.https_proxy_authorization = proxy_auth + params.proxy_opts.http_proxy_authorization = proxy_auth + end + end + + local ok, err = self:connect(params) + if not ok then + return nil, err + end + + local res, err = self:request(params) + if not res then + self:close() + return nil, err + end + + local body, err = res:read_body() + if not body then + self:close() + return nil, err + end + + res.body = body + + if params.keepalive == false then + local ok, err = self:close() + if not ok then + ngx_log(ngx_ERR, err) + end + + else + local ok, err = self:set_keepalive(params.keepalive_timeout, params.keepalive_pool) + if not ok then + ngx_log(ngx_ERR, err) + end + + end + + return res, nil +end + + +function _M.get_client_body_reader(_, chunksize, sock) + chunksize = chunksize or 65536 + + if not sock then + local ok, err + ok, sock, err = pcall(ngx_req_socket) + + if not ok then + return nil, sock -- pcall err + end + + if not sock then + if err == "no body" then + return nil + else + return nil, err + end + end + end + + local headers = ngx_req_get_headers() + local length = headers.content_length + if length then + return _body_reader(sock, tonumber(length), chunksize) + elseif transfer_encoding_is_chunked(headers) then + -- Not yet supported by ngx_lua but should just work... + return _chunked_body_reader(sock, chunksize) + else + return nil + end +end + + +function _M.set_proxy_options(self, opts) + -- TODO: parse and cache these options, instead of parsing them + -- on each request over and over again (lru-cache on module level) + self.proxy_opts = tbl_copy(opts) -- Take by value +end + + +function _M.get_proxy_uri(self, scheme, host) + if not self.proxy_opts then + return nil + end + + -- Check if the no_proxy option matches this host. Implementation adapted + -- from lua-http library (https://github.com/daurnimator/lua-http) + if self.proxy_opts.no_proxy then + if self.proxy_opts.no_proxy == "*" then + -- all hosts are excluded + return nil + end + + local no_proxy_set = {} + -- wget allows domains in no_proxy list to be prefixed by "." + -- e.g. no_proxy=.mit.edu + for host_suffix in ngx_re_gmatch(self.proxy_opts.no_proxy, "\\.?([^,]+)", "jo") do + no_proxy_set[host_suffix[1]] = true + end + + -- From curl docs: + -- matched as either a domain which contains the hostname, or the + -- hostname itself. For example local.com would match local.com, + -- local.com:80, and www.local.com, but not www.notlocal.com. + -- + -- Therefore, we keep stripping subdomains from the host, compare + -- them to the ones in the no_proxy list and continue until we find + -- a match or until there's only the TLD left + repeat + if no_proxy_set[host] then + return nil + end + + -- Strip the next level from the domain and check if that one + -- is on the list + host = ngx_re_sub(host, "^[^.]+\\.", "", "jo") + until not ngx_re_find(host, "\\.", "jo") + end + + if scheme == "http" and self.proxy_opts.http_proxy then + return self.proxy_opts.http_proxy + end + + if scheme == "https" and self.proxy_opts.https_proxy then + return self.proxy_opts.https_proxy + end + + return nil +end + + +-- ---------------------------------------------------------------------------- +-- The following functions are considered DEPRECATED and may be REMOVED in +-- future releases. Please see the notes in `README.md`. +-- ---------------------------------------------------------------------------- + +function _M.ssl_handshake(self, ...) + ngx_log(ngx_DEBUG, "Use of deprecated function `ssl_handshake`") + + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + self.ssl = true + + return sock:sslhandshake(...) +end + + +function _M.connect_proxy(self, proxy_uri, scheme, host, port, proxy_authorization) + ngx_log(ngx_DEBUG, "Use of deprecated function `connect_proxy`") + + -- Parse the provided proxy URI + local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) + if not parsed_proxy_uri then + return nil, err + end + + -- Check that the scheme is http (https is not supported for + -- connections between the client and the proxy) + local proxy_scheme = parsed_proxy_uri[1] + if proxy_scheme ~= "http" then + return nil, "protocol " .. proxy_scheme .. " not supported for proxy connections" + end + + -- Make the connection to the given proxy + local proxy_host, proxy_port = parsed_proxy_uri[2], parsed_proxy_uri[3] + local c, err = self:tcp_only_connect(proxy_host, proxy_port) + if not c then + return nil, err + end + + if scheme == "https" then + -- Make a CONNECT request to create a tunnel to the destination through + -- the proxy. The request-target and the Host header must be in the + -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section + -- 4.3.6 for more details about the CONNECT request + local destination = host .. ":" .. port + local res, err = self:request({ + method = "CONNECT", + path = destination, + headers = { + ["Host"] = destination, + ["Proxy-Authorization"] = proxy_authorization, + } + }) + + if not res then + return nil, err + end + + if res.status < 200 or res.status > 299 then + return nil, "failed to establish a tunnel through a proxy: " .. res.status + end + end + + return c, nil +end + + +function _M.proxy_request(self, chunksize) + ngx_log(ngx_DEBUG, "Use of deprecated function `proxy_request`") + + return self:request({ + method = ngx_req_get_method(), + path = ngx_re_gsub(ngx_var.uri, "\\s", "%20", "jo") .. ngx_var.is_args .. (ngx_var.query_string or ""), + body = self:get_client_body_reader(chunksize), + headers = ngx_req_get_headers(), + }) +end + + +function _M.proxy_response(_, response, chunksize) + ngx_log(ngx_DEBUG, "Use of deprecated function `proxy_response`") + + if not response then + ngx_log(ngx_ERR, "no response provided") + return + end + + ngx.status = response.status + + -- Filter out hop-by-hop headeres + for k, v in pairs(response.headers) do + if not HOP_BY_HOP_HEADERS[str_lower(k)] then + ngx_header[k] = v + end + end + + local reader = response.body_reader + + repeat + local chunk, ok, read_err, print_err + + chunk, read_err = reader(chunksize) + if read_err then + ngx_log(ngx_ERR, read_err) + end + + if chunk then + ok, print_err = ngx_print(chunk) + if not ok then + ngx_log(ngx_ERR, print_err) + end + end + + if read_err or print_err then + break + end + until not chunk +end + + +return _M \ No newline at end of file diff --git a/k8s/dockerfiles/resty/http_connect.lua b/k8s/dockerfiles/resty/http_connect.lua new file mode 100644 index 000000000000..8ca967c9ce88 --- /dev/null +++ b/k8s/dockerfiles/resty/http_connect.lua @@ -0,0 +1,341 @@ +local ffi = require "ffi" +local ngx_re_gmatch = ngx.re.gmatch +local ngx_re_sub = ngx.re.sub +local ngx_re_find = ngx.re.find +local ngx_log = ngx.log +local ngx_WARN = ngx.WARN +local ngx_DEBUG = ngx.DEBUG +local to_hex = require("resty.string").to_hex +local ffi_gc = ffi.gc +local ffi_cast = ffi.cast +local type = type + +local lib_chain, lib_x509, lib_pkey +local openssl_available, res = xpcall(function() + lib_chain = require("resty.openssl.x509.chain") + lib_x509 = require("resty.openssl.x509") + lib_pkey = require("resty.openssl.pkey") +end, debug.traceback) + +if not openssl_available then + ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, \z + mTLS isn't supported without lua-resty-openssl:\n", res) +end + +--[[ +A connection function that incorporates: + - tcp connect + - ssl handshake + - http proxy +Due to this it will be better at setting up a socket pool where connections can +be kept alive. + + +Call it with a single options table as follows: + +client:connect { + scheme = "https" -- scheme to use, or nil for unix domain socket + host = "myhost.com", -- target machine, or a unix domain socket + port = nil, -- port on target machine, will default to 80/443 based on scheme + pool = nil, -- connection pool name, leave blank! this function knows best! + pool_size = nil, -- options as per: https://github.com/openresty/lua-nginx-module#tcpsockconnect + backlog = nil, + + -- ssl options as per: https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake + ssl_reused_session = nil + ssl_server_name = nil, + ssl_send_status_req = nil, + ssl_verify = true, -- NOTE: defaults to true + ctx = nil, -- NOTE: not supported + + -- mTLS options: These require support for mTLS in cosockets, which first + -- appeared in `ngx_http_lua_module` v0.10.23. + ssl_client_cert = nil, + ssl_client_priv_key = nil, + + proxy_opts, -- proxy opts, defaults to global proxy options +} +]] +local function connect(self, options) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + local ok, err + + local request_scheme = options.scheme + local request_host = options.host + local request_port = options.port + + local poolname = options.pool + local pool_size = options.pool_size + local backlog = options.backlog + + if request_scheme and not request_port then + request_port = (request_scheme == "https" and 443 or 80) + elseif request_port and not request_scheme then + return nil, "'scheme' is required when providing a port" + end + + -- ssl settings + local ssl, ssl_reused_session, ssl_server_name + local ssl_verify, ssl_send_status_req, ssl_client_cert, ssl_client_priv_key + if request_scheme == "https" then + ssl = true + ssl_reused_session = options.ssl_reused_session + ssl_server_name = options.ssl_server_name + ssl_send_status_req = options.ssl_send_status_req + ssl_verify = true -- default + if options.ssl_verify == false then + ssl_verify = false + end + ssl_client_cert = options.ssl_client_cert + ssl_client_priv_key = options.ssl_client_priv_key + end + + -- proxy related settings + local proxy, proxy_uri, proxy_authorization, proxy_host, proxy_port, path_prefix + proxy = options.proxy_opts or self.proxy_opts + + if proxy then + if request_scheme == "https" then + proxy_uri = proxy.https_proxy + proxy_authorization = proxy.https_proxy_authorization + else + proxy_uri = proxy.http_proxy + proxy_authorization = proxy.http_proxy_authorization + -- When a proxy is used, the target URI must be in absolute-form + -- (RFC 7230, Section 5.3.2.). That is, it must be an absolute URI + -- to the remote resource with the scheme, host and an optional port + -- in place. + -- + -- Since _format_request() constructs the request line by concatenating + -- params.path and params.query together, we need to modify the path + -- to also include the scheme, host and port so that the final form + -- in conformant to RFC 7230. + path_prefix = "http://" .. request_host .. (request_port == 80 and "" or (":" .. request_port)) + end + if not proxy_uri then + proxy = nil + proxy_authorization = nil + path_prefix = nil + end + end + + if proxy and proxy.no_proxy then + -- Check if the no_proxy option matches this host. Implementation adapted + -- from lua-http library (https://github.com/daurnimator/lua-http) + if proxy.no_proxy == "*" then + -- all hosts are excluded + proxy = nil + + else + local host = request_host + local no_proxy_set = {} + -- wget allows domains in no_proxy list to be prefixed by "." + -- e.g. no_proxy=.mit.edu + for host_suffix in ngx_re_gmatch(proxy.no_proxy, "\\.?([^,]+)") do + no_proxy_set[host_suffix[1]] = true + end + + -- From curl docs: + -- matched as either a domain which contains the hostname, or the + -- hostname itself. For example local.com would match local.com, + -- local.com:80, and www.local.com, but not www.notlocal.com. + -- + -- Therefore, we keep stripping subdomains from the host, compare + -- them to the ones in the no_proxy list and continue until we find + -- a match or until there's only the TLD left + repeat + if no_proxy_set[host] then + proxy = nil + proxy_uri = nil + proxy_authorization = nil + break + end + + -- Strip the next level from the domain and check if that one + -- is on the list + host = ngx_re_sub(host, "^[^.]+\\.", "") + until not ngx_re_find(host, "\\.") + end + end + + if proxy then + local proxy_uri_t + proxy_uri_t, err = self:parse_uri(proxy_uri) + if not proxy_uri_t then + return nil, "uri parse error: " .. err + end + + local proxy_scheme = proxy_uri_t[1] + if proxy_scheme ~= "http" then + return nil, "protocol " .. tostring(proxy_scheme) .. + " not supported for proxy connections" + end + proxy_host = proxy_uri_t[2] + proxy_port = proxy_uri_t[3] + end + + local cert_hash + if ssl and ssl_client_cert and ssl_client_priv_key then + local cert_type = type(ssl_client_cert) + local key_type = type(ssl_client_priv_key) + + if cert_type ~= "cdata" then + return nil, "bad ssl_client_cert: cdata expected, got " .. cert_type + end + + if key_type ~= "cdata" then + return nil, "bad ssl_client_priv_key: cdata expected, got " .. key_type + end + + if not openssl_available then + return nil, "module `resty.openssl.*` not available, mTLS isn't supported without lua-resty-openssl" + end + + -- convert from `void*` to `OPENSSL_STACK*` + local cert_chain, err = lib_chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) + if not cert_chain then + return nil, "failed to dup the ssl_client_cert: " .. err + end + + if #cert_chain < 1 then + return nil, "no cert in ssl_client_cert" + end + + local cert, err = lib_x509.dup(cert_chain[1].ctx) + if not cert then + return nil, "failed to dup the x509: " .. err + end + + -- convert from `void*` to `EVP_PKEY*` + local key, err = lib_pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) + if not key then + return nil, "failed to new the pkey: " .. err + end + + -- should not free the cdata passed in + ffi_gc(key.ctx, nil) + + -- check the private key in order to make sure the caller is indeed the holder of the cert + ok, err = cert:check_private_key(key) + if not ok then + return nil, "the private key doesn't match the cert: " .. err + end + + cert_hash, err = cert:digest("sha256") + if not cert_hash then + return nil, "failed to calculate the digest of the cert: " .. err + end + + cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + end + + -- construct a poolname unique within proxy and ssl info + if not poolname then + poolname = (request_scheme or "") + .. ":" .. request_host + .. ":" .. tostring(request_port) + .. ":" .. tostring(ssl) + .. ":" .. (ssl_server_name or "") + .. ":" .. tostring(ssl_verify) + .. ":" .. (proxy_uri or "") + .. ":" .. (request_scheme == "https" and proxy_authorization or "") + .. ":" .. (cert_hash or "") + -- in the above we only add the 'proxy_authorization' as part of the poolname + -- when the request is https. Because in that case the CONNECT request (which + -- carries the authorization header) is part of the connect procedure, whereas + -- with a plain http request the authorization is part of the actual request. + end + + ngx_log(ngx_DEBUG, "poolname: ", poolname) + + -- do TCP level connection + local tcp_opts = { pool = poolname, pool_size = pool_size, backlog = backlog } + if proxy then + -- proxy based connection + ok, err = sock:connect(proxy_host, proxy_port, tcp_opts) + if not ok then + return nil, "failed to connect to: " .. (proxy_host or "") .. + ":" .. (proxy_port or "") .. + ": " .. err + end + + if ssl and sock:getreusedtimes() == 0 then + -- Make a CONNECT request to create a tunnel to the destination through + -- the proxy. The request-target and the Host header must be in the + -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section + -- 4.3.6 for more details about the CONNECT request + local destination = request_host .. ":" .. request_port + local res + res, err = self:request({ + method = "CONNECT", + path = destination, + headers = { + ["Host"] = destination, + ["Proxy-Authorization"] = proxy_authorization, + } + }) + + if not res then + return nil, "failed to issue CONNECT to proxy: " .. err + end + + if res.status < 200 or res.status > 299 then + return nil, "failed to establish a tunnel through a proxy: " .. res.status + end + end + + elseif not request_port then + -- non-proxy, without port -> unix domain socket + ok, err = sock:connect(request_host, tcp_opts) + if not ok then + return nil, err + end + + else + -- non-proxy, regular network tcp + ok, err = sock:connect(request_host, request_port, tcp_opts) + if not ok then + return nil, err + end + end + + local ssl_session + -- Now do the ssl handshake + if ssl and sock:getreusedtimes() == 0 then + + -- Experimental mTLS support + if ssl_client_cert and ssl_client_priv_key then + if type(sock.setclientcert) ~= "function" then + return nil, "cannot use SSL client cert and key without mTLS support" + + else + ok, err = sock:setclientcert(ssl_client_cert, ssl_client_priv_key) + if not ok then + return nil, "could not set client certificate: " .. err + end + end + end + + ssl_session, err = sock:sslhandshake(ssl_reused_session, ssl_server_name, ssl_verify, ssl_send_status_req) + if not ssl_session then + self:close() + return nil, err + end + end + + self.host = request_host + self.port = request_port + self.keepalive = true + self.ssl = ssl + -- set only for http, https has already been handled + self.http_proxy_auth = request_scheme ~= "https" and proxy_authorization or nil + self.path_prefix = path_prefix + + return true, nil, ssl_session +end + +return connect \ No newline at end of file diff --git a/k8s/dockerfiles/resty/http_headers.lua b/k8s/dockerfiles/resty/http_headers.lua new file mode 100644 index 000000000000..35226147b6d1 --- /dev/null +++ b/k8s/dockerfiles/resty/http_headers.lua @@ -0,0 +1,44 @@ +local rawget, rawset, setmetatable = + rawget, rawset, setmetatable + +local str_lower = string.lower + +local _M = { + _VERSION = '0.17.2', +} + + +-- Returns an empty headers table with internalised case normalisation. +function _M.new() + local mt = { + normalised = {}, + } + + mt.__index = function(t, k) + return rawget(t, mt.normalised[str_lower(k)]) + end + + mt.__newindex = function(t, k, v) + local k_normalised = str_lower(k) + + -- First time seeing this header field? + if not mt.normalised[k_normalised] then + -- Create a lowercased entry in the metatable proxy, with the value + -- of the given field case + mt.normalised[k_normalised] = k + + -- Set the header using the given field case + rawset(t, k, v) + else + -- We're being updated just with a different field case. Use the + -- normalised metatable proxy to give us the original key case, and + -- perorm a rawset() to update the value. + rawset(t, mt.normalised[k_normalised], v) + end + end + + return setmetatable({}, mt) +end + + +return _M \ No newline at end of file