feat(interactive): Add master-slave deployment of Interactive

charts/graphscope-interactive/README.md

@@ -13,6 +13,9 @@ $ kubectl describe svc {your-release-name} -graphscope-interactive-frontend | gr
 #192.168.0.44:7687
 # the first is the gremlin endpoint(currently not supported)
 # the second is the cypher endpoint
+$ kubectl describe svc {your-release-name} -graphscope-interactive-engine | grep "Endpoints:" | awk -F' ' '{print $2}'
+# the first is the admin port
+# the second is the query port
 ```
 
 Delete the deployment via 
@@ -105,3 +108,88 @@ hiactorWorkerNum: 1 # currently only support 1.
 hiactorTimeout: 240000
 
 ```
+
+
+## TODO
+
+- TODO: Support cypher/gremlin queries.
+
+## Installtion
+
+```bash
+helm install lei-test -f settings.yaml . --set odps.access.id="",odps.access.key="",odps.endpoint=""
+export NODE_IP=$(ktl -n kubetask get pod lei-test-graphscope-interactive-primary-0 -o jsonpath="{.status.podIP}")
+export ADMIN_PORT=$(ktl get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[0].ports[0].containerPort}')
+export QUERY_PORT=$(ktl get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}')
+export ADMIN_ENDPOINT=${NODE_IP}:${ADMIN_PORT}
+export QUERY_ENDPOINT=${NODE_IP}:${QUERY_PORT}
+echo "ADMIN_ENDPOINT: ${ADMIN_ENDPOINT}"
+echo "QUERY_ENDPOINT: ${QUERY_ENDPOINT}"
+```
+
+```bash
+export NODE_IP=$(118f -n kubetask get pod lei-test-graphscope-interactive-primary-0 -o jsonpath="{.status.podIP}")
+export ADMIN_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[0].ports[0].containerPort}')
+export QUERY_PORT=$(118f get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}')
+```
+
+```bash
+# to verify the helm char
+helm install lei-test --dry-run
+```
+
+
+## add resty.http to nginx images
+
+A customized nginx image
+
+
+## nginx conf
+<!-- # nginx.conf: |  
+#   events {}
+#   http {
+#       server {  
+#           listen 10000;  
+#           server_name localhost;  
+
+#           location / {  
+#               {{- $baseName := include "graphscope-interactive.secondary.fullname" . }}  
+#               {{- $replicaCount := .Values.backend.replicas }}  
+#               {{- $serviceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }}  
+#               {{- $port := .Values.secondary.service.queryPort }}
+#               proxy_pass http://{{ printf "%s-0.%s:%d" $baseName $serviceName $port }};  
+#               {{- range $i := until $replicaCount }}  
+#               mirror /mirror{{ $i }} {
+#                   internal;
+#                   proxy_pass http://{{ printf "%s-%d.%s:%d%s" $baseName (add $i 1) $serviceName $port $request_uri }};
+#               }
+#               {{- end }}
+#               location /mirror {{ printf "%s-%d.%s:%d" $baseName (add $i 1) $serviceName $port }};
+#               {{- end }}
+#               proxy_set_header Host $host;  
+#               proxy_set_header X-Real-IP $remote_addr;  
+#               proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;  
+#               proxy_set_header X-Forwarded-Proto $scheme;  
+#           }
+#       }
+#   } -->
+
+        # - name: admin-nginx
+        #   image: {{ include "graphscope-interactive.nginx.image" . }} 
+        #   imagePullPolicy: {{ .Values.nginx.image.pullPolicy | quote }}
+        #   # command: ["sleep", "infinity"]
+        #   ports:
+        #     - name: admin-port 
+        #       containerPort: {{ .Values.frontend.service.adminPort }}
+        #   {{- if .Values.resources.frontend }}
+        #   resources: {{- toYaml .Values.resources.frontend | nindent 12 }}
+        #   {{- end }}
+        #   volumeMounts:
+        #     - name: workspace
+        #       mountPath: {{ .Values.workspace }}
+        #     - name: config
+        #       mountPath: {{ include "graphscope-interactive.engineConfigPath" . }}
+        #       subPath: engine_config.yaml
+        #     - name: admin-nginx-config  
+        #       mountPath: /etc/nginx/nginx.conf  
+        #       subPath: nginx.conf

charts/graphscope-interactive/templates/_helpers.tpl

@@ -27,8 +27,12 @@ If release name contains chart name it will be used as a full name.
 {{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "frontend" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
-{{- define "graphscope-interactive.engine.fullname" -}}
-{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "engine" | trunc 63 | trimSuffix "-" -}}
+{{- define "graphscope-interactive.primary.fullname" -}}
+{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "primary" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "graphscope-interactive.secondary.fullname" -}}
+{{- printf "%s-%s" (include "graphscope-interactive.fullname" .) "secondary" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
 
@@ -77,11 +81,46 @@ Return the proper graphscope-interactive frontend image name
 {{- end -}}
 
 {{/*
-Return the proper graphscope-interactive engine image name
+Return the proper graphscope-interactive primary image name
+*/}}
+{{- define "graphscope-interactive.primary.image" -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- with .Values.primary.image -}}
+{{- if .tag -}}
+{{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if .registry -}}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper graphscope-interactive primary image name
+*/}}
+{{- define "graphscope-interactive.nginx.image" -}}
+{{- $tag := .Chart.AppVersion | toString -}}
+{{- with .Values.nginx.image -}}
+{{- if .tag -}}
+{{- $tag = .tag | toString -}}
+{{- end -}}
+{{- if .registry -}}
+{{- printf "%s/%s:%s" .registry .repository $tag -}}
+{{- else -}}
+{{- printf "%s:%s" .repository $tag -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+
+{{/*
+Return the proper graphscope-interactive secondary image name
 */}}
-{{- define "graphscope-interactive.engine.image" -}}
+{{- define "graphscope-interactive.secondary.image" -}}
 {{- $tag := .Chart.AppVersion | toString -}}
-{{- with .Values.engine.image -}}
+{{- with .Values.secondary.image -}}
 {{- if .tag -}}
 {{- $tag = .tag | toString -}}
 {{- end -}}
@@ -159,6 +198,21 @@ Return the engineConfigPath with the graphscope configuration
 /etc/interactive/interactive_config.yaml
 {{- end -}}
 
+
+{{/*
+Primary service name.
+*/}}
+{{- define "graphscope-interactive.primary.serviceName" -}}
+{{- printf "%s.%s.svc.%s" (include "graphscope-interactive.primary.fullname" .) .Release.Namespace .Values.clusterDomain }} 
+{{- end -}}
+
+{{/*
+From where a http client could fetch the schema uri
+*/}}
+{{- define "graphscope-interactive.graphSchemaUri" -}}
+{{- printf "http://%s-0.%s:%d" (include "graphscope-interactive.fullname" .) (include "graphscope-interactive.primary.serviceName" .)  .Values.primary.service.adminPort | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Return the realEngineConfigPath with the graphscope configuration, templated by frontend
 */}}

charts/graphscope-interactive/templates/admin_nginx_conf.yaml

@@ -0,0 +1,159 @@
+apiVersion: v1  
+kind: ConfigMap  
+metadata:  
+  name: {{ .Release.Name }}-admin-nginx-config  
+  namespace: {{ .Release.Namespace }}
+  labels: {{- include "graphscope-interactive.labels" . | nindent 4 }}
+    app.kubernetes.io/component: configmap
+    {{- if .Values.commonLabels }}
+    {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+data:  
+  nginx.conf: |  
+    events {}
+    http {
+        resolver local=on valid=5s;
+        server {  
+            {{- $adminPort := .Values.primary.service.adminPort | int }}
+            listen {{ $adminPort }};  
+            client_body_buffer_size     10M;
+            client_max_body_size        10M;
+            location / {  
+                {{- $primaryBaseName := include "graphscope-interactive.primary.fullname" . }}  
+                {{- $secondaryBaseName := include "graphscope-interactive.secondary.fullname" . }}  
+                {{- $replicaCount := .Values.secondary.replicaCount | int }}  
+                {{- $primaryServiceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.primary.fullname" .) .Release.Namespace .Values.clusterDomain }}
+                {{- $secondaryServiceName := printf "%s.%s.svc.%s" (include "graphscope-interactive.secondary.fullname" .) .Release.Namespace .Values.clusterDomain }}  
+                {{- $port := .Values.secondary.service.adminPort | int }}
+                proxy_pass {{ printf "http://%s-0.%s:%d" $primaryBaseName $primaryServiceName $port | quote }};
+                content_by_lua_block {  
+                    function arrayToString(arr, separator)  
+                        separator = separator or ", " -- Default separator if not provided  
+                        return table.concat(arr, separator)  
+                    end 
+                    function send_request(http, full_uri, method, body_data, headers)  
+                        local httpc = http.new()  
+                        if method == "GET" or method == "DELETE" then  
+                            return httpc:request_uri(full_uri, {  
+                                method = method,  
+                            })  
+                        elseif method == "POST" or method == "PUT" then
+                            return httpc:request_uri(full_uri, {  
+                                method = method,  
+                                body = body_data,  
+                                headers = headers
+                            }) 
+                        else
+                            ngx.log(ngx.ERR, " not recognized method ", method)
+                        end  
+                    end  
+                    local http = require "resty.http" 
+                    local res = {}
+                    local status_codes = {} 
+                    local error_message = nil  -- Initialize a variable to capture error messages
+
+                    local urls = {  
+                        {{ printf "http://%s-0.%s:%d" $primaryBaseName $primaryServiceName $port | quote }};
+                        {{- if eq $replicaCount 1 }}
+                        {{ printf "http://%s-0.%s:%d" $secondaryBaseName $secondaryServiceName $port | quote }}
+                        {{- else }}
+                        {{- range $i := until (sub $replicaCount 1 | int )  }}  
+                        {{ printf "\"http://%s-%d.%s:%d\"," $secondaryBaseName $i $secondaryServiceName $port  }}
+                        {{- end }}
+                        {{ printf "http://%s-%d.%s:%d" $secondaryBaseName (sub $replicaCount 1) $secondaryServiceName $port | quote }}
+                        {{- end }}
+                    }
+
+                    local original_headers = ngx.req.get_headers()  
+                    local request_uri=ngx.var.request_uri
+                    local method = ngx.req.get_method()
+
+                    -- Create a table for modified headers  
+                    local backend_headers = {}  
+
+                    -- Copy the relevant headers, if needed, or modify them  
+                    for key, value in pairs(original_headers) do  
+                        -- You can filter headers if needed (e.g., skip "host" or "authorization")  
+                        if key ~= "Host" and key ~= "User-Agent" and key ~= "Content-Length" then  
+                            backend_headers[key] = value  
+                        end
+                    end
+
+
+                    ngx.req.read_body()  -- Read the request body 
+
+                    -- resize status_codes to the number of replicas
+                    for i = 1, #urls do
+                        status_codes[i] = 0
+                        res[i] = ""
+                    end
+
+                    local threads = {}
+                    local body_data = ngx.req.get_body_data() 
+                    for index, backend in ipairs(urls) do 
+                        -- full_uri is backend + request_uri
+                        local full_uri = backend .. request_uri 
+                        threads[index] = ngx.thread.spawn(function() 
+                            local response, err = send_request(http, full_uri, method, body_data, backend_headers)  
+                            local status_code = 0
+                            if response ~= nil then
+                                status_code = response.status
+                                res[index] = response.body
+                                if response.status < 200 or response.status >= 300 then
+                                    if not error_message then  -- Capture the error message from the first failed request
+                                        error_message = response.body or "Failed request without a body."
+                                    end
+                                end
+                            else
+                                status_code = 500
+                                if err ~= nil then
+                                    ngx.log(ngx.ERR, "Failed to request: ", err)
+                                    if not error_message then  -- Capture error when no response
+                                        error_message = "Error: " .. err
+                                    end
+                                else
+                                    error_message = "Not found"
+                                end
+                            end 
+                            status_codes[index] = status_code
+                        end)
+                    end
+
+                    for _, thread in ipairs(threads) do  
+                        coroutine.resume(thread)  
+                    end
+
+                    for _, thread in ipairs(threads) do
+                        ngx.thread.wait(thread)
+                    end 
+
+                    local success = true
+                    local final_status_code = 200
+                    for i = 1, #urls do
+                        if status_codes[i] < 200 or status_codes[i] >= 300 then
+                            ngx.log(ngx.ERR, "Failed to request: ", urls[i], " with status code: ", status_codes[i], " and response: ", res[i], " index: ", i)
+                            success = false
+                            final_status_code = status_codes[i]
+                            break
+                        end
+                    end
+
+                    ngx.header.content_type = 'application/json'
+                    if success then  
+                        ngx.status = final_status_code
+                        ngx.log(ngx.INFO, "Success: ", arrayToString(res, ", "), " with status code: ", final_status_code)
+                        ngx.say(res[1])
+                        ngx.exit(final_status_code)
+                    else 
+                        ngx.status = final_status_code
+                        ngx.log(ngx.ERR, "Failed to request: ", error_message, " with status code: ", final_status_code)
+                        ngx.say(error_message)
+                        ngx.exit(final_status_code)
+                    end
+                }
+            }  
+        }  
+    }

charts/graphscope-interactive/templates/configmap.yaml

@@ -13,13 +13,13 @@ metadata:
   {{- end }}
 data:
   interactive_config.yaml: |-
-    log_level: {{ .Values.engine.logLevel  }}
+    log_level: {{ .Values.logLevel  }}
     default_graph: {{ .Values.defaultGraph }}
     compute_engine:
       type: hiactor
       workers:
-        - ENGINE_SERVICE_HOST:10000
-      thread_num_per_worker: {{ .Values.engine.threadNumPerWorker }}
+        - localhost:10000
+      thread_num_per_worker: {{ .Values.primary.threadNumPerWorker }}
     compiler:
       planner:
         is_on: true
@@ -29,7 +29,7 @@ data:
           - FilterMatchRule
           - NotMatchToAntiJoinRule
       endpoint:
-        default_listen_address: ENGINE_SERVICE_HOST
+        default_listen_address: localhost
         bolt_connector:
           disabled: false
           port: {{ .Values.frontend.service.cypherPort }}
@@ -38,10 +38,6 @@ data:
           port: {{ .Values.frontend.service.gremlinPort  }}
       query_timeout: {{ .Values.frontend.service.queryTimeout  }}
     http_service:
-      default_listen_address: ENGINE_SERVICE_HOST
-      admin_port: {{ .Values.engine.service.adminPort  }}
-      query_port: {{ .Values.engine.service.queryPort  }}
-  setup.sh: |-
-    #!/bin/bash
-    sudo sed -e "s/ENGINE_SERVICE_HOST/${ENGINE_SERVICE_HOST}/g" ${ENGINE_CONFIG_PATH} > ${REAL_ENGINE_CONFIG_PATH}
-    echo "Finish set ENGINE_SERVICE_HOST to ${ENGINE_SERVICE_HOST}"
+      default_listen_address: localhost
+      admin_port: {{ .Values.primary.service.adminPort  }}
+      query_port: {{ .Values.primary.service.queryPort  }}