Skip to content

Commit

Permalink
introduce proxy_server
Browse files Browse the repository at this point in the history
minor

code refactor

refine proxy server

todo: bind to pvc and deploy on internal

try to test on internal

modify dockerfile

able to launch, todo: fix graph creation and loading
  • Loading branch information
zhanglei1949 committed May 23, 2024
1 parent ae98086 commit 5cfd6bd
Show file tree
Hide file tree
Showing 44 changed files with 2,419 additions and 278 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/hqps-db-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,13 @@ jobs:
bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml python
sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml
- name: Proxy Server test
env:
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
run: |
cd ${GITHUB_WORKSPACE}/flex/tests/hqps
bash hqps_proxy_server_test.sh ${INTERACTIVE_WORKSPACE} ./engine_config_test.yaml
- name: Sample Query test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
Expand Down
23 changes: 23 additions & 0 deletions charts/graphscope-interactive/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## QuickStart

```bash
helm install release -f ./charts/graphscope-interactive/nginx_ingress_custom.yaml oci://ghcr.io/nginxinc/charts/nginx-ingress --version 1.2.1
$ helm install {your-release-name} .
```

Expand All @@ -13,6 +14,9 @@ $ kubectl describe svc {your-release-name} -graphscope-interactive-frontend | gr
#192.168.0.44:7687
# the first is the gremlin endpoint(currently not supported)
# the second is the cypher endpoint
$ kubectl describe svc {your-release-name} -graphscope-interactive-engine | grep "Endpoints:" | awk -F' ' '{print $2}'
# the first is the admin port
# the second is the query port
```

Delete the deployment via
Expand Down Expand Up @@ -105,3 +109,22 @@ hiactorWorkerNum: 1 # currently only support 1.
hiactorTimeout: 240000

```
## TODO
- Currently we don't contains AdminService into deployment, either cypher/gremlin SDK.
## Installtion
```bash
helm install lei-test -f settings.yaml . --set odps.access.id="",odps.access.key="",odps.endpoint=""
export NODE_IP=$(ktl -n kubetask get pod lei-test-graphscope-interactive-primary-0 -o jsonpath="{.status.podIP}")
export ADMIN_PORT=$(ktl get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[0].ports[0].containerPort}')
export QUERY_PORT=$(ktl get pod lei-test-graphscope-interactive-primary-0 -ojsonpath='{.spec.containers[1].ports[0].containerPort}')
export ADMIN_ENDPOINT=${NODE_IP}:${ADMIN_PORT}
export QUERY_ENDPOINT=${NODE_IP}:${QUERY_PORT}
echo "ADMIN_ENDPOINT: ${ADMIN_ENDPOINT}"
echo "QUERY_ENDPOINT: ${QUERY_ENDPOINT}"
```
25 changes: 25 additions & 0 deletions charts/graphscope-interactive/backup/admin_virtual_server.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
apiVersion: k8s.nginx.org/v1
kind: VirtualServer
metadata:
name: admin-service
namespace: {{ .Release.Namespace }}
labels: {{- include "graphscope-interactive.labels" . | nindent 4 }}
app.kubernetes.io/component: virtual-server
{{- if .Values.commonLabels }}
{{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- end }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
host: admin.interactive.com
listener:
http: http-7777
upstreams:
- name: interactive-admin
service: {{ include "graphscope-interactive.primary.fullname" . }}
port: {{ .Values.primary.service.adminPort }}
routes:
- path: /
action:
pass: interactive-admin
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{{- $frontendFullname := include "graphscope-interactive.frontend.fullname" . }}
{{- $engineFullName := include "graphscope-interactive.engine.fullname" . }}
{{- $primaryFullName := include "graphscope-interactive.primary.fullname" . }}
{{- $secondaryFullName := include "graphscope-interactive.secondary.fullname" . }}
{{- $releaseNamespace := .Release.Namespace }}
{{- $clusterDomain := .Values.clusterDomain }}

Expand Down Expand Up @@ -35,13 +36,33 @@ spec:
{{- end }}
labels: {{- include "graphscope-interactive.labels" . | nindent 8 }}
app.kubernetes.io/component: frontend
{{- if .Values.commonLabels }}
{{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 8 }}
{{- if .Values.frontend.podLabels }}
{{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.frontend.podLabels "context" $) | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
{{- if .Values.imagePullSecrets }}
imagePullSecrets: {{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.frontend.hostAliases }}
hostAliases: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.frontend.hostAliases "context" $) | nindent 8 }}
{{- end }}
hostNetwork: {{ .Values.frontend.hostNetwork }}
hostIPC: {{ .Values.frontend.hostIPC }}
{{- if .Values.frontend.schedulerName }}
schedulerName: {{ .Values.frontend.schedulerName | quote }}
{{- end }}
{{- if .Values.nodeSelector }}
nodeSelector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.dnsPolicy }}
dnsPolicy: {{ .Values.dnsPolicy | quote }}
{{- end }}
{{- if .Values.dnsConfig }}
dnsConfig: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.dnsConfig "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.tolerations }}
tolerations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "graphscope-interactive.serviceAccountName" . }}
{{- if .Values.frontend.affinity }}
affinity: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.frontend.affinity "context" $) | nindent 8 }}
Expand Down Expand Up @@ -84,11 +105,12 @@ spec:
cmd="java -cp \"${COMPILER_CLASS_PATH}\" -Djna.library.path=${COMPILER_LIBRARY_PATH} "
cmd="${cmd} -Dgraph.schema=${GRAPH_SCHEMA_PATH} "
cmd="${cmd} com.alibaba.graphscope.GraphServer ${REAL_ENGINE_CONFIG_PATH}"
# TODO: The real engine config path should point to the virtual server endpoint.
echo "Start compiler with command: ${cmd}"
eval ${cmd}
env:
- name: ENGINE_SERVICE_HOST
value: {{ $engineFullName }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}
# - name: ENGINE_SERVICE_HOST
# value: {{ $engineFullName }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}
- name: TIMEOUT
value: {{ .Values.hiactorTimeout | quote }}
- name: ENGINE_CONFIG_PATH
Expand Down Expand Up @@ -133,33 +155,33 @@ spec:
configMap:
name: {{ include "graphscope-interactive.configmapName" . }}
defaultMode: 0755
{{- if and .Values.engine.persistence.enabled .Values.engine.persistence.existingClaim }}
{{- if and .Values.primary.persistence.enabled .Values.primary.persistence.existingClaim }}
- name: workspace
persistentVolumeClaim:
claimName: {{ tpl .Values.engine.persistence.existingClaim . }}
{{- else if not .Values.engine.persistence.enabled }}
claimName: {{ tpl .Values.primary.persistence.existingClaim . }}
{{- else if not .Values.primary.persistence.enabled }}
- name: workspace
emptyDir: {}
{{- else if and .Values.engine.persistence.enabled (not .Values.engine.persistence.existingClaim) }}
{{- else if and .Values.primary.persistence.enabled (not .Values.primary.persistence.existingClaim) }}
volumeClaimTemplates:
- metadata:
name: workspace
{{- if .Values.persistence.annotations }}
annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
annotations: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
{{- end }}
{{- if .Values.persistence.labels }}
labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
labels: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
{{- end }}
spec:
accessModes:
{{- range .Values.engine.persistence.accessModes }}
{{- range .Values.primary.persistence.accessModes }}
- {{ . | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.engine.persistence.size | quote }}
storage: {{ .Values.primary.persistence.size | quote }}
{{ include "graphscope-interactive.storageClass" . | nindent 8 }}
{{- if .Values.engine.persistence.selector }}
selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.engine.persistence.selector "context" $) | nindent 10 }}
{{- if .Values.primary.persistence.selector }}
selector: {{- include "graphscope-interactive.tplvalues.render" (dict "value" .Values.primary.persistence.selector "context" $) | nindent 10 }}
{{- end -}}
{{- end }}
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ metadata:
labels: {{- include "graphscope-interactive.labels" . | nindent 4 }}
app.kubernetes.io/component: frontend
{{- if .Values.commonLabels }}
{{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- end }}
annotations:
{{- if .Values.commonAnnotations }}
{{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.frontend.service.type }}
Expand All @@ -28,7 +28,7 @@ spec:
- name: gremlin
port: {{ .Values.frontend.service.gremlinPort }}
protocol: TCP
targetPort: gremlin
targetPort: {{ .Values.frontend.service.gremlinPort }}
{{- if and (or (eq .Values.frontend.service.type "NodePort") (eq .Values.frontend.service.type "LoadBalancer")) (not (empty .Values.frontend.service.nodePorts.gremlin)) }}
{{- if (not (empty .Values.frontend.service.nodePorts.gremlin)) }}
nodePort: {{ .Values.frontend.service.nodePorts.gremlin }}
Expand All @@ -39,7 +39,7 @@ spec:
- name: cypher
port: {{ .Values.frontend.service.cypherPort }}
protocol: TCP
targetPort: cypher
targetPort: {{ .Values.frontend.service.cypherPort }}
{{- if and (or (eq .Values.frontend.service.type "NodePort") (eq .Values.frontend.service.type "LoadBalancer")) (not (empty .Values.frontend.service.nodePorts.cypher)) }}
{{- if (not (empty .Values.frontend.service.nodePorts.cypher)) }}
nodePort: {{ .Values.frontend.service.nodePorts.cypher }}
Expand Down
19 changes: 19 additions & 0 deletions charts/graphscope-interactive/backup/ingress_test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: http-forwarder
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/rewrite-target: / # 如果需要重写URL,否则可以省略
spec:
rules:
- host: your.domain.com # 替换为你实际的域名
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: target-service
port:
number: 10000
17 changes: 17 additions & 0 deletions charts/graphscope-interactive/backup/nginx_ingress_custom.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
controller:
globalConfiguration:
## Creates the GlobalConfiguration custom resource. Requires controller.enableCustomResources.
create: true

## The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller.
spec: ## Ensure both curly brackets are removed when adding listeners in YAML format.
listeners:
- name: http-10000
port: 10000
protocol: HTTP
service:
customPorts:
- name: http-10000
port: 10000
protocol: TCP
targetPort: 10000
49 changes: 49 additions & 0 deletions charts/graphscope-interactive/backup/query_virtual_server.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
apiVersion: k8s.nginx.org/v1
kind: VirtualServer
metadata:
name: query-service
namespace: {{ .Release.Namespace }}
labels: {{- include "graphscope-interactive.labels" . | nindent 4 }}
app.kubernetes.io/component: virtual-server
{{- if .Values.commonLabels }}
{{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- end }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "graphscope-interactive.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
host: query.interactive.com
listener:
http: http-10000
upstreams:
- name: interactive-post
service: {{ include "graphscope-interactive.primary.fullname" . }}
port: {{ .Values.primary.service.queryPort }}
- name: interactive-get
service: {{ include "graphscope-interactive.secondary.fullname" . }}
port: {{ .Values.secondary.service.queryPort }}
routes:
- path: /
matches:
- conditions:
- variable: $request_method
value: GET
action:
pass: interactive-get
- conditions:
- variable: $request_method
value: POST
action:
pass: interactive-post
- conditions:
- variable: $request_method
value: PUT
action:
pass: interactive-post
- conditions:
- variable: $request_method
value: DELETE
action:
pass: interactive-post
action:
pass: interactive-get
77 changes: 77 additions & 0 deletions charts/graphscope-interactive/settings.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
Kata: &kata
#hostNetwork: true
schedulerName: unified-batch-scheduler
podAnnotations:
alibabacloud.com/assign-in-virtual-node: odps-kata
alibabacloud.com/skip-kubelet-admission: "[\"cpu\",\"memory\",\"alibabacloud.com/acu\",\"pods\"]"
podLabels: &podlabels
alibabacloud/qos: LS
alibabacloud.com/quota-name: yuansi-child-quotaxy

NoKata: &nokata
hostNetwork: false
schedulerName: default-scheduler
podAnnotations:
#alibabacloud.com/assign-in-virtual-node: odps-kata
alibabacloud.com/skip-kubelet-admission: "[\"cpu\",\"memory\",\"alibabacloud.com/acu\",\"pods\"]"
podLabels:
alibabacloud/qos: LS
#alibabacloud.com/quota-name: yuansi-child-quotaxy

Defaults: &defaults
dnsPolicy: &dnspolicy None
dnsConfig: &dnsconfig
nameservers: ["11.163.105.32"]
searches:
- kubetask.svc.cluster.local
- svc.cluster.local
- cluster.local
options:
- name: ndots
value: "2"

tolerations: &toleration
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
- effect: NoSchedule
key: sigma.ali/resource-pool
operator: Exists
#value: ackee_pool
- effect: NoSchedule
key: sigma.ali/is-ecs
operator: Exists
- effect: NoSchedule
key: alibabacloud.com/partition
operator: Exists
- effect: NoSchedule
key: sigma.ali/server-owner
operator: Exists

resources: &resource_min
limits:
cpu: 1000m
memory: 1Gi
requests:
cpu: 1000m
memory: 1Gi

global:
#imageRegistry: registry.cn-hongkong.aliyuncs.com
storageClass: csi-ultron-prjquota

<<: *defaults

primary:
<<: *nokata

secondary:
<<: *nokata

frontend:
<<: *nokata
Loading

0 comments on commit 5cfd6bd

Please sign in to comment.