Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement vineyard runtime engine and controller #3649

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ GOOSEFSRUNTIME_CONTROLLER_IMG ?= ${IMG_REPO}/goosefsruntime-controller
JUICEFSRUNTIME_CONTROLLER_IMG ?= ${IMG_REPO}/juicefsruntime-controller
THINRUNTIME_CONTROLLER_IMG ?= ${IMG_REPO}/thinruntime-controller
EFCRUNTIME_CONTROLLER_IMG ?= ${IMG_REPO}/efcruntime-controller
VINEYARDRUNTIME_CONTROLLER_IMG ?= ${IMG_REPO}/vineyardruntime-controller
CSI_IMG ?= ${IMG_REPO}/fluid-csi
LOADER_IMG ?= ${IMG_REPO}/fluid-dataloader
INIT_USERS_IMG ?= ${IMG_REPO}/init-users
Expand Down Expand Up @@ -54,6 +55,7 @@ BINARY_BUILD += jindoruntime-controller-build
BINARY_BUILD += juicefsruntime-controller-build
BINARY_BUILD += thinruntime-controller-build
BINARY_BUILD += efcruntime-controller-build
BINARY_BUILD += vineyardruntime-controller-build
BINARY_BUILD += csi-build
BINARY_BUILD += webhook-build

Expand All @@ -68,6 +70,7 @@ DOCKER_BUILD += docker-build-webhook
DOCKER_BUILD += docker-build-juicefsruntime-controller
DOCKER_BUILD += docker-build-thinruntime-controller
DOCKER_BUILD += docker-build-efcruntime-controller
DOCKER_BUILD += docker-build-vineyardruntime-controller
DOCKER_BUILD += docker-build-init-users
DOCKER_BUILD += docker-build-crd-upgrader

Expand All @@ -82,6 +85,7 @@ DOCKER_PUSH += docker-push-goosefsruntime-controller
DOCKER_PUSH += docker-push-juicefsruntime-controller
DOCKER_PUSH += docker-push-thinruntime-controller
DOCKER_PUSH += docker-push-efcruntime-controller
DOCKER_PUSH += docker-push-vineyardruntime-controller
# Not need to push init-users image by default
# DOCKER_PUSH += docker-push-init-users
DOCKER_PUSH += docker-push-crd-upgrader
Expand Down Expand Up @@ -174,6 +178,9 @@ juicefsruntime-controller-build:
thinruntime-controller-build:
CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=${GO_MODULE} go build ${GC_FLAGS} -a -o bin/thinruntime-controller -ldflags '-s -w ${LDFLAGS}' cmd/thin/main.go

vineyardruntime-controller-build:
CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=${GO_MODULE} go build ${GC_FLAGS} -a -o bin/vineyardruntime-controller -ldflags '-s -w ${LDFLAGS}' cmd/vineyard/main.go

efcruntime-controller-build:
CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=${GO_MODULE} go build ${GC_FLAGS} -a -o bin/efcruntime-controller -ldflags '${LDFLAGS}' cmd/efc/main.go

Expand Down Expand Up @@ -208,6 +215,9 @@ docker-build-thinruntime-controller:
docker-build-efcruntime-controller:
docker build --no-cache --build-arg TARGETARCH=${ARCH} . -f docker/Dockerfile.efcruntime -t ${EFCRUNTIME_CONTROLLER_IMG}:${GIT_VERSION}

docker-build-vineyardruntime-controller:
docker build --no-cache --build-arg TARGETARCH=${ARCH} . -f docker/Dockerfile.vineyardruntime -t ${VINEYARDRUNTIME_CONTROLLER_IMG}:${GIT_VERSION}

docker-build-csi:
docker build --no-cache . -f docker/Dockerfile.csi -t ${CSI_IMG}:${GIT_VERSION}

Expand Down Expand Up @@ -245,6 +255,9 @@ docker-push-thinruntime-controller: docker-build-thinruntime-controller
docker-push-efcruntime-controller: docker-build-efcruntime-controller
docker push ${EFCRUNTIME_CONTROLLER_IMG}:${GIT_VERSION}

docker-push-vineyardruntime-controller: docker-build-vineyardruntime-controller
docker push ${VINEYARDRUNTIME_CONTROLLER_IMG}:${GIT_VERSION}

docker-push-csi: docker-build-csi
docker push ${CSI_IMG}:${GIT_VERSION}

Expand Down Expand Up @@ -285,6 +298,9 @@ docker-buildx-push-thinruntime-controller:
docker-buildx-push-efcruntime-controller:
docker buildx build --push --platform linux/amd64,linux/arm64 --no-cache . -f docker/Dockerfile.efcruntime -t ${EFCRUNTIME_CONTROLLER_IMG}:${GIT_VERSION}

docker-buildx-push-vineyardruntime-controller:
docker buildx build --push --platform linux/amd64,linux/arm64 --no-cache . -f docker/Dockerfile.vineyardruntime -t ${VINAYARDRUNTIME_CONTROLLER_IMG}:${GIT_VERSION}

docker-buildx-push-csi: generate fmt vet
docker buildx build --push --platform linux/amd64,linux/arm64 --no-cache . -f docker/Dockerfile.csi -t ${CSI_IMG}:${GIT_VERSION}

Expand Down
6 changes: 3 additions & 3 deletions api/v1alpha1/openapi_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions api/v1alpha1/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -2505,7 +2505,7 @@
}
},
"ports": {
"description": "Ports used by Vineyard component. For Master, the default client port is 2379 and peer port is 2380. For Worker, the default rpc port is 9600.",
"description": "Ports used by Vineyard component. For Master, the default client port is 2379 and peer port is 2380. For Worker, the default rpc port is 9600 and the default exporter port is 9144.",
"type": "object",
"additionalProperties": {
"type": "integer",
Expand Down Expand Up @@ -3614,7 +3614,7 @@
}
},
"ports": {
"description": "Ports used by Vineyard component. For Master, the default client port is 2379 and peer port is 2380. For Worker, the default rpc port is 9600.",
"description": "Ports used by Vineyard component. For Master, the default client port is 2379 and peer port is 2380. For Worker, the default rpc port is 9600 and the default exporter port is 9144.",
"type": "object",
"additionalProperties": {
"type": "integer",
Expand Down Expand Up @@ -3743,7 +3743,7 @@
"type": "string"
},
"image": {
"description": "Image for Vineyard Fuse Default is `vineyardcloudnative/vineyard-mount-socket`",
"description": "Image for Vineyard Fuse Default is `vineyardcloudnative/vineyard-fluid-fuse`",
"type": "string"
},
"imagePullPolicy": {
Expand Down
13 changes: 11 additions & 2 deletions api/v1alpha1/vineyardruntime_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@

// Ports used by Vineyard component.
// For Master, the default client port is 2379 and peer port is 2380.
// For Worker, the default rpc port is 9600.
// For Worker, the default rpc port is 9600 and the default exporter port is 9144.
// +optional
Ports map[string]int `json:"ports,omitempty"`

Expand Down Expand Up @@ -142,7 +142,7 @@
// VineyardSockSpec holds the configurations for vineyard client socket
type VineyardSockSpec struct {
// Image for Vineyard Fuse
// Default is `vineyardcloudnative/vineyard-mount-socket`
// Default is `vineyardcloudnative/vineyard-fluid-fuse`
// +optional
Image string `json:"image,omitempty"`

Expand Down Expand Up @@ -269,3 +269,12 @@
func init() {
SchemeBuilder.Register(&VineyardRuntime{}, &VineyardRuntimeList{})
}

// Replicas gets the replicas of runtime worker
func (runtime *VineyardRuntime) Replicas() int32 {
return runtime.Spec.Worker.Replicas

Check warning on line 275 in api/v1alpha1/vineyardruntime_types.go

View check run for this annotation

Codecov / codecov/patch

api/v1alpha1/vineyardruntime_types.go#L274-L275

Added lines #L274 - L275 were not covered by tests
}

func (runtime *VineyardRuntime) GetStatus() *RuntimeStatus {
return &runtime.Status

Check warning on line 279 in api/v1alpha1/vineyardruntime_types.go

View check run for this annotation

Codecov / codecov/patch

api/v1alpha1/vineyardruntime_types.go#L278-L279

Added lines #L278 - L279 were not covered by tests
}
8 changes: 5 additions & 3 deletions charts/fluid/fluid/crds/data.fluid.io_vineyardruntimes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ spec:
runtime is deleted Defaults to OnRuntimeDeleted'
type: string
image:
description: Image for Vineyard Fuse Default is `vineyardcloudnative/vineyard-mount-socket`
description: Image for Vineyard Fuse Default is `vineyardcloudnative/vineyard-fluid-fuse`
type: string
imagePullPolicy:
description: Image pull policy for Vineyard Fuse Default is `IfNotPresent`
Expand Down Expand Up @@ -234,7 +234,8 @@ spec:
type: integer
description: Ports used by Vineyard component. For Master, the
default client port is 2379 and peer port is 2380. For Worker,
the default rpc port is 9600.
the default rpc port is 9600 and the default exporter port is
9144.
type: object
replicas:
description: The replicas of Vineyard component. If not specified,
Expand Down Expand Up @@ -3516,7 +3517,8 @@ spec:
type: integer
description: Ports used by Vineyard component. For Master, the
default client port is 2379 and peer port is 2380. For Worker,
the default rpc port is 9600.
the default rpc port is 9600 and the default exporter port is
9144.
type: object
replicas:
description: The replicas of Vineyard component. If not specified,
Expand Down
2 changes: 1 addition & 1 deletion charts/fluid/fluid/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ runtime:
- operator: Exists
enabled: false
controller:
image: fluidcloudnative/vineyardruntime-controller:v1.0.0-edc69cf
image: fluidcloudnative/vineyardruntime-controller:v1.0.0-7221cf3
imagePullPolicy: Always

webhook:
Expand Down
3 changes: 1 addition & 2 deletions charts/vineyard/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
apiVersion: v1
appVersion: 0.18.2
version: 0.18.2
version: 0.19.3
description: an In-Memory Data Manager for Data-Intensive Analytics
home: https://v6d.io/
keywords:
Expand Down
42 changes: 29 additions & 13 deletions charts/vineyard/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -159,26 +159,41 @@ resources:
{{- end }}
{{- end -}}


{{/*
Generate etcd endpoints for peer or client
*/}}
{{- define "vineyard.etcdEndpoints" -}}
{{- $replicas := int .Values.master.replicas }}
{{- $fullName := include "vineyard.fullname" . }}
{{- $etcdFullname := (printf "%s-%s" $fullName "master") }}
{{- $releaseNamespace := .Release.Namespace }}
{{- $etcdServiceName := (printf "%s-%s" $fullName "master-svc") }}
{{- $etcdEndpoint := list }}
{{- $portType := .portType }}
{{- $port := 0 }}
{{- if eq $portType "peer" }}
{{- $port = int .Values.master.ports.peer }}
{{- range $e, $i := until $replicas }}
{{- $etcdEndpoint = append $etcdEndpoint (printf "%s-%d=http://%s-%d.%s.%s:%d" $etcdFullname $i $etcdFullname $i $etcdServiceName $releaseNamespace $port) }}
{{- end }}
{{- else if eq $portType "client" }}
{{- $port = int .Values.master.ports.client }}
{{- range $e, $i := until $replicas }}
{{- $etcdEndpoint = append $etcdEndpoint (printf "http://%s-%d.%s.%s:%d" $etcdFullname $i $etcdServiceName $releaseNamespace $port) }}
{{- end }}
{{- end }}
{{- join "," $etcdEndpoint }}
{{- end }}

{{/*
Get the master endpoint.
*/}}
{{- define "vineyard.master.endpoint" -}}
{{- if .Values.worker.externalEndpoint -}}
{{- printf "http://%s" .Values.worker.externalEndpoint.uri }}
{{- else -}}
{{- $replicas := int .Values.master.replicas }}
{{- $clientPort := int .Values.master.ports.client }}
{{- $name := include "vineyard.name" . }}
{{- $fullName := include "vineyard.fullname" . }}
{{- $etcdFullname := (printf "%s-%s" $fullName "master") }}
{{- $chart := include "vineyard.chart" . }}
{{- $releaseNamespace := .Release.Namespace }}
{{- $etcdServiceName := (printf "%s-%s" $fullName "master-svc") }}
{{- $etcdEndpoint := list }}
{{- range $e, $i := until $replicas }}
{{- $etcdEndpoint = append $etcdEndpoint (printf "http://%s-%d.%s.%s.svc.cluster.local:%d" $etcdFullname $i $etcdServiceName $releaseNamespace $clientPort) }}
{{- end }}
{{- join "," $etcdEndpoint }}
{{ include "vineyard.etcdEndpoints" (dict "Values" .Values "Release" .Release "portType" "client") }}
{{- end -}}
{{- end -}}

Expand Down Expand Up @@ -296,6 +311,7 @@ Get spill volume from tieredstore.
{{- else }}
- name: {{ $mediumName }}
emptyDir:
medium: {{ eq .type "MEM" | ternary "Memory" "" }}
{{- if .quota }}
{{- /* quota should be transformed to match resource.Quantity. e.g. 20GB -> 20Gi */}}
sizeLimit: {{ .quota | replace "B" "i" }}
Expand Down
2 changes: 1 addition & 1 deletion charts/vineyard/templates/config/vineyard-rpc-conf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,6 @@ metadata:
data:
{{- $rpcEndpoints := list }}
{{- range $e, $i := until $replicas }}
{{- $rpcEndpoints = append $rpcEndpoints (printf "%s-worker-%d.%s-worker-svc.%s.svc.cluster.local:%d" $fullName $i $fullName $namespace $rpcPort) }}
{{- $rpcEndpoints = append $rpcEndpoints (printf "%s-worker-%d.%s-worker-svc.%s:%d" $fullName $i $fullName $namespace $rpcPort) }}
{{- end }}
VINEYARD_RPC_ENDPOINT: {{ join "," $rpcEndpoints }}
16 changes: 8 additions & 8 deletions charts/vineyard/templates/fuse/daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,19 +65,19 @@ spec:
securityContext:
privileged: true
env:
- name: MOUNT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: MOUNT_DIR
value: {{ .Values.fuse.targetPath | dir }}
- name: FUSE_DIR
value: {{ .Values.fuse.targetPath }}
- name: RPC_CONF_DIR
value: {{ .Values.fuse.targetPath }}/rpc-conf
- name: PRESTOP_MARKER
value: /tmp/prestop-marker
- name: VINEYARD_FULL_NAME
value: {{ $fullName }}
lifecycle:
preStop:
exec:
{{/* umount the configmap volume manually to avoid the error of "possibly malicious path detecte" */}}
command: ["sh", "-c", "touch /tmp/prestop-marker && umount {{ .Values.fuse.targetPath }} && umount /runtime-mnt/vineyard/{{ .Release.Namespace }}/{{ $fullName }}/vineyard-fuse/rpc-conf"]
command: ["sh", "-c", "touch /tmp/prestop-marker && umount {{ .Values.fuse.targetPath }} && umount {{ .Values.fuse.targetPath }}/rpc-conf"]
{{- if .Values.master.resources }}
{{ include "vineyard.master.resources" . | indent 10 }}
{{- end }}
Expand All @@ -86,7 +86,7 @@ spec:
mountPath: {{ .Values.fuse.targetPath | dir }}
mountPropagation: Bidirectional
- name: vineyard-rpc-conf
mountPath: /runtime-mnt/vineyard/{{ .Release.Namespace }}/{{ $fullName }}/vineyard-fuse/rpc-conf
mountPath: {{ .Values.fuse.targetPath }}/rpc-conf
restartPolicy: Always
volumes:
- name: vineyard-fuse-mount
Expand Down
14 changes: 4 additions & 10 deletions charts/vineyard/templates/master/statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
{{- $peerPort := int .Values.master.ports.peer }}
{{- $name := include "vineyard.name" . }}
{{- $fullName := include "vineyard.fullname" . }}
{{- $etcdFullname := (printf "%s-%s" $fullName "master") }}
{{- $chart := include "vineyard.chart" . }}
{{- $releaseNamespace := .Release.Namespace }}
{{- $etcdServiceName := (printf "%s-%s" $fullName "master-svc") }}

{{- if not .Values.master.externalEndpoint }}
Expand Down Expand Up @@ -78,17 +76,13 @@ spec:
command:
- etcd
- --name=$(POD_NAME)
- --initial-advertise-peer-urls=http://$(POD_NAME).{{ $etcdServiceName }}.{{ .Release.Namespace }}.svc.cluster.local:{{ $peerPort }}
- --initial-advertise-peer-urls=http://$(POD_NAME).{{ $etcdServiceName }}.{{ .Release.Namespace }}:{{ $peerPort }}
- --listen-peer-urls=http://0.0.0.0:{{ $peerPort }}
- --advertise-client-urls=http://$(POD_NAME).{{ $etcdServiceName }}.{{ .Release.Namespace }}.svc.cluster.local:{{ $clientPort }}
- --advertise-client-urls=http://$(POD_NAME).{{ $etcdServiceName }}.{{ .Release.Namespace }}:{{ $clientPort }}
- --listen-client-urls=http://0.0.0.0:{{ $clientPort }}
- --initial-cluster-token=my-etcd-cluster
- --initial-cluster-token={{ .Release.Name }}
- --initial-cluster-state=new
{{- $initialCluster := list }}
{{- range $e, $i := until $replicas }}
{{- $initialCluster = append $initialCluster (printf "%s-%d=http://%s-%d.%s.%s.svc.cluster.local:%d" $etcdFullname $i $etcdFullname $i $etcdServiceName $releaseNamespace $peerPort) }}
{{- end }}
- --initial-cluster={{ join "," $initialCluster }}
- --initial-cluster={{ include "vineyard.etcdEndpoints" (dict "Values" .Values "Release" .Release "portType" "peer") }}
- --data-dir=/etcd-data
env:
- name: POD_NAME
Expand Down
4 changes: 4 additions & 0 deletions charts/vineyard/templates/worker/service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,7 @@ spec:
port: {{ int .Values.worker.ports.rpc }}
targetPort: {{ int .Values.worker.ports.rpc }}
protocol: TCP
- name: metrics
port: {{ int .Values.worker.ports.exporter }}
targetPort: {{ int .Values.worker.ports.exporter }}
protocol: TCP
25 changes: 25 additions & 0 deletions charts/vineyard/templates/worker/statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,11 @@ spec:
- --spill_upper_rate
- "{{ include "vineyard.spill.upperRate" . }}"
{{- end }}
{{- if not .Values.disablePrometheus }}
- --metrics
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it mean that vineyard reports metrics via exporter? But how vineyardd manage to find out which is the allocated exporter port?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it mean that vineyard reports metrics via exporter?

By vineyardd itself. The --metrics field means vineyardd will print internal metrics info (only about vineyard object) to the stdout and the log_dir field means vineyard will store these metrics info under a directory.

But how vineyardd manage to find out which is the allocated exporter port?

vineyardd doesn't need to know the port, we have an external exporter to convert these logs (/var/log/vineyard) to metrics via https://github.com/fluid-cloudnative/fluid/pull/3649/files#diff-6109bb7b99b63ccc2d90ea9d7c1ca8dece4d3873aabfc77280196643e7f6593cR162.

Copy link
Member

@TrafalgarZZZ TrafalgarZZZ Jan 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks! Got it now. One more question is that, does Vineyardd automatically rotate its log messages? And what will happen if the external exporter fail over? Will the external exporter recover all the metrics from the all the logs?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does Vineyardd automatically rotate its log messages?

No, vineyardd will preserve all log messages.

And what will happen if the external exporter fail over?

It will convert all logs to metrics again.

Will the external exporter recover all the metrics from the all the logs?

Yes, from the log file.

- --log_dir
- /var/log/vineyard
TrafalgarZZZ marked this conversation as resolved.
Show resolved Hide resolved
{{- end }}
env:
- name: VINEYARDD_NAME
value: {{ template "vineyard.fullname" . }}-vineyardd
Expand Down Expand Up @@ -150,9 +155,25 @@ spec:
volumeMounts:
- name: vineyard-socket-volume
mountPath: /var/run/vineyard
{{- if not .Values.disablePrometheus }}
- name: log
mountPath: /var/log/vineyard
{{- end }}
{{- if eq (include "vineyard.checkTieredStore" .) "true" }}
{{- include "vineyard.worker.tieredstoreVolumeMounts" . }}
{{- end }}
{{- if not .Values.disablePrometheus }}
- name: metrics
image: vineyardcloudnative/vineyardd-grok-exporter
imagePullPolicy: {{ .Values.worker.imagePullPolicy }}
ports:
- name: exporter
containerPort: {{ int .Values.worker.ports.exporter }}
protocol: TCP
volumeMounts:
- name: log
TrafalgarZZZ marked this conversation as resolved.
Show resolved Hide resolved
mountPath: /var/log/vineyard
{{- end }}
volumes:
{{- if eq (include "vineyard.checkTieredStore" .) "true" }}
{{- include "vineyard.worker.tieredstoreVolume" . }}
Expand All @@ -161,6 +182,10 @@ spec:
hostPath:
path: /runtime-mnt/vineyard/{{ .Release.Namespace }}/{{ $fullName }}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think /runtime-mnt is configurable. Try to avoid hard code this directory.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make sense to me.

type: DirectoryOrCreate
{{- if not .Values.disablePrometheus }}
- name: log
emptyDir: {}
{{- end }}
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
Expand Down
Loading
Loading