diff --git a/hadoop-ozone/dev-support/k8s/regenerate-examples.sh b/hadoop-ozone/dev-support/k8s/regenerate-examples.sh new file mode 100755 index 000000000000..ffedaa3b725d --- /dev/null +++ b/hadoop-ozone/dev-support/k8s/regenerate-examples.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -u -o pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + +source "hadoop-ozone/dev-support/checks/_lib.sh" + +install_flekszible + +hadoop-ozone/dist/src/main/k8s/examples/regenerate-all.sh diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index f451a893eba7..8239325dd972 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -13,22 +13,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: v1 kind: ConfigMap metadata: name: config data: - + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*" + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*" + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: "/data/storage" - OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata" + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" - OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm" OZONE-SITE.XML_ozone.om.address: "om-0.om" + OZONE-SITE.XML_ozone.recon.address: "recon-0.recon" + OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm" OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm" + OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata" OZONE-SITE.XML_ozone.scm.names: "scm-0.scm" - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout" LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender" LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout" diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss-service.yaml new file mode 100644 index 000000000000..0b8f11752464 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss-service.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss.yaml new file mode 100644 index 000000000000..1084ca0373c5 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: "@docker.image@" + args: ["ozone","httpfs"] + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss-service.yaml new file mode 100644 index 000000000000..a7994a6b91a8 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss-service.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss.yaml new file mode 100644 index 000000000000..f06dc54c6648 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: "/prom" + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: "@docker.image@" + args: ["ozone","recon"] + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + volumes: [] diff --git a/hadoop-ozone/dist/src/main/k8s/examples/README.md b/hadoop-ozone/dist/src/main/k8s/examples/README.md new file mode 100644 index 000000000000..574efc6704a6 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/README.md @@ -0,0 +1,22 @@ + + +# Ozone examples for Kubernetes + +This directory contains example resources for running Ozone in Kubernetes. + +Note that the files are generated based on the [definitions](../definitions) using [Flekszible](https://github.com/elek/flekszible). If you would like to modify them permanently, edit the definition and regenerate the examples by running `regenerate-all.sh` after installing Flekszible. As a developer, you can run `hadoop-ozone/dev-support/k8s/regenerate-examples.sh`, which will also install Flekszible. diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index bb0608dac82d..4e503b7bd05c 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -19,22 +19,22 @@ kind: ConfigMap metadata: name: config data: - HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config CORE-SITE.XML_fs.defaultFS: ofs://om/ CORE-SITE.XML_fs.trash.interval: "1" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*" - HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*" + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.recon.address: recon-0.recon - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml index 4f60be17872b..427c654d543f 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml @@ -18,19 +18,19 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml - scm-statefulset.yaml -- httpfs-service.yaml -- httpfs-statefulset.yaml -- recon-service.yaml -- recon-statefulset.yaml - datanode-public-service.yaml +- httpfs-public-service.yaml - om-public-service.yaml +- recon-public-service.yaml - s3g-public-service.yaml - scm-public-service.yaml -- httpfs-public-service.yaml -- recon-public-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml index 2a0cf869a59c..4e503b7bd05c 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml @@ -19,16 +19,22 @@ kind: ConfigMap metadata: name: config data: + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-public-service.yaml new file mode 100644 index 000000000000..d8586250553a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs-public +spec: + ports: + - port: 14000 + name: rest + selector: + app: ozone + component: httpfs + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-statefulset.yaml new file mode 100644 index 000000000000..7bca21585c13 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-statefulset.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/kustomization.yaml index 3059b9c801e2..427c654d543f 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/kustomization.yaml @@ -18,13 +18,19 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml - scm-statefulset.yaml - datanode-public-service.yaml +- httpfs-public-service.yaml - om-public-service.yaml +- recon-public-service.yaml - s3g-public-service.yaml - scm-public-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-public-service.yaml new file mode 100644 index 000000000000..c737a02f446f --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon-public +spec: + ports: + - port: 9888 + name: ui + selector: + app: ozone + component: recon + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-statefulset.yaml new file mode 100644 index 000000000000..8b9ee191d035 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/recon-statefulset.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: '@docker.image@' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index 1d85fe91f5c9..7a966957364d 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -19,16 +19,22 @@ kind: ConfigMap metadata: name: config data: + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-public-service.yaml new file mode 100644 index 000000000000..d8586250553a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs-public +spec: + ports: + - port: 14000 + name: rest + selector: + app: ozone + component: httpfs + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-statefulset.yaml new file mode 100644 index 000000000000..59abe8547f64 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/httpfs-statefulset.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + env: + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml index cf0cbe152d2e..f654f6f510fe 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/kustomization.yaml @@ -18,8 +18,12 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml @@ -33,7 +37,9 @@ resources: - jaeger-service.yaml - jaeger-statefulset.yaml - datanode-public-service.yaml +- httpfs-public-service.yaml - om-public-service.yaml +- recon-public-service.yaml - s3g-public-service.yaml - scm-public-service.yaml - jaeger-public-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-public-service.yaml new file mode 100644 index 000000000000..c737a02f446f --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon-public +spec: + ports: + - port: 9888 + name: ui + selector: + app: ozone + component: recon + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-statefulset.yaml new file mode 100644 index 000000000000..6466c29595cf --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/recon-statefulset.yaml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: '@docker.image@' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml index 193a0618ab4f..a9315eb2103c 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml @@ -19,13 +19,19 @@ kind: ConfigMap metadata: name: config data: + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata - OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" + OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata + OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml new file mode 100644 index 000000000000..2c076ae8fcd4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/httpfs-statefulset.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/kustomization.yaml index 6b3d553113e6..49b6b6e2606b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/kustomization.yaml @@ -18,8 +18,12 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml new file mode 100644 index 000000000000..445c2e222d75 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/recon-statefulset.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: '@docker.image@' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: [] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml index 859307310507..0825afc37b66 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -19,16 +19,22 @@ kind: ConfigMap metadata: name: config data: + CORE-SITE.XML_fs.defaultFS: ofs://om/ + CORE-SITE.XML_fs.trash.interval: "1" + HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*' + HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*' + OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" OZONE-SITE.XML_hdds.datanode.dir: /data/storage - OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" + OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata - OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.recon.address: recon-0.recon + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3" - OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1" - OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-service.yaml new file mode 100644 index 000000000000..0ab49c2d72e4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: httpfs +spec: + ports: + - port: 14000 + name: rest + clusterIP: None + selector: + app: ozone + component: httpfs diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml new file mode 100644 index 000000000000..2c076ae8fcd4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/httpfs-statefulset.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: httpfs + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: httpfs + serviceName: httpfs + replicas: 1 + template: + metadata: + labels: + app: ozone + component: httpfs + spec: + containers: + - name: httpfs + image: '@docker.image@' + args: + - ozone + - httpfs + livenessProbe: + httpGet: + path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop + port: 14000 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/kustomization.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/kustomization.yaml index 6b3d553113e6..49b6b6e2606b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/kustomization.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/kustomization.yaml @@ -18,8 +18,12 @@ resources: - config-configmap.yaml - datanode-service.yaml - datanode-statefulset.yaml +- httpfs-service.yaml +- httpfs-statefulset.yaml - om-service.yaml - om-statefulset.yaml +- recon-service.yaml +- recon-statefulset.yaml - s3g-service.yaml - s3g-statefulset.yaml - scm-service.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-service.yaml new file mode 100644 index 000000000000..9c52d393d55d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: recon +spec: + ports: + - port: 9888 + name: ui + clusterIP: None + selector: + app: ozone + component: recon diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml new file mode 100644 index 000000000000..445c2e222d75 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/recon-statefulset.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: recon + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: recon + serviceName: recon + replicas: 1 + template: + metadata: + labels: + app: ozone + component: recon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9888" + prometheus.io/path: /prom + spec: + securityContext: + fsGroup: 1000 + containers: + - name: recon + image: '@docker.image@' + args: + - ozone + - recon + env: + - name: WAITFOR + value: scm-0.scm:9876 + livenessProbe: + tcpSocket: + port: 9891 + initialDelaySeconds: 30 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: [] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi