HDDS-11845. Extract k8s definitions for HttpFS and Recon from getting-started example

hadoop-ozone/dev-support/k8s/regenerate-examples.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -u -o pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+source "hadoop-ozone/dev-support/checks/_lib.sh"
+
+install_flekszible
+
+hadoop-ozone/dist/src/main/k8s/examples/regenerate-all.sh

hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml

@@ -13,22 +13,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: config
 data:
-
+  CORE-SITE.XML_fs.defaultFS: ofs://om/
+  CORE-SITE.XML_fs.trash.interval: "1"
+  HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
+  HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*"
+  HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*"
+  OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
   OZONE-SITE.XML_hdds.datanode.dir: "/data/storage"
-  OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata"
+  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
+  OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata"
-  OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm"
   OZONE-SITE.XML_ozone.om.address: "om-0.om"
+  OZONE-SITE.XML_ozone.recon.address: "recon-0.recon"
+  OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm"
   OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm"
+  OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata"
   OZONE-SITE.XML_ozone.scm.names: "scm-0.scm"
-  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
-  OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
-  OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
   LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout"
   LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender"
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout"

hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss-service.yaml

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: Service
+metadata:
+  name: httpfs
+spec:
+  ports:
+  - port: 14000
+    name: rest
+  clusterIP: None
+  selector:
+     app: ozone
+     component: httpfs

hadoop-ozone/dist/src/main/k8s/definitions/ozone/httpfs-ss.yaml

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: httpfs
+  labels:
+    app.kubernetes.io/component: ozone
+spec:
+  selector:
+    matchLabels:
+      app: ozone
+      component: httpfs
+  serviceName: httpfs
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: ozone
+        component: httpfs
+    spec:
+      containers:
+      - name: httpfs
+        image: "@docker.image@"
+        args: ["ozone","httpfs"]
+        livenessProbe:
+          httpGet:
+            path: /webhdfs/v1/?op=LISTSTATUS&user.name=hadoop
+            port: 14000
+          initialDelaySeconds: 30

hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss-service.yaml

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: Service
+metadata:
+  name: recon
+spec:
+  ports:
+  - port: 9888
+    name: ui
+  clusterIP: None
+  selector:
+     app: ozone
+     component: recon

hadoop-ozone/dist/src/main/k8s/definitions/ozone/recon-ss.yaml

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: recon
+  labels:
+    app.kubernetes.io/component: ozone
+spec:
+  selector:
+    matchLabels:
+      app: ozone
+      component: recon
+  serviceName: recon
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: ozone
+        component: recon
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9888"
+        prometheus.io/path: "/prom"
+    spec:
+      securityContext:
+        fsGroup: 1000
+      containers:
+      - name: recon
+        image: "@docker.image@"
+        args: ["ozone","recon"]
+        env:
+        - name: WAITFOR
+          value: scm-0.scm:9876
+        livenessProbe:
+          tcpSocket:
+            port: 9891
+          initialDelaySeconds: 30
+      volumes: []

hadoop-ozone/dist/src/main/k8s/examples/README.md

@@ -0,0 +1,22 @@
+<!---
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+--->
+
+# Ozone examples for Kubernetes
+
+This directory contains example resources for running Ozone in Kubernetes.
+
+Note that the files are generated based on the [definitions](../definitions) using [Flekszible](https://github.com/elek/flekszible).  If you would like to modify them permanently, edit the definition and regenerate the examples by running `regenerate-all.sh` after installing Flekszible.  As a developer, you can run `hadoop-ozone/dev-support/k8s/regenerate-examples.sh`, which will also install Flekszible.

hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml

@@ -19,22 +19,22 @@ kind: ConfigMap
 metadata:
   name: config
 data:
-  HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
   CORE-SITE.XML_fs.defaultFS: ofs://om/
   CORE-SITE.XML_fs.trash.interval: "1"
-  HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*"
-  HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*"
+  HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
+  HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*'
+  HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*'
+  OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
   OZONE-SITE.XML_hdds.datanode.dir: /data/storage
-  OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata
+  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
+  OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
-  OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.om.address: om-0.om
+  OZONE-SITE.XML_ozone.recon.address: recon-0.recon
+  OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm
+  OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata
   OZONE-SITE.XML_ozone.scm.names: scm-0.scm
-  OZONE-SITE.XML_ozone.recon.address: recon-0.recon
-  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
-  OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
-  OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
   LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout
   LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout

hadoop-ozone/dist/src/main/k8s/examples/getting-started/kustomization.yaml

@@ -18,19 +18,19 @@ resources:
 - config-configmap.yaml
 - datanode-service.yaml
 - datanode-statefulset.yaml
+- httpfs-service.yaml
+- httpfs-statefulset.yaml
 - om-service.yaml
 - om-statefulset.yaml
+- recon-service.yaml
+- recon-statefulset.yaml
 - s3g-service.yaml
 - s3g-statefulset.yaml
 - scm-service.yaml
 - scm-statefulset.yaml
-- httpfs-service.yaml
-- httpfs-statefulset.yaml
-- recon-service.yaml
-- recon-statefulset.yaml
 - datanode-public-service.yaml
+- httpfs-public-service.yaml
 - om-public-service.yaml
+- recon-public-service.yaml
 - s3g-public-service.yaml
 - scm-public-service.yaml
-- httpfs-public-service.yaml
-- recon-public-service.yaml

hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml

@@ -19,16 +19,22 @@ kind: ConfigMap
 metadata:
   name: config
 data:
+  CORE-SITE.XML_fs.defaultFS: ofs://om/
+  CORE-SITE.XML_fs.trash.interval: "1"
+  HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
+  HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*'
+  HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*'
+  OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
   OZONE-SITE.XML_hdds.datanode.dir: /data/storage
-  OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata
+  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
+  OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
-  OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.om.address: om-0.om
+  OZONE-SITE.XML_ozone.recon.address: recon-0.recon
+  OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm
+  OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata
   OZONE-SITE.XML_ozone.scm.names: scm-0.scm
-  OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
-  OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
-  OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
   LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout
   LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender
   LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout

hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-public-service.yaml

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: httpfs-public
+spec:
+  ports:
+  - port: 14000
+    name: rest
+  selector:
+    app: ozone
+    component: httpfs
+  type: NodePort

hadoop-ozone/dist/src/main/k8s/examples/minikube/httpfs-service.yaml

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: httpfs
+spec:
+  ports:
+  - port: 14000
+    name: rest
+  clusterIP: None
+  selector:
+    app: ozone
+    component: httpfs