azure.simple.conf

#
# Copyright (c) 2017-2019 Cloudera, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Sample Cloudera Altus Director configuration file based on the Cloudera Azure reference architecture:
# http://www.cloudera.com/documentation/other/reference-architecture/PDF/cloudera_ref_arch_azure.pdf
#
# Simple non-HA cluster with a single master node, 3 worker nodes
#


#
# Cluster name
#
# If environmentName and deploymentName are not defined they will get the value of 'name'.
# Must be unique.
#

name: C6-Simple-Azure

#
# Environment name
#

environmentName: Azure-nonHA

#
# Deployment name
# Used to name the Cloudera Manager instance in Cloudera Altus Director.
# Must be unique.
#

deploymentName: Cloudera-Manager-on-Azure


#
# Cloud provider configuration (credentials, region or zone, and more)
#

provider {
    type: azure

    #
    # ID of Azure region to use. NOTE: region must support Premium Storage
    # See: https://azure.microsoft.com/en-us/global-infrastructure/regions/
    #

    region: "region_REPLACE_ME"

    #
    # Azure Cloud Environment to use. Valid values are:
    #   - azure
    #   - azure-us-government
    #   - azure-germany
    #

    azureCloudEnvironment: "azureCloudEnvironment_REPLACE_ME"

    #
    # Azure Active Directory Subscription ID.
    #

    subscriptionId: "subscriptionId_REPLACE_ME"

    #
    # Tenant ID (from AAD)
    #

    tenantId: "tenantId_REPLACE_ME"

    #
    # Azure Active Directory Application Client ID.
    #

    clientId: "clientId_REPLACE_ME"

    #
    # Client Secret
    #

    clientSecret: "clientSecret_REPLACE_ME"
}


#
# SSH credentials to use to connect to the machines
#

ssh {
    username: "username_REPLACE_ME"
    privateKey: privateKey_REPLACE_ME # with an absolute path to .pem file, ${HOME} may be used
}

#
# Instance templates
#
# See azure.reference.conf for a breakdown of instance template fields.
#

instances {

    # fields that are common to all nodes
    base {
        type: STANDARD_D32S_V3
        image: cloudera-centos-72-latest
        automatic: false
        networkSecurityGroupResourceGroup: "networkSecurityGroupResourceGroup_REPLACE_ME"
        networkSecurityGroup: "networkSecurityGroup_REPLACE_ME"
        virtualNetworkResourceGroup: "virtualNetworkResourceGroup_REPLACE_ME"
        virtualNetwork: "virtualNetwork_REPLACE_ME"
        subnetName: "subnetName_REPLACE_ME"
        hostFqdnSuffix: "hostFqdnSuffix_REPLACE_ME"
        tags {
            owner: ${?USER}
        }
        bootstrapScripts: [ ${?bootstrap-script.os-generic} ]
    }

    # fields for master nodes
    master: ${instances.base} {
        computeResourceGroup: "master_computeResourceGroup_REPLACE_ME"
        availabilitySet: "master_availabilitySet_REPLACE_ME"
        instanceNamePrefix: "master_instanceNamePrefix_REPLACE_ME"
        storageAccountType: "Premium_LRS"
        dataDiskCount: 4
        dataDiskSize: 512
        managedDisks: Yes
        publicIP: No
    }

    # fields for worker nodes
    worker: ${instances.base} {
        computeResourceGroup: "worker_computeResourceGroup_REPLACE_ME"
        availabilitySet: "worker_availabilitySet_REPLACE_ME"
        instanceNamePrefix: "worker_instanceNamePrefix_REPLACE_ME"
        storageAccountType: "Standard_LRS"
        dataDiskCount: 11
        dataDiskSize: 1024
        managedDisks: Yes
        publicIP: No
    }

    # fields for edge nodes
    edge: ${instances.base} {
        computeResourceGroup: "edge_computeResourceGroup_REPLACE_ME"
        availabilitySet: "edge_availabilitySet_REPLACE_ME"
        instanceNamePrefix: "edge_instanceNamePrefix_REPLACE_ME"
        storageAccountType: "Standard_LRS"
        dataDiskCount: 1
        dataDiskSize: 512
        managedDisks: Yes
        # Change this to Yes to allow accessing edge/CM nodes via public IP
        publicIP: No
    }
}

include "azure-os-generic-bootstrap.conf"

#
# Configuration for Cloudera Manager. Cloudera Altus Director can use an existing Cloudera
# Manager installation, or bootstrap everything from scratch for a new cluster.
#

cloudera-manager {

    instance: ${instances.edge} {

        #
        # Additional tags for the Cloudera Manager instance
        #

        tags {
            application: "Cloudera Manager 6"
        }
    }

    #
    # Automatically activate 60-Day Cloudera Enterprise Trial
    #

    enableEnterpriseTrial: true

    #
    # Configuration for Cloudera Manager and its management services
    #
    # Configuration properties for CLOUDERA_MANAGER are documented at
    # https://www.cloudera.com/documentation/enterprise/6/properties/6.3/topics/cm_props_cmserver.html
    #
    # Configuration properties for the Cloudera Management services are documented at
    # https://www.cloudera.com/documentation/enterprise/6/properties/6.3/topics/cm_props_mgmtservice.html
    #
    # Configuration properties for Hosts are documented at
    # https://www.cloudera.com/documentation/enterprise/6/properties/6.3/topics/cm_props_host.html
    #
    # Properties here ensure that non-ephemeral disks are used for all directories and files
    #

    configs {

        SERVICEMONITOR {
            mgmt_log_dir: /data0/log/cloudera-scm-firehose
            firehose_storage_dir: /data0/lib/cloudera-service-monitor
        }

        ACTIVITYMONITOR {
            mgmt_log_dir: /data0/log/cloudera-scm-firehose
        }

        HOSTMONITOR {
            mgmt_log_dir: /data0/log/cloudera-scm-firehose
            firehose_storage_dir: /data0/lib/cloudera-host-monitor
        }

        REPORTSMANAGER {
            headlamp_scratch_dir: /data0/lib/cloudera-scm-headlamp
            mgmt_log_dir: /data0/log/cloudera-scm-headlamp
        }

        EVENTSERVER {
            mgmt_log_dir: /data0/log/cloudera-scm-eventserver
            eventserver_index_dir: /data0/lib/cloudera-scm-eventserver
        }

        ALERTPUBLISHER {
            mgmt_log_dir: /data0/log/cloudera-scm-alertpublisher
        }

        NAVIGATOR {
            mgmt_log_dir: /data0/log/cloudera-scm-navigator
        }

        NAVIGATORMETASERVER {
            audit_event_log_dir: /data0/log/cloudera-scm-navigator/audit
            data_dir: /data0/lib/cloudera-scm-navigator
            mgmt_log_dir: /data0/log/cloudera-scm-navigator
        }

        # Configuration properties for all hosts
        # HOSTS { ... }

    }

}


#
# Simple single-master Cluster description
#

cluster {

    # The table of products and their versions that need to be installed. Each
    # product must have a corresponding parcel in the parcelRepositories
    # configured in this section. The specified version for a product will be
    # used to find a suitable parcel. Specifying a version that is satisfied by
    # more than one parcel among those available will result in a configuration
    # error. Specify more granular versions to avoid conflicts.

    products {
        CDH: 6
    }

    #
    # Optional override of CDH parcel repositories
    #
    # This defaults to the Cloudera Enterprise release corresponding to the
    # Altus Director version.
    #

    # parcelRepositories: ["https://archive.cloudera.com/cdh6/6.3.0/parcels/"]

    #
    # Services to include in the cluster
    #

    services: [
      HDFS,
      YARN,
      ZOOKEEPER,
      HBASE,
      HIVE,
      HUE,
      IMPALA,
      OOZIE,
      SPARK_ON_YARN
    ]

    #
    # Custom service configurations
    #
    # Configuration keys containing special characters (e.g., '.', ':') must be enclosed in double
    # quotes.
    #
    # Configuration properties for CDH roles and services are documented at
    # https://www.cloudera.com/documentation/enterprise/6/properties/6.3/topics/cm_props_cdh620.html
    #
    # Properties here ensure that non-ephemeral disks are used for all directories and files
    #

    configs {

        HIVE {
            audit_event_log_dir: /data0/log/hive/audit
            lineage_event_log_dir: /data0/log/hive/lineage
        }

        HBASE {
            audit_event_log_dir: /data0/log/hbase/audit
        }

    }

    #
    # This reference configuration follows the Cloudera Azure Reference Architecture, modified
    # for a simple single-master cluster.
    #

    masters {
        count: 1

        instance: ${instances.master} {
            tags {
                group: masters
            }
        }

        roles {
            ZOOKEEPER: [SERVER]
            HDFS: [NAMENODE, SECONDARYNAMENODE]
            YARN: [RESOURCEMANAGER, JOBHISTORY]
            HBASE: [MASTER, HBASETHRIFTSERVER] # Alternately [HBASERESTSERVER], for HUE Integration
            HUE: [HUE_SERVER]
            OOZIE: [OOZIE_SERVER]
            IMPALA: [CATALOGSERVER, STATESTORE]
            HIVE: [HIVESERVER2, HIVEMETASTORE, WEBHCAT, GATEWAY]
            SPARK_ON_YARN: [SPARK_YARN_HISTORY_SERVER]
        }

        configs {

            HDFS {
                NAMENODE {
                    namenode_log_dir: /data0/log/hadoop-hdfs
                    dfs_name_dir_list: /data1/dfs/nn
                }
                SECONDARYNAMENODE {
                    secondarynamenode_log_dir: /data0/log/hadoop-hdfs
                    fs_checkpoint_dir_list: /data3/dfs/snn
                }
            }

            ZOOKEEPER {
                SERVER {
                    zk_server_log_dir: /data0/log/zookeeper
                    dataDir: /data2/zookeeper
                    dataLogDir: /data2/zookeeper
                }
            }

            YARN {
                RESOURCEMANAGER {
                    resource_manager_log_dir: /data0/log/hadoop-yarn
                }
                JOBHISTORY {
                    mr2_jobhistory_log_dir: /data0/log/hadoop-mapreduce
                }
            }

            HBASE {
                MASTER {
                    hbase_master_log_dir: /data0/log/hbase
                }
                HBASETHRIFTSERVER {
                    hbase_thriftserver_log_dir: /data0/log/hbase
                }
                #HBASERESTSERVER {
                #    hbase_restserver_log_dir: /data0/log/hbase
                #}
            }

            HIVE {
                HIVEMETASTORE {
                    hive_log_dir: /data0/log/hive
                }
                HIVESERVER2 {
                    hive_log_dir: /data0/log/hive
                }
                WEBHCAT {
                    hcatalog_log_dir: /data0/log/hcatalog
                }
            }

            OOZIE {
                OOZIE_SERVER {
                    oozie_log_dir: /data0/log/oozie
                }
            }

            HUE {
                HUE_SERVER {
                    hue_server_log_dir: /data0/log/hue
                }
            }

            IMPALA {
                CATALOGSERVER {
                    log_dir: /data0/log/catalogd
                }
                STATESTORE {
                    log_dir: /data0/log/statestore
                }
            }

            SPARK_ON_YARN {
                SPARK_YARN_HISTORY_SERVER {
                    log_dir: /data0/log/spark
                }
            }

        }
    }

    workers {

        #
        # The desired number of instances for this instance group. Cloudera Altus Director attempts
        # to launch this many instances, but will not fail bootstrap as long as the minimum number
        # of instances, specified with minCount below, succeed.
        #

        count: 3

        #
        # Minimum number of instances required for this instance group.
        # Altus Director will fail bootstrap of a cluster if minCount number of instances are not
        # available in this cloud environment.
        # If minCount is not specified then minCount is set to count.
        #

        minCount: 3

        instance: ${instances.worker} {
            tags {
                group: worker
            }
        }

        roles {
            HDFS: [DATANODE]
            YARN: [NODEMANAGER]
            HBASE: [REGIONSERVER]
            IMPALA: [IMPALAD]
        }

        configs {

            HDFS {
                DATANODE {
                    datanode_log_dir: /data0/log/hadoop-hdfs
                    dfs_data_dir_list: "/data1/dfs/dn,/data2/dfs/dn,/data3/dfs/dn,/data4/dfs/dn,/data5/dfs/dn,/data6/dfs/dn,/data7/dfs/dn,/data8/dfs/dn,/data9/dfs/dn,/data10/dfs/dn"
                }
            }

            YARN {
                NODEMANAGER {
                    node_manager_log_dir: /data0/log/hadoop-yarn
                    yarn_nodemanager_log_dirs: "/data1/log/hadoop-yarn/container,/data2/log/hadoop-yarn/container,/data3/log/hadoop-yarn/container,/data4/log/hadoop-yarn/container,/data5/log/hadoop-yarn/container,/data6/log/hadoop-yarn/container,/data7/log/hadoop-yarn/container,/data8/log/hadoop-yarn/container,/data9/log/hadoop-yarn/container,/data10/log/hadoop-yarn/container"
                    yarn_nodemanager_local_dirs: "/data1/yarn,/data2/yarn,/data3/yarn,/data4/yarn,/data5/yarn,/data6/yarn,/data7/yarn,/data8/yarn,/data9/yarn,/data10/yarn"
                }
            }

            HBASE {
                REGIONSERVER {
                    hbase_regionserver_log_dir: /data0/log/hbase
                }
            }

            IMPALA {
                IMPALAD {
                    log_dir: /data0/log/impalad
                    lineage_event_log_dir: /data0/log/impalad/lineage
                    audit_event_log_dir: /data0/log/impalad/audit
                    scratch_dirs: "/data1/impala/impalad,/data2/impala/impalad,/data3/impala/impalad,/data4/impala/impalad,/data5/impala/impalad,/data6/impala/impalad,/data7/impala/impalad,/data8/impala/impalad,/data9/impala/impalad,/data10/impala/impalad"
                }
            }

        }
    }

    gateways {
        count: 1

        instance: ${instances.edge} {
            tags {
                group: edge
            }
        }

        roles {
            HIVE: [GATEWAY]
            HBASE: [GATEWAY]
            SPARK_ON_YARN: [GATEWAY]
        }

        configs {

            SPARK_ON_YARN {
                GATEWAY {
                    lineage_event_log_dir : /data0/log/spark/lineage
                }
            }

        }
    }
}