From 2dbb4550334e5f949d5af22f7e79cb01a516bdd0 Mon Sep 17 00:00:00 2001 From: kkasa Date: Tue, 16 Jan 2018 06:32:15 +0100 Subject: [PATCH] AMBARI-22787 - Add new Ambari Infra Manager component to Ambari infra stack --- .../ambari-infra-manager/docker/bin/start.sh | 2 +- .../docker/infra-manager-docker-compose.sh | 5 ++ .../DocumentArchivingConfiguration.java | 16 ++-- ....java => DocumentArchivingProperties.java} | 6 +- ...java => DocumentArchivingPropertyMap.java} | 16 ++-- .../job/archive/FileNameSuffixFormatter.java | 2 +- .../src/main/resources/infra-manager-env.sh | 2 +- .../main/resources/infra-manager.properties | 83 ++++++++++--------- .../ambari/infra/job/JobPropertiesTest.java | 26 +++--- 9 files changed, 82 insertions(+), 76 deletions(-) rename ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/{DocumentExportProperties.java => DocumentArchivingProperties.java} (97%) rename ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/{DocumentExportPropertyMap.java => DocumentArchivingPropertyMap.java} (69%) diff --git a/ambari-infra/ambari-infra-manager/docker/bin/start.sh b/ambari-infra/ambari-infra-manager/docker/bin/start.sh index 2402778740c..8c33d32b7e2 100755 --- a/ambari-infra/ambari-infra-manager/docker/bin/start.sh +++ b/ambari-infra/ambari-infra-manager/docker/bin/start.sh @@ -16,6 +16,6 @@ export INFRA_MANAGER_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,address=5007,server=y,suspend=n" touch /root/infra-manager.log -/root/ambari-infra-manager/infraManager.sh > /root/infra-manager.log +/root/ambari-infra-manager/bin/infraManager.sh start > /root/infra-manager.log tail -f /root/infra-manager.log diff --git a/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh b/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh index e5df48cc559..7e6654fef82 100755 --- a/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh +++ b/ambari-infra/ambari-infra-manager/docker/infra-manager-docker-compose.sh @@ -78,6 +78,11 @@ HADOOP_VERSION=3.0.0 EOF } +function get_docker_ip() { + local ip=$(ifconfig en0 | grep inet | awk '$1=="inet" {print $2}') + echo $ip +} + function setup_profile() { cat << EOF > $sdir/Profile AWS_ACCESS_KEY_ID=test diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java index 4fdc5dab408..837b9c4d03d 100644 --- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java +++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java @@ -48,7 +48,7 @@ public class DocumentArchivingConfiguration { private static final DocumentWiper NOT_DELETE = (firstDocument, lastDocument) -> { }; @Inject - private DocumentExportPropertyMap propertyMap; + private DocumentArchivingPropertyMap propertyMap; @Inject private StepBuilderFactory steps; @@ -66,12 +66,12 @@ public class DocumentArchivingConfiguration { @PostConstruct public void createJobs() { - if (propertyMap == null || propertyMap.getSolrDataExport() == null) + if (propertyMap == null || propertyMap.getSolrDataArchiving() == null) return; - propertyMap.getSolrDataExport().values().forEach(DocumentExportProperties::validate); + propertyMap.getSolrDataArchiving().values().forEach(DocumentArchivingProperties::validate); - propertyMap.getSolrDataExport().keySet().forEach(jobName -> { + propertyMap.getSolrDataArchiving().keySet().forEach(jobName -> { LOG.info("Registering data archiving job {}", jobName); Job job = logExportJob(jobName, exportStep); jobRegistryBeanPostProcessor.postProcessAfterInitialization(job, jobName); @@ -94,7 +94,7 @@ public Step exportStep(DocumentExporter documentExporter) { @StepScope public DocumentExporter documentExporter(DocumentItemReader documentItemReader, @Value("#{stepExecution.jobExecution.id}") String jobId, - @Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentExportProperties properties, + @Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentArchivingProperties properties, InfraManagerDataConfig infraManagerDataConfig, @Value("#{jobParameters[end]}") String intervalEnd, DocumentWiper documentWiper) { @@ -139,7 +139,7 @@ public DocumentExporter documentExporter(DocumentItemReader documentItemReader, @Bean @StepScope - public DocumentWiper documentWiper(@Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentExportProperties properties, + public DocumentWiper documentWiper(@Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentArchivingProperties properties, SolrDAO solrDAO) { if (isBlank(properties.getSolr().getDeleteQueryText())) return NOT_DELETE; @@ -148,7 +148,7 @@ public DocumentWiper documentWiper(@Value("#{stepExecution.jobExecution.executio @Bean @StepScope - public SolrDAO solrDAO(@Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentExportProperties properties) { + public SolrDAO solrDAO(@Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentArchivingProperties properties) { return new SolrDAO(properties.getSolr()); } @@ -161,7 +161,7 @@ private File outFile(String collection, File directoryPath, String suffix) { @Bean @StepScope public DocumentItemReader reader(ObjectSource documentSource, - @Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentExportProperties properties) { + @Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentArchivingProperties properties) { return new DocumentItemReader(documentSource, properties.getReadBlockSize()); } diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportProperties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java similarity index 97% rename from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportProperties.java rename to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java index d37edf4cc8d..b90402a4658 100644 --- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExportProperties.java +++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java @@ -38,7 +38,7 @@ import static org.apache.commons.csv.CSVFormat.DEFAULT; import static org.apache.commons.lang.StringUtils.isBlank; -public class DocumentExportProperties extends JobProperties { +public class DocumentArchivingProperties extends JobProperties { private int readBlockSize; private int writeBlockSize; private ExportDestination destination; @@ -55,8 +55,8 @@ public class DocumentExportProperties extends JobProperties { - private Map solrDataExport; +public class DocumentArchivingPropertyMap implements PropertyMap { + private Map solrDataArchiving; - public Map getSolrDataExport() { - return solrDataExport; + public Map getSolrDataArchiving() { + return solrDataArchiving; } - public void setSolrDataExport(Map solrDataExport) { - this.solrDataExport = solrDataExport; + public void setSolrDataArchiving(Map solrDataArchiving) { + this.solrDataArchiving = solrDataArchiving; } @Override - public Map getPropertyMap() { - return getSolrDataExport(); + public Map getPropertyMap() { + return getSolrDataArchiving(); } } diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatter.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatter.java index 9b4e662e100..85ec00b0118 100644 --- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatter.java +++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatter.java @@ -28,7 +28,7 @@ public class FileNameSuffixFormatter { private static final DateTimeFormatter SOLR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern(SOLR_DATE_FORMAT_TEXT); - public static FileNameSuffixFormatter from(DocumentExportProperties properties) { + public static FileNameSuffixFormatter from(DocumentArchivingProperties properties) { return new FileNameSuffixFormatter(properties.getFileNameSuffixColumn(), properties.getFileNameSuffixDateFormat()); } diff --git a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager-env.sh b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager-env.sh index 6897ea47199..9a371fd7cc0 100644 --- a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager-env.sh +++ b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager-env.sh @@ -19,7 +19,7 @@ export INFRA_MANAGER_OPTS="" # Log Search debug options # export INFRA_MANAGER_DEBUG=true -# exoprt INFRA_MANAGER_DEBUG_SUSPEND=n +# export INFRA_MANAGER_DEBUG_SUSPEND=n export INFRA_MANAGER_DEBUG_PORT=5005 # Log Search memory diff --git a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties index 70c46d33985..aea2b882a36 100644 --- a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties +++ b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + infra-manager.batch.db.file=job-repository.db infra-manager.batch.db.init=true infra-manager.batch.db.username=admin @@ -20,48 +21,48 @@ management.security.enabled=false management.health.solr.enabled=false infra-manager.server.data.folder=/tmp/ambariInfraManager -infra-manager.jobs.solr_data_export.export_service_logs.solr.zoo_keeper_connection_string=zookeeper:2181 -infra-manager.jobs.solr_data_export.export_service_logs.solr.collection=hadoop_logs -infra-manager.jobs.solr_data_export.export_service_logs.solr.query_text=logtime:[${start} TO ${end}] -infra-manager.jobs.solr_data_export.export_service_logs.solr.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}] -infra-manager.jobs.solr_data_export.export_service_logs.solr.sort_column[0]=logtime -infra-manager.jobs.solr_data_export.export_service_logs.solr.sort_column[1]=id -infra-manager.jobs.solr_data_export.export_service_logs.read_block_size=100 -infra-manager.jobs.solr_data_export.export_service_logs.write_block_size=150 -infra-manager.jobs.solr_data_export.export_service_logs.destination=LOCAL -infra-manager.jobs.solr_data_export.export_service_logs.local_destination_directory=/tmp/ambariInfraManager -infra-manager.jobs.solr_data_export.export_service_logs.file_name_suffix_column=logtime -infra-manager.jobs.solr_data_export.export_service_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX -infra-manager.jobs.solr_data_export.archive_audit_logs.solr.zoo_keeper_connection_string=zookeeper:2181 -infra-manager.jobs.solr_data_export.archive_audit_logs.solr.collection=audit_logs -infra-manager.jobs.solr_data_export.archive_audit_logs.solr.query_text=logtime:[${start} TO ${end}] -infra-manager.jobs.solr_data_export.archive_audit_logs.solr.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}] -infra-manager.jobs.solr_data_export.archive_audit_logs.solr.sort_column[0]=logtime -infra-manager.jobs.solr_data_export.archive_audit_logs.solr.sort_column[1]=id -infra-manager.jobs.solr_data_export.archive_audit_logs.solr.delete_query_text=logtime:[${start.logtime} TO ${end.logtime}} OR (logtime:${end.logtime} AND id:[* TO ${end.id}]) -infra-manager.jobs.solr_data_export.archive_audit_logs.read_block_size=100 -infra-manager.jobs.solr_data_export.archive_audit_logs.write_block_size=150 -infra-manager.jobs.solr_data_export.archive_audit_logs.destination=HDFS +infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.zoo_keeper_connection_string=zookeeper:2181 +infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.collection=hadoop_logs +infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.query_text=logtime:[${start} TO ${end}] +infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}] +infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.sort_column[0]=logtime +infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.sort_column[1]=id +infra-manager.jobs.solr_data_archiving.archive_service_logs.read_block_size=100 +infra-manager.jobs.solr_data_archiving.archive_service_logs.write_block_size=150 +infra-manager.jobs.solr_data_archiving.archive_service_logs.destination=LOCAL +infra-manager.jobs.solr_data_archiving.archive_service_logs.local_destination_directory=/tmp/ambariInfraManager +infra-manager.jobs.solr_data_archiving.archive_service_logs.file_name_suffix_column=logtime +infra-manager.jobs.solr_data_archiving.archive_service_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX +infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.zoo_keeper_connection_string=zookeeper:2181 +infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.collection=audit_logs +infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.query_text=logtime:[${start} TO ${end}] +infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}] +infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.sort_column[0]=logtime +infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.sort_column[1]=id +infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.delete_query_text=logtime:[${start.logtime} TO ${end.logtime}} OR (logtime:${end.logtime} AND id:[* TO ${end.id}]) +infra-manager.jobs.solr_data_archiving.archive_audit_logs.read_block_size=100 +infra-manager.jobs.solr_data_archiving.archive_audit_logs.write_block_size=150 +infra-manager.jobs.solr_data_archiving.archive_audit_logs.destination=S3 # TODO: logtime may not be enough: The same filename can be generated when more than write_block_size count docs has the same logtime value -infra-manager.jobs.solr_data_export.archive_audit_logs.file_name_suffix_column=logtime -infra-manager.jobs.solr_data_export.archive_audit_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX -infra-manager.jobs.solr_data_export.archive_audit_logs.hdfs_endpoint=hdfs://namenode:9000/ -infra-manager.jobs.solr_data_export.archive_audit_logs.hdfs_destination_directory=/test_audit_logs -#infra-manager.jobs.solr_data_export.archive_audit_logs.s3_access_file=.csv -infra-manager.jobs.solr_data_export.archive_audit_logs.s3_key_prefix=solr_archive_ -infra-manager.jobs.solr_data_export.archive_audit_logs.s3_bucket_name=testbucket -infra-manager.jobs.solr_data_export.archive_audit_logs.s3_endpoint=http://fakes3:4569 +infra-manager.jobs.solr_data_archiving.archive_audit_logs.file_name_suffix_column=logtime +infra-manager.jobs.solr_data_archiving.archive_audit_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX +infra-manager.jobs.solr_data_archiving.archive_audit_logs.hdfs_endpoint=hdfs://namenode:9000/ +infra-manager.jobs.solr_data_archiving.archive_audit_logs.hdfs_destination_directory=/test_audit_logs +#infra-manager.jobs.solr_data_archiving.archive_audit_logs.s3_access_file=.csv +infra-manager.jobs.solr_data_archiving.archive_audit_logs.s3_key_prefix=solr_archive_ +infra-manager.jobs.solr_data_archiving.archive_audit_logs.s3_bucket_name=testbucket +infra-manager.jobs.solr_data_archiving.archive_audit_logs.s3_endpoint=http://fakes3:4569 # TODO: configure ranger audit logs -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.zoo_keeper_connection_string=zookeeper:2181 -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.read_block_size=100 -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.write_block_size=150 -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.file_name_suffix_column=logtime -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.destination_directory_path=/tmp/ambariInfraManager -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.collection=hadoop_logs -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.query_text=logtime:[* TO "${end}"] -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.filter_query_text=(logtime:"${logtime}" AND id:{"${id}" TO *]) OR logtime:{"${logtime}" TO "${end}"] -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.sort_column[0]=logtime -#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.sort_column[1]=id +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.zoo_keeper_connection_string=zookeeper:2181 +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.read_block_size=100 +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.write_block_size=150 +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.file_name_suffix_column=logtime +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.destination_directory_path=/tmp/ambariInfraManager +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.query.collection=hadoop_logs +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.query.query_text=logtime:[* TO "${end}"] +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.query.filter_query_text=(logtime:"${logtime}" AND id:{"${id}" TO *]) OR logtime:{"${logtime}" TO "${end}"] +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.query.sort_column[0]=logtime +#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.query.sort_column[1]=id infra-manager.jobs.solr_data_deleting.delete_audit_logs.zoo_keeper_connection_string=zookeeper:2181 infra-manager.jobs.solr_data_deleting.delete_audit_logs.collection=audit_logs -infra-manager.jobs.solr_data_deleting.delete_audit_logs.filter_field=logtime \ No newline at end of file +infra-manager.jobs.solr_data_deleting.delete_audit_logs.filter_field=logtime diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobPropertiesTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobPropertiesTest.java index 21bcdb798e0..3b7caabd02e 100644 --- a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobPropertiesTest.java +++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobPropertiesTest.java @@ -1,6 +1,6 @@ package org.apache.ambari.infra.job; -import org.apache.ambari.infra.job.archive.DocumentExportProperties; +import org.apache.ambari.infra.job.archive.DocumentArchivingProperties; import org.apache.ambari.infra.job.archive.SolrProperties; import org.junit.Test; @@ -28,26 +28,26 @@ public class JobPropertiesTest { @Test public void testDeepCopy() throws Exception { - DocumentExportProperties documentExportProperties = new DocumentExportProperties(); - documentExportProperties.setLocalDestinationDirectory("/tmp"); - documentExportProperties.setFileNameSuffixColumn(".json"); - documentExportProperties.setReadBlockSize(10); - documentExportProperties.setWriteBlockSize(20); + DocumentArchivingProperties documentArchivingProperties = new DocumentArchivingProperties(); + documentArchivingProperties.setLocalDestinationDirectory("/tmp"); + documentArchivingProperties.setFileNameSuffixColumn(".json"); + documentArchivingProperties.setReadBlockSize(10); + documentArchivingProperties.setWriteBlockSize(20); SolrProperties solr = new SolrProperties(); solr.setZooKeeperConnectionString("localhost:2181"); solr.setFilterQueryText("id:1167"); solr.setQueryText("name:'Joe'"); solr.setCollection("Users"); solr.setSortColumn(new String[] {"name"}); - documentExportProperties.setSolr(solr); + documentArchivingProperties.setSolr(solr); - DocumentExportProperties parsed = documentExportProperties.deepCopy(); + DocumentArchivingProperties parsed = documentArchivingProperties.deepCopy(); - assertThat(parsed.getLocalDestinationDirectory(), is(documentExportProperties.getLocalDestinationDirectory())); - assertThat(parsed.getFileNameSuffixColumn(), is(documentExportProperties.getFileNameSuffixColumn())); - assertThat(parsed.getReadBlockSize(), is(documentExportProperties.getReadBlockSize())); - assertThat(parsed.getWriteBlockSize(), is(documentExportProperties.getWriteBlockSize())); - assertThat(parsed.getSolr().getZooKeeperConnectionString(), is(documentExportProperties.getSolr().getZooKeeperConnectionString())); + assertThat(parsed.getLocalDestinationDirectory(), is(documentArchivingProperties.getLocalDestinationDirectory())); + assertThat(parsed.getFileNameSuffixColumn(), is(documentArchivingProperties.getFileNameSuffixColumn())); + assertThat(parsed.getReadBlockSize(), is(documentArchivingProperties.getReadBlockSize())); + assertThat(parsed.getWriteBlockSize(), is(documentArchivingProperties.getWriteBlockSize())); + assertThat(parsed.getSolr().getZooKeeperConnectionString(), is(documentArchivingProperties.getSolr().getZooKeeperConnectionString())); assertThat(parsed.getSolr().getQueryText(), is(solr.getQueryText())); assertThat(parsed.getSolr().getFilterQueryText(), is(solr.getFilterQueryText())); assertThat(parsed.getSolr().getCollection(), is(solr.getCollection()));