From fdc72f09ffb0676eeb1604e467ed7577988509bc Mon Sep 17 00:00:00 2001 From: divyapandian Date: Tue, 31 Aug 2021 09:24:46 +0000 Subject: [PATCH 1/3] Committing patch TEZ-3820 with updates to counters and configs --- .../app/rm/TezAMRMClientAsyncProvider.java | 88 +++ .../findbugs-exclude.xml | 16 + .../pom.xml | 97 +++ .../ats/ATSV2HistoryLoggingService.java | 355 +++++++++++ .../HistoryEventTimelineConversionAtsv2.java | 596 ++++++++++++++++++ .../javadoc/resources/META-INF/LICENSE.txt | 202 ++++++ .../javadoc/resources/META-INF/NOTICE.txt | 6 + .../src/main/resources/META-INF/LICENSE.txt | 202 ++++++ .../src/main/resources/META-INF/NOTICE.txt | 6 + 9 files changed, 1568 insertions(+) create mode 100644 tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsyncProvider.java create mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/findbugs-exclude.xml create mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/pom.xml create mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/java/org/apache/tez/dag/history/logging/ats/ATSV2HistoryLoggingService.java create mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversionAtsv2.java create mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/LICENSE.txt create mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/NOTICE.txt create mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/LICENSE.txt create mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/NOTICE.txt diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsyncProvider.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsyncProvider.java new file mode 100644 index 0000000000..689230c5aa --- /dev/null +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsyncProvider.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.dag.app.rm; + +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import java.util.List; + + +public class TezAMRMClientAsyncProvider /*implements AMRMClientAsync.CallbackHandler*/ { + + private static TezAMRMClientAsync INSTANCE; + // private static AMRMClient INSTANCE; + + public static synchronized TezAMRMClientAsync createAMRMClientAsync( + int intervalMs, AMRMClientAsync.CallbackHandler callbackHandler) { + if (INSTANCE == null) { + INSTANCE = + TezAMRMClientAsync.createAMRMClientAsync(1000, callbackHandler); + } + return INSTANCE; + } + + public static synchronized TezAMRMClientAsync getAMRMClientAsync() { + return INSTANCE; + } + + + /* public static synchronized AMRMClient createAMRMClientAsync(){ + + if(INSTANCE == null){ + + INSTANCE= AMRMClient.createAMRMClient(); + + } + + return INSTANCE; + }*/ + + + + /* @Override + public void onContainersCompleted(List statuses) { + } + + @Override + public void onContainersAllocated(List containers) { + } + + @Override + public void onShutdownRequest() { + // LOG.warn("Shutting down"); + // end.set(true); + System.out.println("onshutdownrequest"); + } + + @Override + public void onNodesUpdated(List updatedNodes) { + } + + @Override + public float getProgress() { + return 0; + } + @Override + public void onError(Throwable e) { + // LOG.error("Unexpected error", e); + // end.set(true); + System.out.println("error"); + }*/ + +} diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/findbugs-exclude.xml b/tez-plugins/tez-yarn-timeline-history-with-atsv2/findbugs-exclude.xml new file mode 100644 index 0000000000..5b11308f6d --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-atsv2/findbugs-exclude.xml @@ -0,0 +1,16 @@ + + + + diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/pom.xml b/tez-plugins/tez-yarn-timeline-history-with-atsv2/pom.xml new file mode 100644 index 0000000000..5f8217632b --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-atsv2/pom.xml @@ -0,0 +1,97 @@ + + + + 4.0.0 + + org.apache.tez + tez-plugins + 0.9.2 + + tez-yarn-timeline-history-with-atsv2 + + + + org.apache.tez + tez-api + + + org.apache.tez + tez-common + + + org.apache.tez + tez-dag + 0.9.2 + + + org.apache.tez + tez-tests + test-jar + test + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-yarn-api + 2.10.1 + + + org.apache.hadoop + hadoop-yarn-common + 2.10.1 + + + org.apache.hadoop + hadoop-yarn-client + 2.10.1 + + + org.apache.hadoop + hadoop-yarn-server-tests + test + test-jar + + + org.slf4j + slf4j-api + + + org.mockito + mockito-all + test + + + junit + junit + test + + + + + + + org.apache.rat + apache-rat-plugin + + + + + + diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/java/org/apache/tez/dag/history/logging/ats/ATSV2HistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/java/org/apache/tez/dag/history/logging/ats/ATSV2HistoryLoggingService.java new file mode 100644 index 0000000000..c01a6bd425 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/java/org/apache/tez/dag/history/logging/ats/ATSV2HistoryLoggingService.java @@ -0,0 +1,355 @@ +package org.apache.tez.dag.history.logging.ats; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.client.api.TimelineV2Client; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.tez.common.TezUtilsInternal; +import org.apache.tez.dag.api.TezConfiguration; +import org.apache.tez.dag.api.TezConstants; +import org.apache.tez.dag.app.rm.TezAMRMClientAsync; +import org.apache.tez.dag.app.rm.TezAMRMClientAsyncProvider; +import org.apache.tez.dag.history.DAGHistoryEvent; +import org.apache.tez.dag.history.HistoryEventType; +import org.apache.tez.dag.history.events.DAGRecoveredEvent; +import org.apache.tez.dag.history.events.DAGSubmittedEvent; +import org.apache.tez.dag.history.logging.EntityTypes; +import org.apache.tez.dag.history.logging.HistoryLoggingService; +import org.apache.tez.dag.records.TezDAGID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class ATSV2HistoryLoggingService extends HistoryLoggingService { + + private static final Logger LOG = + LoggerFactory.getLogger(ATSV2HistoryLoggingService.class); + public static final String TEZ_PREFIX = "tez_"; + + TimelineV2Client timelineV2Client; + private HashSet skippedDAGs = new HashSet(); + + BlockingQueue eventQueue = + new LinkedBlockingQueue(); + protected Thread eventHandlingThread; + private AtomicBoolean stopped = new AtomicBoolean(false); + private AtomicBoolean registered = new AtomicBoolean(false); + private final Object lock = new Object(); + private int eventCounter = 0; + private int eventsProcessed = 0; + private boolean timelineServiceEnabled = true; + private long maxTimeToWaitOnShutdown; + private boolean waitForeverOnShutdown = false; + + private long maxPollingTimeMillis; + + // Number of bytes of config which can be published in one shot to ATSv2. + public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024; + + public ATSV2HistoryLoggingService() { + super(ATSV2HistoryLoggingService.class.getName()); + } + + @Override protected void serviceInit(Configuration conf) throws Exception { + boolean historyLoggingEnabled = + conf.getBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED, + TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT); + if (!historyLoggingEnabled) { + LOG.info("ATSV2Service: History Logging disabled. " + + TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED + " set to false"); + return; + } + + maxTimeToWaitOnShutdown = + conf.getLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS, + TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT); + maxPollingTimeMillis = + conf.getInt(TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT, + TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT); + + if (maxTimeToWaitOnShutdown < 0) { + waitForeverOnShutdown = true; + } + + timelineServiceEnabled = conf.getBoolean( + YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED); + if (!timelineServiceEnabled && + conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") + .equals(ATSV2HistoryLoggingService.class.getName())) { + LOG.warn(ATSV2HistoryLoggingService.class.getName() + + " is disabled due to Timeline Service being disabled, " + + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false"); + } + super.serviceInit(conf); + } + + @Override protected void serviceStart() throws Exception { + if (!timelineServiceEnabled) { + return; + } + super.serviceStart(); + } + + private void createAndRegisterTimelineClient(TezAMRMClientAsync amRmClient) throws Exception { + if (amRmClient == null) { + LOG.warn("Cannot register TimeClient without amRmClient"); + return; + } + Configuration conf = getConfig(); + if (timelineServiceEnabled) { + timelineV2Client = + TimelineV2Client.createTimelineClient(appContext.getApplicationID()); + timelineV2Client.init(conf); + LOG.info("timelineV2Client inited."); + amRmClient.registerTimelineV2Client(timelineV2Client); + + LOG.info("timelineV2Client registered using amRmClient."); + timelineV2Client.start(); + + eventHandlingThread = new Thread(createThread()); + eventHandlingThread.setName("ATSV2HistoryEventHandlingThread"); + eventHandlingThread.start(); + + registered.set(true); + } + } + + @Override protected void serviceStop() throws Exception { + if (!timelineServiceEnabled || !registered.get()) { + return; + } + LOG.info( + "Stopping ATSV2Service" + ", eventQueueBacklog=" + eventQueue.size()); + stopped.set(true); + // if (eventHandlingThread != null) { + // eventHandlingThread.interrupt(); + // } + try { + TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), + appContext.getApplicationID()); + synchronized (lock) { + if (eventHandlingThread != null) { + eventHandlingThread.interrupt(); + } + if (!eventQueue.isEmpty()) { + LOG.warn( + "ATSV2Service being stopped" + ", eventQueueBacklog=" + eventQueue + .size() + ", maxTimeLeftToFlush=" + maxTimeToWaitOnShutdown + + ", waitForever=" + waitForeverOnShutdown); + long startTime = appContext.getClock().getTime(); + long endTime = startTime + maxTimeToWaitOnShutdown; + while (waitForeverOnShutdown || (endTime >= appContext.getClock() + .getTime())) { + try { + DAGHistoryEvent event = + eventQueue.poll(maxPollingTimeMillis, TimeUnit.MILLISECONDS); + if (event == null) { + LOG.info("Event queue empty, stopping ATS Service"); + break; + } + try { + handleEvents(event); + } catch (Exception e) { + LOG.warn("Error handling event", e); + } + } catch (InterruptedException e) { + LOG.info("ATSService interrupted while shutting down. Exiting." + + " EventQueueBacklog=" + eventQueue.size()); + } + } + } + } + } finally { + appContext.getHadoopShim().clearHadoopCallerContext(); + } + if (!eventQueue.isEmpty()) { + LOG.warn("Did not finish flushing eventQueue before stopping ATSService" + + ", eventQueueBacklog=" + eventQueue.size()); + } + if (timelineV2Client != null) { + timelineV2Client.stop(); + } + // stop all the components + super.serviceStop(); + } + + private void handleEvents(DAGHistoryEvent event) { + TezDAGID dagID = event.getDagID(); + + if (event.getDagID() != null && skippedDAGs.contains(event.getDagID())) { + return; + } + LOG.info("createTimelineEntity called " + event.getHistoryEvent().getEventType()); + TimelineEntity timelineEntity = HistoryEventTimelineConversionAtsv2.createTimelineEntity(event.getHistoryEvent()); + + try { + // TODO identify which event need to publish in sync and async + timelineV2Client.putEntitiesAsync(timelineEntity); + } catch (IOException | YarnException e) { + LOG.error( + "Failed to publish Event " + event.getHistoryEvent().getEventType() + + " for the dag : " + dagID, e); + return; + } + + if (event.getHistoryEvent().getEventType() + .equals(HistoryEventType.DAG_SUBMITTED)) { + publishConfigsOnDagSubmittedEvent( + (DAGSubmittedEvent) event.getHistoryEvent()); + } + } + + private void publishConfigsOnDagSubmittedEvent(DAGSubmittedEvent event) { + String dagId = event.getDagID().toString(); + String entityId = TEZ_PREFIX + dagId; + String entityType = EntityTypes.TEZ_DAG_ID.name(); + TimelineEntity dagConfigEntity = new TimelineEntity(); + dagConfigEntity.setId(entityId); + dagConfigEntity.setType(entityType); + + try { + int configSize = 0; + for (Map.Entry entry : event.getConf()) { + int size = entry.getKey().length() + entry.getValue().length(); + configSize += size; + if (configSize > ATS_CONFIG_PUBLISH_SIZE_BYTES) { + if (dagConfigEntity.getConfigs().size() > 0) { + timelineV2Client.putEntitiesAsync(dagConfigEntity); + // create new entity + dagConfigEntity = new TimelineEntity(); + dagConfigEntity.setId(entityId); + dagConfigEntity.setType(entityType); + } + configSize = size; + } + dagConfigEntity.addConfig(entry.getKey(), entry.getValue()); + } + if (configSize > 0) { + LOG.info("before timelineV2Client putentities" + dagConfigEntity); + timelineV2Client.putEntities(dagConfigEntity); + LOG.info("after timelineV2Client putentities"); + } + } catch (IOException | YarnException e) { + LOG.error("Exception while publishing configs on JOB_SUBMITTED Event " + + " for the job : " + event.getDagID(), e); + } + } + + @Override public void handle(DAGHistoryEvent event) { + if (timelineServiceEnabled) { + if (!registered.get()) { + TezAMRMClientAsync amRmClient = TezAMRMClientAsyncProvider.getAMRMClientAsync(); + if (amRmClient != null) { + try { + createAndRegisterTimelineClient(amRmClient); + } catch(Exception ex){ + LOG.error("Error while create and register timelineClient", ex); + + } + } + } + eventQueue.add(event); + } + } + + Runnable createThread() { + return new Runnable() { + @Override public void run() { + boolean interrupted = false; + TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), + appContext.getApplicationID()); + while (!stopped.get() && !Thread.currentThread().isInterrupted() + && !interrupted) { + + // Log the size of the event-queue every so often. + if (eventCounter != 0 && eventCounter % 1000 == 0) { + if (eventsProcessed != 0 && !eventQueue.isEmpty()) { + LOG.info("Event queue stats" + ", eventsProcessedSinceLastUpdate=" + + eventsProcessed + ", eventQueueSize=" + eventQueue.size()); + } + eventCounter = 0; + eventsProcessed = 0; + } else { + ++eventCounter; + } + + synchronized (lock) { + try { + DAGHistoryEvent event = + eventQueue.poll(maxPollingTimeMillis, TimeUnit.MILLISECONDS); + if (event == null) { + continue; + } + if (!isValidEvent(event)) { + continue; + } + + try { + handleEvents(event); + eventsProcessed += 1; + } catch (Exception e) { + LOG.warn("Error handling events", e); + } + } catch (InterruptedException e) { + // Finish processing events and then return + interrupted = true; + } + } + } + } + }; + } + + private boolean isValidEvent(DAGHistoryEvent event) { + HistoryEventType eventType = event.getHistoryEvent().getEventType(); + TezDAGID dagId = event.getDagID(); + + if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) { + DAGSubmittedEvent dagSubmittedEvent = + (DAGSubmittedEvent) event.getHistoryEvent(); + String dagName = dagSubmittedEvent.getDAGName(); + if ((dagName != null && dagName + .startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) + || (!dagSubmittedEvent.isHistoryLoggingEnabled())) { + // Skip recording pre-warm DAG events + skippedDAGs.add(dagId); + return false; + } + } + if (eventType.equals(HistoryEventType.DAG_RECOVERED)) { + DAGRecoveredEvent dagRecoveredEvent = + (DAGRecoveredEvent) event.getHistoryEvent(); + if (!dagRecoveredEvent.isHistoryLoggingEnabled()) { + skippedDAGs.add(dagRecoveredEvent.getDagID()); + return false; + } + } + if (eventType.equals(HistoryEventType.DAG_FINISHED)) { + // Remove from set to keep size small + // No more events should be seen after this point. + if (skippedDAGs.remove(dagId)) { + return false; + } + } + + if (dagId != null && skippedDAGs.contains(dagId)) { + // Skip pre-warm DAGs + return false; + } + + return true; + } + +} + + diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversionAtsv2.java b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversionAtsv2.java new file mode 100644 index 0000000000..794a834c7b --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversionAtsv2.java @@ -0,0 +1,596 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tez.dag.history.logging.ats; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; +import org.apache.tez.common.ATSConstants; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; +import org.apache.tez.dag.api.EdgeProperty; +import org.apache.tez.dag.api.TezUncheckedException; +import org.apache.tez.dag.api.oldrecords.TaskAttemptState; +import org.apache.tez.dag.api.records.DAGProtos.CallerContextProto; +import org.apache.tez.dag.app.web.AMWebController; +import org.apache.tez.dag.history.HistoryEvent; +import org.apache.tez.dag.history.HistoryEventType; +import org.apache.tez.dag.history.events.AMLaunchedEvent; +import org.apache.tez.dag.history.events.AMStartedEvent; +import org.apache.tez.dag.history.events.AppLaunchedEvent; +import org.apache.tez.dag.history.events.ContainerLaunchedEvent; +import org.apache.tez.dag.history.events.ContainerStoppedEvent; +import org.apache.tez.dag.history.events.DAGFinishedEvent; +import org.apache.tez.dag.history.events.DAGInitializedEvent; +import org.apache.tez.dag.history.events.DAGRecoveredEvent; +import org.apache.tez.dag.history.events.DAGStartedEvent; +import org.apache.tez.dag.history.events.DAGSubmittedEvent; +import org.apache.tez.dag.history.events.TaskAttemptFinishedEvent; +import org.apache.tez.dag.history.events.TaskAttemptStartedEvent; +import org.apache.tez.dag.history.events.TaskFinishedEvent; +import org.apache.tez.dag.history.events.TaskStartedEvent; +import org.apache.tez.dag.history.events.VertexConfigurationDoneEvent; +import org.apache.tez.dag.history.events.VertexFinishedEvent; +import org.apache.tez.dag.history.events.VertexInitializedEvent; +import org.apache.tez.dag.history.events.VertexStartedEvent; +import org.apache.tez.dag.history.logging.EntityTypes; +import org.apache.tez.dag.history.utils.DAGUtils; +import org.apache.tez.dag.records.TezVertexID; + +public class HistoryEventTimelineConversionAtsv2{ + + public static final String TEZ_PREFIX = "tez_"; + + private static void validateEvent(HistoryEvent event) { + if (!event.isHistoryEvent()) { + throw new UnsupportedOperationException( + "Invalid Event, does not support history" + ", eventType=" + event + .getEventType()); + } + } + + public static TimelineEntity createTimelineEntity(HistoryEvent historyEvent) { + validateEvent(historyEvent); + + switch (historyEvent.getEventType()) { + case APP_LAUNCHED: + return createAppLaunchedEntity((AppLaunchedEvent) historyEvent); + case AM_LAUNCHED: + return createAMLaunchedEntity((AMLaunchedEvent) historyEvent); + case AM_STARTED: + return createAMStartedEntity((AMStartedEvent) historyEvent); + case CONTAINER_LAUNCHED: + return createContainerLaunchedEntity( + (ContainerLaunchedEvent) historyEvent); + case CONTAINER_STOPPED: + return createContainerStoppedEntity((ContainerStoppedEvent) historyEvent); + case DAG_SUBMITTED: + return createDAGSubmittedEntity((DAGSubmittedEvent) historyEvent); + case DAG_INITIALIZED: + return createDAGInitializedEntity((DAGInitializedEvent) historyEvent); + case DAG_STARTED: + return createDAGStartedEntity((DAGStartedEvent) historyEvent); + case DAG_FINISHED: + return createDAGFinishedEntity((DAGFinishedEvent) historyEvent); + case VERTEX_INITIALIZED: + return createVertexInitializedEntity( + (VertexInitializedEvent) historyEvent); + case VERTEX_STARTED: + return createVertexStartedEntity((VertexStartedEvent) historyEvent); + case VERTEX_FINISHED: + return createVertexFinishedEntity((VertexFinishedEvent) historyEvent); + case TASK_STARTED: + return createTaskStartedEntity((TaskStartedEvent) historyEvent); + case TASK_FINISHED: + return createTaskFinishedEntity((TaskFinishedEvent) historyEvent); + case TASK_ATTEMPT_STARTED: + return createTaskAttemptStartedEntity( + (TaskAttemptStartedEvent) historyEvent); + case TASK_ATTEMPT_FINISHED: + return createTaskAttemptFinishedEntity( + (TaskAttemptFinishedEvent) historyEvent); + case VERTEX_CONFIGURE_DONE: + return createVertexReconfigureDoneEntity( + (VertexConfigurationDoneEvent) historyEvent); + case DAG_RECOVERED: + return createDAGRecoveredEntity((DAGRecoveredEvent) historyEvent); + case VERTEX_COMMIT_STARTED: + case VERTEX_GROUP_COMMIT_STARTED: + case VERTEX_GROUP_COMMIT_FINISHED: + case DAG_COMMIT_STARTED: + throw new UnsupportedOperationException( + "Invalid Event, does not support history" + ", eventType=" + + historyEvent.getEventType()); + default: + throw new UnsupportedOperationException( + "Unhandled Event" + ", eventType=" + historyEvent.getEventType()); + } + + } + + private static TimelineEntity createAppLaunchedEntity( + AppLaunchedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getApplicationId().toString(), + EntityTypes.TEZ_APPLICATION.name(), event.getLaunchTime()); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getLaunchTime()); + entity.addEvent(tEvent); + + entity.addInfo(ATSConstants.CONFIG, + DAGUtils.convertConfigurationToATSMap(event.getConf())); + entity.addInfo(ATSConstants.APPLICATION_ID, + event.getApplicationId().toString()); + entity.addInfo(ATSConstants.USER, event.getUser()); + if (event.getVersion() != null) { + entity.addInfo(ATSConstants.TEZ_VERSION, + DAGUtils.convertTezVersionToATSMap(event.getVersion())); + } + entity.addInfo(ATSConstants.DAG_AM_WEB_SERVICE_VERSION, AMWebController.VERSION); + + + return entity; + } + + private static TimelineEntity createAMLaunchedEntity(AMLaunchedEvent event) { + TimelineEntity entity = createBaseEntity( + TEZ_PREFIX + event.getApplicationAttemptId().toString(), + EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), event.getAppSubmitTime()); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getAppSubmitTime()); + entity.addEvent(tEvent); + entity.addInfo(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime()); + entity.addInfo(ATSConstants.APPLICATION_ID, + event.getApplicationAttemptId().getApplicationId().toString()); + entity.addInfo(ATSConstants.APPLICATION_ATTEMPT_ID, + event.getApplicationAttemptId().toString()); + entity.addInfo(ATSConstants.USER, event.getUser()); + + return entity; + } + + private static TimelineEntity createAMStartedEntity(AMStartedEvent event) { + TimelineEntity entity = createBaseEntity( + TEZ_PREFIX + event.getApplicationAttemptId().toString(), + EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), null); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getStartTime()); + entity.addEvent(tEvent); + + return entity; + } + + private static TimelineEntity createContainerLaunchedEntity( + ContainerLaunchedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getContainerId().toString(), + EntityTypes.TEZ_CONTAINER_ID.name(), event.getLaunchTime()); + entity.addIsRelatedToEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), + "tez_" + event.getApplicationAttemptId().toString()); + + + entity.addInfo(ATSConstants.CONTAINER_ID, + event.getContainerId().toString()); + entity.setCreatedTime(event.getLaunchTime()); + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getLaunchTime()); + entity.addEvent(tEvent); + + return entity; + } + + private static TimelineEntity createContainerStoppedEntity( + ContainerStoppedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getContainerId().toString(), + EntityTypes.TEZ_CONTAINER_ID.name(), null); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getStoppedTime()); + entity.addEvent(tEvent); + + // In case, a container is stopped in a different attempt + entity.addIsRelatedToEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), + "tez_" + event.getApplicationAttemptId().toString()); + + entity.addInfo(ATSConstants.EXIT_STATUS, event.getExitStatus()); + entity.addInfo(ATSConstants.FINISH_TIME, event.getStoppedTime()); + + return entity; + } + + private static TimelineEntity createDAGSubmittedEntity( + DAGSubmittedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getDagID().toString(), + EntityTypes.TEZ_DAG_ID.name(), event.getSubmitTime()); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getSubmitTime()); + entity.addEvent(tEvent); + + + entity.addIsRelatedToEntity(EntityTypes.TEZ_APPLICATION.name(), + "tez_" + event.getApplicationAttemptId().getApplicationId().toString()); + entity.addIsRelatedToEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(), + "tez_" + event.getApplicationAttemptId().toString()); + + + if (event.getDAGPlan().hasCallerContext() + && event.getDAGPlan().getCallerContext().hasCallerId()) { + CallerContextProto callerContext = event.getDagPlan().getCallerContext(); + entity.addInfo(ATSConstants.CALLER_CONTEXT_ID, callerContext.getCallerId()); + entity.addInfo(ATSConstants.CALLER_CONTEXT, callerContext.getContext()); + } + + entity.addInfo(ATSConstants.APPLICATION_ID, + event.getApplicationAttemptId().getApplicationId().toString()); + entity.addInfo(ATSConstants.APPLICATION_ATTEMPT_ID, + event.getApplicationAttemptId().toString()); + entity.addInfo(ATSConstants.USER, event.getUser()); + entity.addInfo(ATSConstants.DAG_AM_WEB_SERVICE_VERSION, AMWebController.VERSION); + entity.addInfo(ATSConstants.IN_PROGRESS_LOGS_URL + "_" + + event.getApplicationAttemptId().getAttemptId(), event.getContainerLogs()); + if (event.getDAGPlan().hasCallerContext() + && event.getDAGPlan().getCallerContext().hasCallerId() + && event.getDAGPlan().getCallerContext().hasCallerType()) { + entity.addInfo(ATSConstants.CALLER_CONTEXT_ID, + event.getDAGPlan().getCallerContext().getCallerId()); + entity.addInfo(ATSConstants.CALLER_CONTEXT_TYPE, + event.getDAGPlan().getCallerContext().getCallerType()); + } + if (event.getQueueName() != null) { + entity.addInfo(ATSConstants.DAG_QUEUE_NAME, event.getQueueName()); + } + + try { + entity.addInfo(ATSConstants.DAG_PLAN, + DAGUtils.convertDAGPlanToATSMap(event.getDAGPlan())); + } catch (IOException e) { + throw new TezUncheckedException(e); + } + return entity; + } + + private static TimelineEntity createDAGInitializedEntity( + DAGInitializedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getDagID().toString(), + EntityTypes.TEZ_DAG_ID.name(), null); + entity.addInfo(ATSConstants.INIT_TIME, event.getInitTime()); + if (event.getVertexNameIDMap() != null) { + Map nameIdStrMap = new TreeMap(); + for (Entry entry : event.getVertexNameIDMap().entrySet()) { + nameIdStrMap.put(entry.getKey(), entry.getValue().toString()); + } + entity.addInfo(ATSConstants.VERTEX_NAME_ID_MAPPING, nameIdStrMap); + } + + return entity; + } + + private static TimelineEntity createDAGStartedEntity(DAGStartedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getDagID().toString(), + EntityTypes.TEZ_DAG_ID.name(), null); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getStartTime()); + entity.addEvent(tEvent); + entity.addInfo(ATSConstants.START_TIME, event.getStartTime()); + entity.addInfo(ATSConstants.STATUS, event.getDagState().toString()); + return entity; + } + + private static TimelineEntity createDAGFinishedEntity( + DAGFinishedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getDagID().toString(), + EntityTypes.TEZ_DAG_ID.name(), null); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getFinishTime()); + entity.addEvent(tEvent); + + entity.addInfo(ATSConstants.START_TIME, event.getStartTime()); + entity.addInfo(ATSConstants.FINISH_TIME, event.getFinishTime()); + entity.addInfo(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime())); + entity.addInfo(ATSConstants.STATUS, event.getState().name()); + entity.addInfo(ATSConstants.DIAGNOSTICS, event.getDiagnostics()); + entity.addInfo(ATSConstants.COMPLETION_APPLICATION_ATTEMPT_ID, + event.getApplicationAttemptId().toString()); + + + final Map dagTaskStats = event.getDagTaskStats(); + if (dagTaskStats != null) { + for(Entry entry : dagTaskStats.entrySet()) { + entity.addInfo(entry.getKey(), entry.getValue()); + } + } + + entity.addInfo(ATSConstants.COUNTERS, + DAGUtils.convertCountersToATSMap(event.getTezCounters())); + + return entity; + } + + private static TimelineEntity createVertexInitializedEntity( + VertexInitializedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getVertexID().toString(), + EntityTypes.TEZ_VERTEX_ID.name(), event.getInitedTime()); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getInitedTime()); + entity.addEvent(tEvent); + + entity.addIsRelatedToEntity(EntityTypes.TEZ_DAG_ID.name(), + event.getVertexID().getDAGId().toString()); + entity.addInfo(ATSConstants.VERTEX_NAME, event.getVertexName()); + entity.addInfo(ATSConstants.INIT_REQUESTED_TIME, event.getInitRequestedTime()); + entity.addInfo(ATSConstants.INIT_TIME, event.getInitedTime()); + entity.addInfo(ATSConstants.NUM_TASKS, event.getNumTasks()); + entity.addInfo(ATSConstants.PROCESSOR_CLASS_NAME, event.getProcessorName()); + if (event.getServicePluginInfo() != null) { + entity.addInfo(ATSConstants.SERVICE_PLUGIN, + DAGUtils.convertServicePluginToATSMap(event.getServicePluginInfo())); + } + return entity; + } + + private static TimelineEntity createVertexStartedEntity( + VertexStartedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getVertexID().toString(), + EntityTypes.TEZ_VERTEX_ID.name(), null); + + TimelineEvent tEvent = craeteBaseEvent(event.getEventType().name(), + event.getStartRequestedTime()); + entity.addEvent(tEvent); + + + entity.addInfo(ATSConstants.START_REQUESTED_TIME, event.getStartRequestedTime()); + entity.addInfo(ATSConstants.START_TIME, event.getStartTime()); + entity.addInfo(ATSConstants.STATUS, event.getVertexState().toString()); + + return entity; + } + + private static TimelineEntity createVertexFinishedEntity( + VertexFinishedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getVertexID().toString(), + EntityTypes.TEZ_VERTEX_ID.name(), null); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getFinishTime()); + entity.addEvent(tEvent); + entity.addInfo(ATSConstants.VERTEX_NAME, event.getVertexName()); + entity.addInfo(ATSConstants.FINISH_TIME, event.getFinishTime()); + entity.addInfo(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime())); + entity.addInfo(ATSConstants.STATUS, event.getState().name()); + + entity.addInfo(ATSConstants.DIAGNOSTICS, event.getDiagnostics()); + entity.addInfo(ATSConstants.COUNTERS, + DAGUtils.convertCountersToATSMap(event.getTezCounters())); + entity.addInfo(ATSConstants.STATS, + DAGUtils.convertVertexStatsToATSMap(event.getVertexStats())); + if (event.getServicePluginInfo() != null) { + entity.addInfo(ATSConstants.SERVICE_PLUGIN, + DAGUtils.convertServicePluginToATSMap(event.getServicePluginInfo())); + } + + final Map vertexTaskStats = event.getVertexTaskStats(); + if (vertexTaskStats != null) { + for(Entry entry : vertexTaskStats.entrySet()) { + entity.addInfo(entry.getKey(), entry.getValue()); + } + } + + return entity; + } + + private static TimelineEntity createTaskStartedEntity( + TaskStartedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getTaskID().toString(), + EntityTypes.TEZ_TASK_ID.name(), event.getStartTime()); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getStartTime()); + entity.addEvent(tEvent); + + entity.addIsRelatedToEntity(EntityTypes.TEZ_VERTEX_ID.name(), + event.getTaskID().getVertexID().toString()); + entity.addInfo(ATSConstants.START_TIME, event.getStartTime()); + entity.addInfo(ATSConstants.SCHEDULED_TIME, event.getScheduledTime()); + entity.addInfo(ATSConstants.STATUS, event.getState().name()); + + return entity; + } + + private static TimelineEntity createTaskFinishedEntity( + TaskFinishedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getTaskID().toString(), + EntityTypes.TEZ_TASK_ID.name(), null); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getFinishTime()); + entity.addEvent(tEvent); + entity.addInfo(ATSConstants.START_TIME, event.getStartTime()); + entity.addInfo(ATSConstants.FINISH_TIME, event.getFinishTime()); + entity.addInfo(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime())); + entity.addInfo(ATSConstants.STATUS, event.getState().name()); + entity.addInfo(ATSConstants.NUM_FAILED_TASKS_ATTEMPTS, event.getNumFailedAttempts()); + if (event.getSuccessfulAttemptID() != null) { + entity.addInfo(ATSConstants.SUCCESSFUL_ATTEMPT_ID, + event.getSuccessfulAttemptID().toString()); + } + + entity.addInfo(ATSConstants.DIAGNOSTICS, event.getDiagnostics()); + entity.addInfo(ATSConstants.COUNTERS, + DAGUtils.convertCountersToATSMap(event.getTezCounters())); + return entity; + } + + private static TimelineEntity createTaskAttemptStartedEntity( + TaskAttemptStartedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getTaskAttemptID().toString(), + EntityTypes.TEZ_TASK_ATTEMPT_ID.name(), event.getStartTime()); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getStartTime()); + entity.addEvent(tEvent); + + entity.addIsRelatedToEntity(EntityTypes.TEZ_TASK_ID.name(), + event.getTaskAttemptID().getTaskID().toString()); + + entity.addInfo(ATSConstants.START_TIME, event.getStartTime()); + if (event.getInProgressLogsUrl() != null) { + entity.addInfo(ATSConstants.IN_PROGRESS_LOGS_URL, event.getInProgressLogsUrl()); + } + if (event.getCompletedLogsUrl() != null) { + entity.addInfo(ATSConstants.COMPLETED_LOGS_URL, event.getCompletedLogsUrl()); + } + entity.addInfo(ATSConstants.NODE_ID, event.getNodeId().toString()); + entity.addInfo(ATSConstants.NODE_HTTP_ADDRESS, event.getNodeHttpAddress()); + entity.addInfo(ATSConstants.CONTAINER_ID, event.getContainerId().toString()); + entity.addInfo(ATSConstants.STATUS, TaskAttemptState.RUNNING.name()); + + + return entity; + } + + private static TimelineEntity createTaskAttemptFinishedEntity( + TaskAttemptFinishedEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getTaskAttemptID().toString(), + EntityTypes.TEZ_TASK_ATTEMPT_ID.name(), null); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getFinishTime()); + entity.addEvent(tEvent); + + if (event.getTaskFailureType() != null) { + entity.addInfo(ATSConstants.TASK_FAILURE_TYPE, event.getTaskFailureType().name()); + } + entity.addInfo(ATSConstants.CREATION_TIME, event.getCreationTime()); + entity.addInfo(ATSConstants.ALLOCATION_TIME, event.getAllocationTime()); + entity.addInfo(ATSConstants.START_TIME, event.getStartTime()); + entity.addInfo(ATSConstants.FINISH_TIME, event.getFinishTime()); + if (event.getCreationCausalTA() != null) { + entity.addInfo(ATSConstants.CREATION_CAUSAL_ATTEMPT, + event.getCreationCausalTA().toString()); + } + entity.addInfo(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime())); + entity.addInfo(ATSConstants.STATUS, event.getState().name()); + if (event.getTaskAttemptError() != null) { + entity.addInfo(ATSConstants.TASK_ATTEMPT_ERROR_ENUM, event.getTaskAttemptError().name()); + } + entity.addInfo(ATSConstants.DIAGNOSTICS, event.getDiagnostics()); + entity.addInfo(ATSConstants.COUNTERS, + DAGUtils.convertCountersToATSMap(event.getCounters())); + if (event.getDataEvents() != null && !event.getDataEvents().isEmpty()) { + entity.addInfo(ATSConstants.LAST_DATA_EVENTS, + DAGUtils.convertDataEventDependecyInfoToATS(event.getDataEvents())); + } + if (event.getNodeId() != null) { + entity.addInfo(ATSConstants.NODE_ID, event.getNodeId().toString()); + } + if (event.getContainerId() != null) { + entity.addInfo(ATSConstants.CONTAINER_ID, event.getContainerId().toString()); + } + if (event.getInProgressLogsUrl() != null) { + entity.addInfo(ATSConstants.IN_PROGRESS_LOGS_URL, event.getInProgressLogsUrl()); + } + if (event.getCompletedLogsUrl() != null) { + entity.addInfo(ATSConstants.COMPLETED_LOGS_URL, event.getCompletedLogsUrl()); + } + if (event.getNodeHttpAddress() != null) { + entity.addInfo(ATSConstants.NODE_HTTP_ADDRESS, event.getNodeHttpAddress()); + } + + return entity; + } + + private static TimelineEntity createVertexReconfigureDoneEntity( + VertexConfigurationDoneEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getVertexID().toString(), + EntityTypes.TEZ_VERTEX_ID.name(), null); + + TimelineEvent tEvent = craeteBaseEvent(event.getEventType().name(), + event.getReconfigureDoneTime()); + //entity.addEvent(tEvent); + + Map eventInfo = new HashMap(); + if (event.getSourceEdgeProperties() != null && !event.getSourceEdgeProperties().isEmpty()) { + Map updatedEdgeManagers = new HashMap(); + for (Entry entry : + event.getSourceEdgeProperties().entrySet()) { + updatedEdgeManagers.put(entry.getKey(), + DAGUtils.convertEdgeProperty(entry.getValue())); + } + eventInfo.put(ATSConstants.UPDATED_EDGE_MANAGERS, updatedEdgeManagers); + } + eventInfo.put(ATSConstants.NUM_TASKS, event.getNumTasks()); + tEvent.setInfo(eventInfo); + entity.addEvent(tEvent); + + entity.addInfo(ATSConstants.NUM_TASKS, event.getNumTasks()); + + return entity; + } + + private static TimelineEntity createDAGRecoveredEntity( + DAGRecoveredEvent event) { + TimelineEntity entity = + createBaseEntity(TEZ_PREFIX + event.getDagID().toString(), + EntityTypes.TEZ_DAG_ID.name(), null); + + TimelineEvent tEvent = + craeteBaseEvent(event.getEventType().name(), event.getRecoveredTime()); + entity.addEvent(tEvent); + + return entity; + } + + private static TimelineEntity createBaseEntity(String entityId, + String entityType, Long createdTime) { + TimelineEntity entity = new TimelineEntity(); + entity.setId(entityId); + entity.setType(entityType); + entity.setCreatedTime(createdTime); + return entity; + } + + private static TimelineEvent craeteBaseEvent(String eventType, + long createdTime) { + TimelineEvent tEvent = new TimelineEvent(); + tEvent.setId(eventType); + tEvent.setTimestamp(createdTime); + return tEvent; + } +} diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/LICENSE.txt b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/LICENSE.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/NOTICE.txt b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/NOTICE.txt new file mode 100644 index 0000000000..3f36fcc6ba --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/NOTICE.txt @@ -0,0 +1,6 @@ +Apache Tez +Copyright (c) 2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/LICENSE.txt b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/LICENSE.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/NOTICE.txt b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/NOTICE.txt new file mode 100644 index 0000000000..3f36fcc6ba --- /dev/null +++ b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/NOTICE.txt @@ -0,0 +1,6 @@ +Apache Tez +Copyright (c) 2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + From 5f8fe1e54b05f1656f067b378e4d3c595fe2fce4 Mon Sep 17 00:00:00 2001 From: divyapandian Date: Tue, 21 Sep 2021 07:08:22 +0000 Subject: [PATCH 2/3] Changes from TEZ-3820 patch Updated with Counters and Config publish in ATSv2 --- tez-dag/findbugs-exclude.xml | 267 ------------------ .../javadoc/resources/META-INF/LICENSE.txt | 202 ------------- .../javadoc/resources/META-INF/NOTICE.txt | 6 - .../src/main/resources/META-INF/LICENSE.txt | 202 ------------- .../src/main/resources/META-INF/NOTICE.txt | 6 - 5 files changed, 683 deletions(-) delete mode 100644 tez-dag/findbugs-exclude.xml delete mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/LICENSE.txt delete mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/NOTICE.txt delete mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/LICENSE.txt delete mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/NOTICE.txt diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/findbugs-exclude.xml deleted file mode 100644 index 50422ff0e0..0000000000 --- a/tez-dag/findbugs-exclude.xml +++ /dev/null @@ -1,267 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/LICENSE.txt b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/LICENSE.txt deleted file mode 100644 index d645695673..0000000000 --- a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/NOTICE.txt b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/NOTICE.txt deleted file mode 100644 index 3f36fcc6ba..0000000000 --- a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/javadoc/resources/META-INF/NOTICE.txt +++ /dev/null @@ -1,6 +0,0 @@ -Apache Tez -Copyright (c) 2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/LICENSE.txt b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/LICENSE.txt deleted file mode 100644 index d645695673..0000000000 --- a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/NOTICE.txt b/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/NOTICE.txt deleted file mode 100644 index 3f36fcc6ba..0000000000 --- a/tez-plugins/tez-yarn-timeline-history-with-atsv2/src/main/resources/META-INF/NOTICE.txt +++ /dev/null @@ -1,6 +0,0 @@ -Apache Tez -Copyright (c) 2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - From a2aef92d41ff7668bfe65294b9059af307b10d6d Mon Sep 17 00:00:00 2001 From: divyapandian Date: Tue, 21 Sep 2021 07:09:48 +0000 Subject: [PATCH 3/3] Changes from TEZ-3820 patch Updated with Counters and Config publish in ATSv2 --- .../findbugs-exclude.xml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 tez-plugins/tez-yarn-timeline-history-with-atsv2/findbugs-exclude.xml diff --git a/tez-plugins/tez-yarn-timeline-history-with-atsv2/findbugs-exclude.xml b/tez-plugins/tez-yarn-timeline-history-with-atsv2/findbugs-exclude.xml deleted file mode 100644 index 5b11308f6d..0000000000 --- a/tez-plugins/tez-yarn-timeline-history-with-atsv2/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - -