Skip to content

HADOOP-19107. Drop support for HBase v1 timeline service & upgrade HBase v2 (#6629). #7017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
steveloughran merged 3 commits into from
Oct 9, 2024
Merged

HADOOP-19107. Drop support for HBase v1 timeline service & upgrade HBase v2 (#6629). #7017

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
42ba5d1
HADOOP-19107. Drop support for HBase v1 & upgrade HBase v2 (#6629).
ayushtkn Apr 22, 2024
9fad755
Attempt fix spotbugs & remove unrelated changes.
ayushtkn Aug 31, 2024
2fc01e1
Remove yarn ui related change.
ayushtkn Aug 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions BUILDING.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,14 +163,7 @@ Maven build goals:
YARN Application Timeline Service V2 build options:

YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage. The supported
versions of Apache HBase are 1.7.1 (default) and 2.2.4.

* HBase 1.7.1 is used by default to build Hadoop. The official releases are ready to use if you
plan on running Timeline Service v2 with HBase 1.7.1.

* Use -Dhbase.profile=2.0 to build Hadoop with HBase 2.2.4. Provide this option if you plan
on running Timeline Service v2 with HBase 2.x.

version of Apache HBase is 2.5.8.

Snappy build options:

Expand Down
8 changes: 4 additions & 4 deletions LICENSE-binary
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,10 +309,10 @@ org.apache.commons:commons-validator:1.6
org.apache.curator:curator-client:5.2.0
org.apache.curator:curator-framework:5.2.0
org.apache.curator:curator-recipes:5.2.0
org.apache.hbase:hbase-annotations:1.7.1
org.apache.hbase:hbase-client:1.7.1
org.apache.hbase:hbase-common:1.7.1
org.apache.hbase:hbase-protocol:1.7.1
org.apache.hbase:hbase-annotations:2.5.8
org.apache.hbase:hbase-client:2.5.8
org.apache.hbase:hbase-common:2.5.8
org.apache.hbase:hbase-protocol:2.5.8
org.apache.htrace:htrace-core:3.1.0-incubating
org.apache.htrace:htrace-core4:4.1.0-incubating
org.apache.httpcomponents:httpclient:4.5.13
Expand Down
2 changes: 1 addition & 1 deletion hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@
</moduleSet>
<moduleSet>
<includes>
<include>org.apache.hadoop:${hbase-server-artifactid}</include>
<include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-server-2</include>
</includes>
<binaries>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory>
Expand Down
59 changes: 19 additions & 40 deletions hadoop-project/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,7 @@
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>2.0</snakeyaml.version>
<sshd.version>2.11.0</sshd.version>
<hbase.one.version>1.7.1</hbase.one.version>
<hbase.two.version>2.2.4</hbase.two.version>
<hbase.version>2.5.8-hadoop3</hbase.version>
<junit.version>4.13.2</junit.version>
<junit.jupiter.version>5.8.2</junit.jupiter.version>
<junit.vintage.version>5.8.2</junit.vintage.version>
Expand Down Expand Up @@ -501,6 +500,11 @@
<version>${hadoop.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
<version>${hadoop.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
Expand Down Expand Up @@ -1807,6 +1811,10 @@
<groupId>org.apache.yetus</groupId>
<artifactId>audience-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>org.osgi</groupId>
<artifactId>org.osgi.core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -1871,6 +1879,14 @@
<groupId>org.apache.yetus</groupId>
<artifactId>audience-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
</exclusion>
<exclusion>
<groupId>org.checkerframework</groupId>
<artifactId>checker-qual</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -2678,33 +2694,6 @@
</plugins>
</build>
</profile>
<!-- The profile for building against HBase 1.2.x
This is the default.
-->
<profile>
<id>hbase1</id>
<activation>
<property>
<name>!hbase.profile</name>
</property>
</activation>
<properties>
<hbase.version>${hbase.one.version}</hbase.version>
<hbase-compatible-hadoop.version>2.8.5</hbase-compatible-hadoop.version>
<hbase-compatible-guava.version>12.0.1</hbase-compatible-guava.version>
<hbase-compatible-guice.version>4.0</hbase-compatible-guice.version>
<hbase-server-artifactid>hadoop-yarn-server-timelineservice-hbase-server-1</hbase-server-artifactid>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>${hbase-server-artifactid}</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
</profile>
<!-- The profile for building against HBase 2.0.0.
Activate using: mvn -Dhbase.profile=2.0
-->
Expand All @@ -2717,20 +2706,10 @@
</property>
</activation>
<properties>
<hbase.version>${hbase.two.version}</hbase.version>
<hbase-compatible-hadoop.version>2.8.5</hbase-compatible-hadoop.version>
<hbase-compatible-guava.version>11.0.2</hbase-compatible-guava.version>
<hbase-server-artifactid>hadoop-yarn-server-timelineservice-hbase-server-2</hbase-server-artifactid>
<hbase-compatible-guice.version>4.0</hbase-compatible-guice.version>
<hbase-compatible-jetty.version>9.3.27.v20190418</hbase-compatible-jetty.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>${hbase-server-artifactid}</artifactId>
<version>${hadoop.version}</version>
</dependency>

</dependencies>
</dependencyManagement>
</profile>
Expand Down
131 changes: 27 additions & 104 deletions ...ect/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,6 @@
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand Down Expand Up @@ -182,19 +178,6 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${hbase-compatible-guava.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>${hbase-compatible-guice.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.xerial.snappy</groupId>
Expand Down Expand Up @@ -393,13 +376,6 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.jmockit</groupId>
<artifactId>jmockit</artifactId>
<version>1.24</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
Expand Down Expand Up @@ -454,6 +430,25 @@
<artifactId>commons-lang3</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this direct
dependency -->
<!-- This is needed by HBaseTestingUtility -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -494,86 +489,14 @@
</execution>
</executions>
</plugin>
<!-- The fork value is deliberately set to 0 to avoid VM crash while running tests
on Jenkins, removing this leads to tests crashing silently due to VM crash -->
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkCount>0</forkCount>
</configuration>
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>hbase1</id>
<activation>
<property>
<name>!hbase.profile</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-1</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>

<profile>
<id>hbase2</id>
<activation>
<property>
<name>hbase.profile</name>
<value>2.0</value>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this direct
dependency -->
<!-- This is needed by HBaseTestingUtility -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<scope>test</scope>
<version>${hbase-compatible-jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<scope>test</scope>
<version>${hbase-compatible-jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
<scope>test</scope>
<version>${hbase-compatible-jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<scope>test</scope>
<version>${hbase-compatible-jetty.version}</version>
</dependency>
</dependencies>
</profile>

</profiles>
</project>
39 changes: 0 additions & 39 deletions ...java/org/apache/hadoop/yarn/server/timelineservice/reader/DummyTimelineReaderMetrics.java
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions .../hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,6 @@

import javax.ws.rs.core.MediaType;

import mockit.Mock;
import mockit.MockUp;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
Expand All @@ -54,7 +52,6 @@
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
import org.apache.hadoop.yarn.server.timelineservice.metrics.TimelineReaderMetrics;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
Expand All @@ -77,17 +74,9 @@ public class TestTimelineReaderWebServicesHBaseStorage
private static long dayTs =
HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
private static String doAsUser = "remoteuser";
private static final DummyTimelineReaderMetrics METRICS
= new DummyTimelineReaderMetrics();

@BeforeClass
public static void setupBeforeClass() throws Exception {
new MockUp<TimelineReaderMetrics>() {
@Mock
public TimelineReaderMetrics getInstance() {
return METRICS;
}
};
setup();
loadData();
initialize();
Expand Down
Loading