Skip to content

Commit eec9cd2

Browse files
ayushtknayushtkn
authored
HADOOP-19107. Drop support for HBase v1 & upgrade HBase v2 (apache#6629). Contributed by Ayush Saxena
1 parent a386ac1 commit eec9cd2

File tree

24 files changed

+82
-1947
lines changed

24 files changed

+82
-1947
lines changed

‎BUILDING.txt

+1-8
Original file line numberDiff line numberDiff line change
@@ -163,14 +163,7 @@ Maven build goals:
163163
YARN Application Timeline Service V2 build options:
164164

165165
YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage. The supported
166-
versions of Apache HBase are 1.7.1 (default) and 2.2.4.
167-
168-
* HBase 1.7.1 is used by default to build Hadoop. The official releases are ready to use if you
169-
plan on running Timeline Service v2 with HBase 1.7.1.
170-
171-
* Use -Dhbase.profile=2.0 to build Hadoop with HBase 2.2.4. Provide this option if you plan
172-
on running Timeline Service v2 with HBase 2.x.
173-
166+
version of Apache HBase is 2.5.8.
174167

175168
Snappy build options:
176169

‎LICENSE-binary

+4-4
Original file line numberDiff line numberDiff line change
@@ -310,10 +310,10 @@ org.apache.commons:commons-validator:1.6
310310
org.apache.curator:curator-client:5.2.0
311311
org.apache.curator:curator-framework:5.2.0
312312
org.apache.curator:curator-recipes:5.2.0
313-
org.apache.hbase:hbase-annotations:1.7.1
314-
org.apache.hbase:hbase-client:1.7.1
315-
org.apache.hbase:hbase-common:1.7.1
316-
org.apache.hbase:hbase-protocol:1.7.1
313+
org.apache.hbase:hbase-annotations:2.5.8
314+
org.apache.hbase:hbase-client:2.5.8
315+
org.apache.hbase:hbase-common:2.5.8
316+
org.apache.hbase:hbase-protocol:2.5.8
317317
org.apache.htrace:htrace-core:3.1.0-incubating
318318
org.apache.htrace:htrace-core4:4.1.0-incubating
319319
org.apache.httpcomponents:httpclient:4.5.13

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@
245245
</moduleSet>
246246
<moduleSet>
247247
<includes>
248-
<include>org.apache.hadoop:${hbase-server-artifactid}</include>
248+
<include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-server-2</include>
249249
</includes>
250250
<binaries>
251251
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory>

‎hadoop-project/pom.xml

+19-40
Original file line numberDiff line numberDiff line change
@@ -203,8 +203,7 @@
203203
<swagger-annotations-version>1.5.4</swagger-annotations-version>
204204
<snakeyaml.version>2.0</snakeyaml.version>
205205
<sshd.version>2.11.0</sshd.version>
206-
<hbase.one.version>1.7.1</hbase.one.version>
207-
<hbase.two.version>2.2.4</hbase.two.version>
206+
<hbase.version>2.5.8-hadoop3</hbase.version>
208207
<junit.version>4.13.2</junit.version>
209208
<junit.jupiter.version>5.8.2</junit.jupiter.version>
210209
<junit.vintage.version>5.8.2</junit.vintage.version>
@@ -501,6 +500,11 @@
501500
<version>${hadoop.version}</version>
502501
<type>test-jar</type>
503502
</dependency>
503+
<dependency>
504+
<groupId>org.apache.hadoop</groupId>
505+
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
506+
<version>${hadoop.version}</version>
507+
</dependency>
504508

505509
<dependency>
506510
<groupId>org.apache.hadoop</groupId>
@@ -1811,6 +1815,10 @@
18111815
<groupId>org.apache.yetus</groupId>
18121816
<artifactId>audience-annotations</artifactId>
18131817
</exclusion>
1818+
<exclusion>
1819+
<groupId>org.osgi</groupId>
1820+
<artifactId>org.osgi.core</artifactId>
1821+
</exclusion>
18141822
</exclusions>
18151823
</dependency>
18161824
<dependency>
@@ -1875,6 +1883,14 @@
18751883
<groupId>org.apache.yetus</groupId>
18761884
<artifactId>audience-annotations</artifactId>
18771885
</exclusion>
1886+
<exclusion>
1887+
<groupId>com.google.errorprone</groupId>
1888+
<artifactId>error_prone_annotations</artifactId>
1889+
</exclusion>
1890+
<exclusion>
1891+
<groupId>org.checkerframework</groupId>
1892+
<artifactId>checker-qual</artifactId>
1893+
</exclusion>
18781894
</exclusions>
18791895
</dependency>
18801896
<dependency>
@@ -2677,33 +2693,6 @@
26772693
</plugins>
26782694
</build>
26792695
</profile>
2680-
<!-- The profile for building against HBase 1.2.x
2681-
This is the default.
2682-
-->
2683-
<profile>
2684-
<id>hbase1</id>
2685-
<activation>
2686-
<property>
2687-
<name>!hbase.profile</name>
2688-
</property>
2689-
</activation>
2690-
<properties>
2691-
<hbase.version>${hbase.one.version}</hbase.version>
2692-
<hbase-compatible-hadoop.version>2.8.5</hbase-compatible-hadoop.version>
2693-
<hbase-compatible-guava.version>12.0.1</hbase-compatible-guava.version>
2694-
<hbase-compatible-guice.version>4.0</hbase-compatible-guice.version>
2695-
<hbase-server-artifactid>hadoop-yarn-server-timelineservice-hbase-server-1</hbase-server-artifactid>
2696-
</properties>
2697-
<dependencyManagement>
2698-
<dependencies>
2699-
<dependency>
2700-
<groupId>org.apache.hadoop</groupId>
2701-
<artifactId>${hbase-server-artifactid}</artifactId>
2702-
<version>${hadoop.version}</version>
2703-
</dependency>
2704-
</dependencies>
2705-
</dependencyManagement>
2706-
</profile>
27072696
<!-- The profile for building against HBase 2.0.0.
27082697
Activate using: mvn -Dhbase.profile=2.0
27092698
-->
@@ -2716,20 +2705,10 @@
27162705
</property>
27172706
</activation>
27182707
<properties>
2719-
<hbase.version>${hbase.two.version}</hbase.version>
2720-
<hbase-compatible-hadoop.version>2.8.5</hbase-compatible-hadoop.version>
2721-
<hbase-compatible-guava.version>11.0.2</hbase-compatible-guava.version>
2722-
<hbase-server-artifactid>hadoop-yarn-server-timelineservice-hbase-server-2</hbase-server-artifactid>
2723-
<hbase-compatible-guice.version>4.0</hbase-compatible-guice.version>
2724-
<hbase-compatible-jetty.version>9.3.27.v20190418</hbase-compatible-jetty.version>
27252708
</properties>
27262709
<dependencyManagement>
27272710
<dependencies>
2728-
<dependency>
2729-
<groupId>org.apache.hadoop</groupId>
2730-
<artifactId>${hbase-server-artifactid}</artifactId>
2731-
<version>${hadoop.version}</version>
2732-
</dependency>
2711+
27332712
</dependencies>
27342713
</dependencyManagement>
27352714
</profile>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml

+27-104
Original file line numberDiff line numberDiff line change
@@ -104,10 +104,6 @@
104104
<groupId>com.sun.jersey</groupId>
105105
<artifactId>jersey-json</artifactId>
106106
</exclusion>
107-
<exclusion>
108-
<groupId>commons-logging</groupId>
109-
<artifactId>commons-logging</artifactId>
110-
</exclusion>
111107
</exclusions>
112108
</dependency>
113109

@@ -182,19 +178,6 @@
182178
<scope>test</scope>
183179
</dependency>
184180

185-
<dependency>
186-
<groupId>com.google.guava</groupId>
187-
<artifactId>guava</artifactId>
188-
<version>${hbase-compatible-guava.version}</version>
189-
<scope>test</scope>
190-
</dependency>
191-
192-
<dependency>
193-
<groupId>com.google.inject</groupId>
194-
<artifactId>guice</artifactId>
195-
<version>${hbase-compatible-guice.version}</version>
196-
<scope>test</scope>
197-
</dependency>
198181

199182
<dependency>
200183
<groupId>org.xerial.snappy</groupId>
@@ -393,13 +376,6 @@
393376
<scope>test</scope>
394377
</dependency>
395378

396-
<dependency>
397-
<groupId>org.jmockit</groupId>
398-
<artifactId>jmockit</artifactId>
399-
<version>1.24</version>
400-
<scope>test</scope>
401-
</dependency>
402-
403379
<dependency>
404380
<groupId>org.assertj</groupId>
405381
<artifactId>assertj-core</artifactId>
@@ -454,6 +430,25 @@
454430
<artifactId>commons-lang3</artifactId>
455431
<scope>test</scope>
456432
</dependency>
433+
<dependency>
434+
<groupId>org.apache.hadoop</groupId>
435+
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
436+
<scope>test</scope>
437+
<exclusions>
438+
<exclusion>
439+
<groupId>org.apache.hadoop</groupId>
440+
<artifactId>hadoop-common</artifactId>
441+
</exclusion>
442+
</exclusions>
443+
</dependency>
444+
<!-- 'mvn dependency:analyze' fails to detect use of this direct
445+
dependency -->
446+
<!-- This is needed by HBaseTestingUtility -->
447+
<dependency>
448+
<groupId>org.mockito</groupId>
449+
<artifactId>mockito-core</artifactId>
450+
<scope>test</scope>
451+
</dependency>
457452
</dependencies>
458453

459454
<build>
@@ -494,86 +489,14 @@
494489
</execution>
495490
</executions>
496491
</plugin>
492+
<!-- The fork value is deliberately set to 0 to avoid VM crash while running tests
493+
on Jenkins, removing this leads to tests crashing silently due to VM crash -->
494+
<plugin>
495+
<artifactId>maven-surefire-plugin</artifactId>
496+
<configuration>
497+
<forkCount>0</forkCount>
498+
</configuration>
499+
</plugin>
497500
</plugins>
498501
</build>
499-
500-
<profiles>
501-
<profile>
502-
<id>hbase1</id>
503-
<activation>
504-
<property>
505-
<name>!hbase.profile</name>
506-
</property>
507-
</activation>
508-
<dependencies>
509-
<dependency>
510-
<groupId>org.apache.hadoop</groupId>
511-
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-1</artifactId>
512-
<scope>test</scope>
513-
<exclusions>
514-
<exclusion>
515-
<groupId>org.apache.hadoop</groupId>
516-
<artifactId>hadoop-common</artifactId>
517-
</exclusion>
518-
</exclusions>
519-
</dependency>
520-
</dependencies>
521-
</profile>
522-
523-
<profile>
524-
<id>hbase2</id>
525-
<activation>
526-
<property>
527-
<name>hbase.profile</name>
528-
<value>2.0</value>
529-
</property>
530-
</activation>
531-
<dependencies>
532-
<dependency>
533-
<groupId>org.apache.hadoop</groupId>
534-
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
535-
<scope>test</scope>
536-
<exclusions>
537-
<exclusion>
538-
<groupId>org.apache.hadoop</groupId>
539-
<artifactId>hadoop-common</artifactId>
540-
</exclusion>
541-
</exclusions>
542-
</dependency>
543-
<!-- 'mvn dependency:analyze' fails to detect use of this direct
544-
dependency -->
545-
<!-- This is needed by HBaseTestingUtility -->
546-
<dependency>
547-
<groupId>org.mockito</groupId>
548-
<artifactId>mockito-core</artifactId>
549-
<scope>test</scope>
550-
</dependency>
551-
<dependency>
552-
<groupId>org.eclipse.jetty</groupId>
553-
<artifactId>jetty-server</artifactId>
554-
<scope>test</scope>
555-
<version>${hbase-compatible-jetty.version}</version>
556-
</dependency>
557-
<dependency>
558-
<groupId>org.eclipse.jetty</groupId>
559-
<artifactId>jetty-servlet</artifactId>
560-
<scope>test</scope>
561-
<version>${hbase-compatible-jetty.version}</version>
562-
</dependency>
563-
<dependency>
564-
<groupId>org.eclipse.jetty</groupId>
565-
<artifactId>jetty-webapp</artifactId>
566-
<scope>test</scope>
567-
<version>${hbase-compatible-jetty.version}</version>
568-
</dependency>
569-
<dependency>
570-
<groupId>org.eclipse.jetty</groupId>
571-
<artifactId>jetty-util</artifactId>
572-
<scope>test</scope>
573-
<version>${hbase-compatible-jetty.version}</version>
574-
</dependency>
575-
</dependencies>
576-
</profile>
577-
578-
</profiles>
579502
</project>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/DummyTimelineReaderMetrics.java

-39
This file was deleted.

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java

-11
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@
3737

3838
import javax.ws.rs.core.MediaType;
3939

40-
import mockit.Mock;
41-
import mockit.MockUp;
4240
import org.apache.hadoop.conf.Configuration;
4341
import org.apache.hadoop.security.UserGroupInformation;
4442
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -54,7 +52,6 @@
5452
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
5553
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
5654
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
57-
import org.apache.hadoop.yarn.server.timelineservice.metrics.TimelineReaderMetrics;
5855
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
5956
import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
6057
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -77,17 +74,9 @@ public class TestTimelineReaderWebServicesHBaseStorage
7774
private static long dayTs =
7875
HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
7976
private static String doAsUser = "remoteuser";
80-
private static final DummyTimelineReaderMetrics METRICS
81-
= new DummyTimelineReaderMetrics();
8277

8378
@BeforeClass
8479
public static void setupBeforeClass() throws Exception {
85-
new MockUp<TimelineReaderMetrics>() {
86-
@Mock
87-
public TimelineReaderMetrics getInstance() {
88-
return METRICS;
89-
}
90-
};
9180
setup();
9281
loadData();
9382
initialize();

0 commit comments

Comments
 (0)