Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
3b0169a
HDDS-4936. Change ozone maven groupId from org.apache.hadoop to org.a…
ChenSammi Mar 10, 2021
e80870e
HDDS-5008. Remove hadoop from ozone module artifactId.
ChenSammi Mar 23, 2021
81334ed
Fix checkstyle
adoroszlai Mar 23, 2021
0f7bf9f
Update module names in coverage.sh
adoroszlai Mar 23, 2021
715a956
Fix GH workflow
adoroszlai Mar 23, 2021
2a24161
Fix module names
adoroszlai Mar 23, 2021
57157df
Fix ozone-manager artifact name
adoroszlai Mar 23, 2021
98b6a07
Fix IDEA run configs
adoroszlai Mar 23, 2021
665539f
HDDS-4936. Change ozone maven groupId from org.apache.hadoop to org.a…
ChenSammi Mar 10, 2021
4346326
HDDS-5008. Remove hadoop from ozone module artifactId.
ChenSammi Mar 23, 2021
9c91aa4
Merge commit '98b6a07c2b5f4af2b003f68f2b4e5d2f6a27d37c' into HEAD
elek Mar 31, 2021
2773207
Merge remote-tracking branch 'origin/master' into HDDS-4936
adoroszlai Apr 14, 2021
a458e10
Merge remote-tracking branch 'origin/master' into HDDS-4936-3
ChenSammi Apr 21, 2021
2bf6d46
Merge remote-tracking branch 'origin/master' into HDDS-4936
adoroszlai Apr 22, 2021
f7c3887
Merge remote-tracking branch 'origin/master' into HDDS-4936
adoroszlai May 12, 2021
7487165
Fix tarball name for dependency check
adoroszlai May 12, 2021
e53ee99
Fix update-jar-report.sh
adoroszlai May 12, 2021
51ae145
Merge remote-tracking branch 'origin/master' into HDDS-4936
adoroszlai May 13, 2021
7161036
Merge remote-tracking branch 'origin/master' into HDDS-4936-05-27-1
ChenSammi May 27, 2021
7333bc3
Fix tarball name in kubernetes check
adoroszlai May 27, 2021
4abff67
Fix path in cache cleanup
adoroszlai May 27, 2021
7d5a6fe
trigger new CI check
adoroszlai May 27, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions .github/workflows/post-commit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,13 @@ jobs:
if: matrix.java == '8'
with:
name: ozone-bin
path: hadoop-ozone/dist/target/hadoop-ozone*.tar.gz
path: hadoop-ozone/dist/target/ozone*.tar.gz
retention-days: 1
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds*
rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone*
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
basic:
runs-on: ubuntu-18.04
Expand Down Expand Up @@ -113,8 +113,8 @@ jobs:
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds*
rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone*
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
dependency:
needs: compile
Expand All @@ -131,7 +131,7 @@ jobs:
- name: Untar binaries
run: |
mkdir dist
tar -C dist --strip-components 1 -xzf hadoop-ozone*.tar.gz
tar -C dist --strip-components 1 -xzf ozone*.tar.gz
- name: Execute tests
run: |
export OZONE_DIST_DIR=`pwd`/dist
Expand Down Expand Up @@ -171,7 +171,7 @@ jobs:
- name: Untar binaries
run: |
mkdir -p /mnt/ozone/hadoop-ozone/dist/target
tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target
tar xzvf ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target
sudo chmod -R a+rwX /mnt/ozone/hadoop-ozone/dist/target
- name: Execute tests
run: |
Expand Down Expand Up @@ -210,7 +210,7 @@ jobs:
- name: Untar binaries
run: |
mkdir -p /mnt/ozone/hadoop-ozone/dist/target
tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target
tar xzvf ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target
- name: Execute tests
run: |
cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
Expand Down Expand Up @@ -264,8 +264,8 @@ jobs:
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds*
rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone*
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
coverage:
runs-on: ubuntu-18.04
Expand Down Expand Up @@ -294,7 +294,7 @@ jobs:
- name: Untar binaries
run: |
mkdir -p hadoop-ozone/dist/target
tar xzvf target/artifacts/ozone-bin/hadoop-ozone*.tar.gz -C hadoop-ozone/dist/target
tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target
- name: Calculate combined coverage
run: ./hadoop-ozone/dev-support/checks/coverage.sh
- name: Setup java 11
Expand All @@ -315,6 +315,6 @@ jobs:
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds*
rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone*
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
22 changes: 11 additions & 11 deletions hadoop-hdds/client/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -18,30 +18,30 @@
https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds</artifactId>
<version>1.2.0-SNAPSHOT</version>
</parent>

<artifactId>hadoop-hdds-client</artifactId>
<artifactId>hdds-client</artifactId>
<version>1.2.0-SNAPSHOT</version>
<description>Apache Ozone Distributed Data Store Client Library</description>
<name>Apache Ozone HDDS Client</name>
<packaging>jar</packaging>

<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-test-utils</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-test-utils</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-common</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
Expand All @@ -52,8 +52,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
</dependency>

Expand Down
26 changes: 13 additions & 13 deletions hadoop-hdds/common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,20 @@
https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds</artifactId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-hdds-common</artifactId>
<artifactId>hdds-common</artifactId>
<version>1.2.0-SNAPSHOT</version>
<description>Apache Ozone Distributed Data Store Common</description>
<name>Apache Ozone HDDS Common</name>
<packaging>jar</packaging>

<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-hadoop-dependency-client</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-hadoop-dependency-client</artifactId>
</dependency>
<dependency>
<groupId>info.picocli</groupId>
Expand Down Expand Up @@ -66,12 +66,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-config</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-config</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-test-utils</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-test-utils</artifactId>
<scope>test</scope>
</dependency>

Expand Down Expand Up @@ -188,12 +188,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-interface-client</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-interface-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-interface-admin</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-interface-admin</artifactId>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
Expand Down
10 changes: 5 additions & 5 deletions hadoop-hdds/config/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds</artifactId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-hdds-config</artifactId>
<artifactId>hdds-config</artifactId>
<version>1.2.0-SNAPSHOT</version>
<description>Apache Ozone Distributed Data Store Config Tools</description>
<name>Apache Ozone HDDS Config</name>
Expand All @@ -38,8 +38,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-test-utils</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-test-utils</artifactId>
<scope>test</scope>
</dependency>

Expand Down
42 changes: 21 additions & 21 deletions hadoop-hdds/container-service/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -18,43 +18,43 @@
https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds</artifactId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-hdds-container-service</artifactId>
<artifactId>hdds-container-service</artifactId>
<version>1.2.0-SNAPSHOT</version>
<description>Apache Ozone Distributed Data Store Container Service</description>
<name>Apache Ozone HDDS Container Service</name>
<packaging>jar</packaging>

<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-test-utils</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-test-utils</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-common</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-server-framework</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-client</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-client</artifactId>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
Expand All @@ -77,8 +77,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-docs</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-docs</artifactId>
<scope>provided</scope>
</dependency>

Expand All @@ -98,8 +98,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-hadoop-dependency-test</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-hadoop-dependency-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -152,15 +152,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-server-framework</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-server-framework</artifactId>
<outputDirectory>${project.build.outputDirectory}
</outputDirectory>
<includes>webapps/static/**/*.*</includes>
</artifactItem>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-docs</artifactId>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-docs</artifactId>
<outputDirectory>${project.build.outputDirectory}/webapps/hddsDatanode</outputDirectory>
<includes>docs/**/*.*</includes>
</artifactItem>
Expand Down
4 changes: 2 additions & 2 deletions hadoop-hdds/docs/content/interface/O3fs.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ This will make this bucket to be the default Hadoop compatible file system and r
You also need to add the ozone-filesystem-hadoop3.jar file to the classpath:

{{< highlight bash >}}
export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-hadoop3-*.jar:$HADOOP_CLASSPATH
export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/ozone-filesystem-hadoop3-*.jar:$HADOOP_CLASSPATH
{{< /highlight >}}

(Note: with Hadoop 2.x, use the `hadoop-ozone-filesystem-hadoop2-*.jar`)
(Note: with Hadoop 2.x, use the `ozone-filesystem-hadoop2-*.jar`)

Once the default Filesystem has been setup, users can run commands like ls, put, mkdir, etc.
For example,
Expand Down
8 changes: 4 additions & 4 deletions hadoop-hdds/docs/content/interface/O3fs.zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,10 @@ ozone sh bucket create /volume/bucket
你还需要将 ozone-filesystem.jar 文件加入 classpath:

{{< highlight bash >}}
export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-hadoop3-*.jar:$HADOOP_CLASSPATH
export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/ozone-filesystem-hadoop3-*.jar:$HADOOP_CLASSPATH
{{< /highlight >}}

(注意:当使用Hadoop 2.x时,应该在classpath上添加hadoop-ozone-filesystem-hadoop2-*.jar)
(注意:当使用Hadoop 2.x时,应该在classpath上添加ozone-filesystem-hadoop2-*.jar)

当配置了默认的文件系统之后,用户可以运行 ls、put、mkdir 等命令,比如:

Expand Down Expand Up @@ -137,10 +137,10 @@ hdfs dfs -ls o3fs://bucket.volume.om-host.example.com:6789/key
你还需要将 ozone-filesystem.jar 文件加入 classpath:

{{< highlight bash >}}
export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-hadoop3-*.jar:$HADOOP_CLASSPATH
export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/ozone-filesystem-hadoop3-*.jar:$HADOOP_CLASSPATH
{{< /highlight >}}

(注意:当使用Hadoop 2.x时,应该在classpath上添加hadoop-ozone-filesystem-hadoop2-*.jar)
(注意:当使用Hadoop 2.x时,应该在classpath上添加ozone-filesystem-hadoop2-*.jar)

当配置了默认的文件系统之后,用户可以运行 ls、put、mkdir 等命令,比如:

Expand Down
4 changes: 2 additions & 2 deletions hadoop-hdds/docs/content/interface/Ofs.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ This will make all the volumes and buckets to be the default Hadoop compatible f
You also need to add the ozone-filesystem-hadoop3.jar file to the classpath:

{{< highlight bash >}}
export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-hadoop3-*.jar:$HADOOP_CLASSPATH
export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/ozone-filesystem-hadoop3-*.jar:$HADOOP_CLASSPATH
{{< /highlight >}}

(Note: with Hadoop 2.x, use the `hadoop-ozone-filesystem-hadoop2-*.jar`)
(Note: with Hadoop 2.x, use the `ozone-filesystem-hadoop2-*.jar`)

Once the default Filesystem has been setup, users can run commands like ls, put, mkdir, etc.
For example:
Expand Down
6 changes: 3 additions & 3 deletions hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ And create a custom `core-site.xml`.
Copy the `ozonefs.jar` file from an ozone distribution (__use the hadoop2 version!__)

```
kubectl cp om-0:/opt/hadoop/share/ozone/lib/hadoop-ozone-filesystem-hadoop2-VERSION.jar hadoop-ozone-filesystem-hadoop2.jar
kubectl cp om-0:/opt/hadoop/share/ozone/lib/ozone-filesystem-hadoop2-VERSION.jar ozone-filesystem-hadoop2.jar
```


Expand All @@ -93,7 +93,7 @@ ADD core-site.xml /opt/hadoop/conf/core-site.xml
ADD ozone-site.xml /opt/hadoop/conf/ozone-site.xml
ENV HADOOP_CONF_DIR=/opt/hadoop/conf
ENV SPARK_EXTRA_CLASSPATH=/opt/hadoop/conf
ADD hadoop-ozone-filesystem-hadoop2.jar /opt/hadoop-ozone-filesystem-hadoop2.jar
ADD ozone-filesystem-hadoop2.jar /opt/ozone-filesystem-hadoop2.jar
```

```bash
Expand Down Expand Up @@ -142,7 +142,7 @@ bin/spark-submit \
--conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
--conf spark.kubernetes.container.image=myrepo/spark-ozone \
--conf spark.kubernetes.container.image.pullPolicy=Always \
--jars /opt/hadoop-ozone-filesystem-hadoop2.jar \
--jars /opt/ozone-filesystem-hadoop2.jar \
local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar \
o3fs://test.s3v.ozone-om-0.ozone-om:9862/alice.txt
```
Expand Down
Loading