diff --git a/hadoop-tools/hadoop-gcp/src/site/markdown/tools/hadoop-gcp/testing.md b/hadoop-tools/hadoop-gcp/src/site/markdown/tools/hadoop-gcp/testing.md new file mode 100644 index 0000000000000..a56d7e6c395f8 --- /dev/null +++ b/hadoop-tools/hadoop-gcp/src/site/markdown/tools/hadoop-gcp/testing.md @@ -0,0 +1,70 @@ + + +# Testing the GCS filesystem client and its features + + + +This module includes both unit tests, which can run in isolation without +connecting to the GCS service, and integration tests, which require a working +connection to GCS to interact with a bucket. Unit test suites follow the naming +convention `Test*.java`. Integration tests follow the naming convention +`ITest*.java`. + +## Setting up the tests + +To integration test the GCS filesystem client, you need to provide +`auth-keys.xml` which passes in authentication details to the test runner. + +It is a Hadoop XML configuration file, which must be placed into +`hadoop-tools/hadoop-gcp/src/test/resources`. + +### File `core-site.xml` + +This file pre-exists and sources the configurations created +under `auth-keys.xml`. + +For most purposes you will not need to edit this file unless you +need to apply a specific, non-default property change during the tests. + +### File `auth-keys.xml` + +The presence of this file triggers the testing of the GCS classes. + +Without this file, *none of the integration tests in this module will be +executed*. + +* `fs.contract.test.fs.gs` : the URL of the bucket for GCS filesystem contract tests + +Example: + +```xml + + + fs.contract.test.fs.gs + gs://your bucket name + + + +``` + +## Running the Tests + +After completing the configuration, execute the test run through Maven. + +This has to be run from a GCP VM. This limitation will be removed later. + +```bash +mvn clean verify +``` \ No newline at end of file diff --git a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/contract/GoogleContract.java b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/contract/GoogleContract.java index aa131981caf2b..af65e2c516be8 100644 --- a/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/contract/GoogleContract.java +++ b/hadoop-tools/hadoop-gcp/src/test/java/org/apache/hadoop/fs/gs/contract/GoogleContract.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs.gs.contract; -import org.apache.hadoop.fs.gs.TestConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; @@ -29,12 +28,6 @@ public class GoogleContract extends AbstractBondedFSContract { public GoogleContract(Configuration conf) { super(conf); addConfResource(CONTRACT_XML); - conf.set("fs.contract.test.fs.gs", "gs://arunchacko-oss-test-bucket"); // TODO: - - TestConfiguration testConf = TestConfiguration.getInstance(); - if (testConf.getProjectId() != null) { - conf.set("fs.gs.project.id", testConf.getProjectId()); - } } @Override diff --git a/hadoop-tools/hadoop-gcp/src/test/resources/contract/gs.xml b/hadoop-tools/hadoop-gcp/src/test/resources/contract/gs.xml new file mode 100644 index 0000000000000..1de34245a5d1a --- /dev/null +++ b/hadoop-tools/hadoop-gcp/src/test/resources/contract/gs.xml @@ -0,0 +1,21 @@ + + + + +