From 50f106d4c50884ce471a66c00df322270fe4a91c Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Fri, 24 Jan 2025 01:23:32 -0800 Subject: [PATCH] feat: Add firestoreInDatastoreMode for datastore emulator (#1698) * feat: Add firestoreInDatastoreMode for datastore emulator * chore: generate libraries at Mon Jan 6 17:28:37 UTC 2025 * update emulator min version * update version to 2.2.2 * update version to 2.3.0 * update version to 2.2.1 * update version to 2.3.1 * chore: generate libraries at Mon Jan 13 23:32:38 UTC 2025 * Fix ci testing environment with java 11 * Add Java 11 enviroment to run tests * Fix java env formatting * set maven env * set maven env * testing java 11 * Add sample test env variable * Add sample test env variable * Add sample test env variable * run with java8 but build with 11 * Add java 11 as java_home sample build * Add java_home for presubmit * Fix SUREFIRE_JVM_OPT for sample build * Fix JAVA_HOME for sample build * Fix JAVA_HOME for sample build * Modify Java env for presubmit integration build * Add setjava to builder script * Add setjava to builder script * Add setjava to builder script * Add debugging info to builder script * Add debugging info to builder script * Add debugging info to builder script * Add debugging info to builder script * formatting * Add debugging * Add debugging * Add debugging * Add debugging * Add debugging * Add debugging * Resolve conflicts * Change sample build to use java11 container * chore: generate libraries at Thu Jan 23 23:20:47 UTC 2025 --------- Co-authored-by: cloud-java-bot --- .cloudbuild/samples_build.yaml | 10 ++--- .github/workflows/ci.yaml | 10 ++++- .kokoro/build.sh | 12 ++++++ .kokoro/common.sh | 2 +- .kokoro/presubmit/integration.cfg | 2 +- .../testing/LocalDatastoreHelper.java | 38 +++++++++++++++++-- .../testing/ITLocalDatastoreHelperTest.java | 24 ++++++++++++ 7 files changed, 87 insertions(+), 11 deletions(-) diff --git a/.cloudbuild/samples_build.yaml b/.cloudbuild/samples_build.yaml index 204a7b254..3dd2ff94c 100644 --- a/.cloudbuild/samples_build.yaml +++ b/.cloudbuild/samples_build.yaml @@ -1,19 +1,19 @@ steps: -- name: gcr.io/cloud-devrel-public-resources/java8 +- name: gcr.io/cloud-devrel-public-resources/java11 entrypoint: ls args: [ '-alt', ] -- name: gcr.io/cloud-devrel-public-resources/java8 +- name: gcr.io/cloud-devrel-public-resources/java11 entrypoint: curl args: [ '--header', 'Metadata-Flavor: Google', 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email' ] -- name: gcr.io/cloud-devrel-public-resources/java8 +- name: gcr.io/cloud-devrel-public-resources/java11 entrypoint: pwd -- name: gcr.io/cloud-devrel-public-resources/java8 +- name: gcr.io/cloud-devrel-public-resources/java11 entrypoint: bash args: [ '.kokoro/build.sh' @@ -22,7 +22,7 @@ steps: - 'JOB_TYPE=samples' - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-sample' - 'KOKORO_GITHUB_PULL_REQUEST_NUMBER=$_PR_NUMBER' -- name: gcr.io/cloud-devrel-public-resources/java8 +- name: gcr.io/cloud-devrel-public-resources/java11 entrypoint: echo args: [ 'Sample job succeeded', diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b91fa381f..941f2f3b4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -59,6 +59,7 @@ jobs: env: JOB_TYPE: test windows: + # Building using Java 11 and run the tests with Java 8 runtime runs-on: windows-latest steps: - name: Support longpaths @@ -66,8 +67,15 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: - distribution: temurin java-version: 8 + distribution: temurin + - name: "Set jvm system property environment variable for surefire plugin (unit tests)" + run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}\bin\java" >> $GITHUB_ENV + shell: bash + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 - run: java -version - run: .kokoro/build.bat env: diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 8a894939c..c4b65f8b5 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -60,6 +60,18 @@ javadoc) RETURN_CODE=$? ;; integration) + # Kokoro integration tests use both JDK 11 and JDK 8. Integration + # tests require JDK 11 export as JAVA env variable to run cloud datastore + # emulator (https://cloud.google.com/sdk/docs/release-notes#39300_2022-07-12). + # For Java 8 environment, we will still run the tests using Java 8 with + # SUREFIRE_JVM_OPT for Maven surefire plugin: + # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm + if [[ -n "${JAVA11_HOME}" && -n "${JAVA8_HOME}" ]] + then + export JAVA=${JAVA11_HOME}/bin/java + export SUREFIRE_JVM_OPT=-Djvm=${JAVA8_HOME}/bin/java + fi + mvn -B ${INTEGRATION_TEST_ARGS} \ -ntp \ -Penable-integration-tests \ diff --git a/.kokoro/common.sh b/.kokoro/common.sh index f8f957af1..aeb3568bb 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -52,7 +52,7 @@ function retry_with_backoff { return $exit_code } -## Helper functionss +## Helper functions function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; } function msg() { println "$*" >&2; } function println() { printf '%s\n' "$(now) $*"; } diff --git a/.kokoro/presubmit/integration.cfg b/.kokoro/presubmit/integration.cfg index fd5cd6689..71431d938 100644 --- a/.kokoro/presubmit/integration.cfg +++ b/.kokoro/presubmit/integration.cfg @@ -35,4 +35,4 @@ env_vars: { env_vars: { key: "SECRET_MANAGER_KEYS" value: "java-it-service-account" -} +} \ No newline at end of file diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/LocalDatastoreHelper.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/LocalDatastoreHelper.java index 79d10220c..20321f812 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/LocalDatastoreHelper.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/LocalDatastoreHelper.java @@ -53,12 +53,13 @@ public class LocalDatastoreHelper extends BaseEmulatorHelper { private final double consistency; private final Path gcdPath; private boolean storeOnDisk; + private boolean firestoreInDatastoreMode; // Gcloud emulator settings private static final String GCLOUD_CMD_TEXT = "gcloud beta emulators datastore start"; private static final String GCLOUD_CMD_PORT_FLAG = "--host-port="; private static final String VERSION_PREFIX = "cloud-datastore-emulator "; - private static final String MIN_VERSION = "2.0.2"; + private static final String MIN_VERSION = "2.3.1"; // Downloadable emulator settings private static final String BIN_NAME = "cloud-datastore-emulator/cloud_datastore_emulator"; @@ -74,6 +75,8 @@ public class LocalDatastoreHelper extends BaseEmulatorHelper { private static final String PROJECT_FLAG = "--project="; private static final double DEFAULT_CONSISTENCY = 0.9; private static final String DEFAULT_PROJECT_ID = PROJECT_ID_PREFIX + UUID.randomUUID(); + private static final String FIRESTORE_IN_DATASTORE_MODE_FLAG = + "--use-firestore-in-datastore-mode"; private static final Logger LOGGER = Logger.getLogger(LocalDatastoreHelper.class.getName()); @@ -102,6 +105,7 @@ public static class Builder { private int port; private Path dataDir; private boolean storeOnDisk = true; + private boolean firestoreInDatastoreMode = false; private String projectId; private Builder() {} @@ -110,6 +114,7 @@ private Builder(LocalDatastoreHelper helper) { this.consistency = helper.consistency; this.dataDir = helper.gcdPath; this.storeOnDisk = helper.storeOnDisk; + this.firestoreInDatastoreMode = helper.firestoreInDatastoreMode; } public Builder setConsistency(double consistency) { @@ -137,6 +142,11 @@ public Builder setStoreOnDisk(boolean storeOnDisk) { return this; } + public Builder setFirestoreInDatastoreMode(boolean firestoreInDatastoreMode) { + this.firestoreInDatastoreMode = firestoreInDatastoreMode; + return this; + } + /** Creates a {@code LocalDatastoreHelper} object. */ public LocalDatastoreHelper build() { return new LocalDatastoreHelper(this); @@ -152,14 +162,21 @@ private LocalDatastoreHelper(Builder builder) { this.consistency = builder.consistency > 0 ? builder.consistency : DEFAULT_CONSISTENCY; this.gcdPath = builder.dataDir; this.storeOnDisk = builder.storeOnDisk; + this.firestoreInDatastoreMode = builder.firestoreInDatastoreMode; String binName = BIN_NAME; if (isWindows()) { binName = BIN_NAME.replace("/", "\\"); } List gcloudCommand = new ArrayList<>(Arrays.asList(GCLOUD_CMD_TEXT.split(" "))); gcloudCommand.add(GCLOUD_CMD_PORT_FLAG + "localhost:" + getPort()); - gcloudCommand.add(CONSISTENCY_FLAG + builder.consistency); gcloudCommand.add(PROJECT_FLAG + projectId); + if (builder.firestoreInDatastoreMode) { + gcloudCommand.add(FIRESTORE_IN_DATASTORE_MODE_FLAG); + } else { + // At most one of --consistency | --use-firestore-in-datastore-mode can be specified. + // --consistency will be ignored with --use-firestore-in-datastore-mode. + gcloudCommand.add(CONSISTENCY_FLAG + builder.consistency); + } if (!builder.storeOnDisk) { gcloudCommand.add("--no-store-on-disk"); } @@ -170,8 +187,16 @@ private LocalDatastoreHelper(Builder builder) { new GcloudEmulatorRunner(gcloudCommand, VERSION_PREFIX, MIN_VERSION); List binCommand = new ArrayList<>(Arrays.asList(binName, "start")); binCommand.add("--testing"); + if (builder.firestoreInDatastoreMode) { + // Downloadable emulator runner takes the flag in a different + // format: --firestore_in_datastore_mode + binCommand.add("--firestore_in_datastore_mode"); + } else { + // At most one of --consistency | --firestore_in_datastore_mode can be specified. + // --consistency will be ignored with --firestore_in_datastore_mode. + binCommand.add(CONSISTENCY_FLAG + getConsistency()); + } binCommand.add(BIN_CMD_PORT_FLAG + getPort()); - binCommand.add(CONSISTENCY_FLAG + getConsistency()); DownloadableEmulatorRunner downloadRunner = new DownloadableEmulatorRunner(binCommand, EMULATOR_URL, MD5_CHECKSUM, ACCESS_TOKEN); this.emulatorRunners = ImmutableList.of(gcloudRunner, downloadRunner); @@ -235,6 +260,13 @@ public boolean isStoreOnDisk() { return storeOnDisk; } + /** + * Returns {@code true} use firestore-in-datastore-mode, otherwise {@code false} use native mode. + */ + public boolean isFirestoreInDatastoreMode() { + return firestoreInDatastoreMode; + } + /** * Creates a local Datastore helper with the specified settings for project ID and consistency. * diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/testing/ITLocalDatastoreHelperTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/testing/ITLocalDatastoreHelperTest.java index dfa2255a0..cad6d4fbe 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/testing/ITLocalDatastoreHelperTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/testing/ITLocalDatastoreHelperTest.java @@ -76,11 +76,13 @@ public void testCreateWithBuilder() { .setConsistency(0.75) .setPort(8081) .setStoreOnDisk(false) + .setFirestoreInDatastoreMode(true) .setDataDir(dataDir) .build(); assertTrue(Math.abs(0.75 - helper.getConsistency()) < TOLERANCE); assertTrue(helper.getProjectId().startsWith(PROJECT_ID_PREFIX)); assertFalse(helper.isStoreOnDisk()); + assertTrue(helper.isFirestoreInDatastoreMode()); assertEquals(8081, helper.getPort()); assertEquals(dataDir, helper.getGcdPath()); LocalDatastoreHelper incompleteHelper = LocalDatastoreHelper.newBuilder().build(); @@ -103,11 +105,13 @@ public void testCreateWithToBuilder() throws IOException { .setConsistency(0.75) .setPort(8081) .setStoreOnDisk(false) + .setFirestoreInDatastoreMode(true) .setDataDir(dataDir) .build(); assertTrue(Math.abs(0.75 - helper.getConsistency()) < TOLERANCE); assertTrue(helper.getProjectId().startsWith(PROJECT_ID_PREFIX)); assertFalse(helper.isStoreOnDisk()); + assertTrue(helper.isFirestoreInDatastoreMode()); assertEquals(8081, helper.getPort()); assertEquals(dataDir, helper.getGcdPath()); LocalDatastoreHelper actualHelper = helper.toBuilder().build(); @@ -119,10 +123,12 @@ public void testCreateWithToBuilder() throws IOException { .setConsistency(0.85) .setPort(9091) .setStoreOnDisk(true) + .setFirestoreInDatastoreMode(false) .setDataDir(dataDir) .build(); assertTrue(Math.abs(0.85 - actualHelper.getConsistency()) < TOLERANCE); assertTrue(actualHelper.isStoreOnDisk()); + assertFalse(actualHelper.isFirestoreInDatastoreMode()); assertEquals(9091, actualHelper.getPort()); assertEquals(dataDir, actualHelper.getGcdPath()); LocalDatastoreHelper.deleteRecursively(dataDir); @@ -206,10 +212,28 @@ public void testStartStopResetWithBuilder() } } + @Test + public void testCreateWithFirestoreInDatastoreMode() + throws IOException, InterruptedException, TimeoutException { + LocalDatastoreHelper helper = + LocalDatastoreHelper.newBuilder().setFirestoreInDatastoreMode(true).build(); + assertTrue(helper.isFirestoreInDatastoreMode()); + helper.start(); + Datastore datastore = helper.getOptions().getService(); + Key key = datastore.newKeyFactory().setKind("kind").newKey("name"); + Entity expected = Entity.newBuilder(key).build(); + datastore.put(expected); + assertNotNull(datastore.get(key)); + Entity actual = datastore.get(key); + assertEquals(expected, actual); + helper.stop(); + } + public void assertLocalDatastoreHelpersEquivelent( LocalDatastoreHelper expected, LocalDatastoreHelper actual) { assertEquals(expected.getConsistency(), actual.getConsistency(), 0); assertEquals(expected.isStoreOnDisk(), actual.isStoreOnDisk()); + assertEquals(expected.isFirestoreInDatastoreMode(), actual.isFirestoreInDatastoreMode()); assertEquals(expected.getGcdPath(), actual.getGcdPath()); } }