build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

plugins {
  id 'base'
  // Enable publishing build scans
  id 'com.gradle.build-scan' version '2.3' apply false
  // This plugin provides a task to determine which dependencies have updates.
  // Additionally, the plugin checks for updates to Gradle itself.
  //
  // See https://github.com/ben-manes/gradle-versions-plugin for further details.
  id 'com.github.ben-manes.versions' version '0.20.0'
  // Apply one top level rat plugin to perform any required license enforcement analysis
  id 'org.nosphere.apache.rat' version '0.4.0'
  // Enable gradle-based release management
  id 'net.researchgate.release' version '2.6.0'
  id 'org.apache.beam.module'
  id "org.sonarqube" version "2.7"
}

/*************************************************************************************************/
// Configure the root project

// Plugins which require online access should not be enabled when running in offline mode.
if (!gradle.startParameter.isOffline()) {
  apply plugin: "com.gradle.build-scan"
}

// JENKINS_HOME and BUILD_ID set automatically during Jenkins execution
def isCIBuild = ['JENKINS_HOME', 'BUILD_ID'].every System.&getenv
if (isCIBuild) {
  buildScan {
    // Build Scan enabled and TOS accepted for Jenkins lab build. This does not apply to builds on
    // non-Jenkins machines. Developers need to separately enable and accept TOS to use build scans.
    termsOfServiceUrl = 'https://gradle.com/terms-of-service'
    termsOfServiceAgree = 'yes'
    publishAlways()
  }
}

rat {
  // Set input directory to that of the root project instead of the CWD. This
  // makes .gitignore rules (added below) work properly.
  inputDir = project.rootDir

  def exclusions = [
    // Ignore files we track but do not distribute
    "**/.github/**/*",

    "**/package-list",
    "**/test.avsc",
    "**/user.avsc",
    "**/test/resources/**/*.txt",
    "**/test/**/.placeholder",

    // Default eclipse excludes neglect subprojects

    // Proto/grpc generated wrappers
    "**/apache_beam/portability/api/*_pb2*.py",
    "**/go/pkg/beam/**/*.pb.go",

    // Ignore go.sum files, which don't permit headers
    "**/go.sum",

    // Ignore Go test data files
    "**/go/data/**",

    // VCF test files
    "**/apache_beam/testing/data/vcf/*",

    // JDBC package config files
    "**/META-INF/services/java.sql.Driver",

    // Website build files
    "**/Gemfile.lock",
    "**/Rakefile",
    "**/.htaccess",
    "website/www/site/assets/scss/_bootstrap.scss",
    "website/www/site/assets/scss/bootstrap/**/*",
    "website/www/site/static/images/mascot/*.ai",
    "website/www/site/static/js/bootstrap*.js",
    "website/www/site/static/js/bootstrap/**/*",
    "website/www/site/themes",
    "website/www/yarn.lock",
    "website/www/package.json",

    // Ignore ownership files
    "ownership/**/*",
    "**/OWNERS",

    // Json doesn't support comments.
    "**/*.json",

    // Katas files
    "learning/katas/**/course-remote-info.yaml",
    "learning/katas/**/section-remote-info.yaml",
    "learning/katas/**/lesson-remote-info.yaml",
    "learning/katas/**/task-remote-info.yaml",
    "learning/katas/*/IO/**/*.txt",

    // test p8 file for SnowflakeIO
    "sdks/java/io/snowflake/src/test/resources/test_rsa_key.p8",

    // Mockito extensions
    "sdks/java/io/amazon-web-services2/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",
    "sdks/java/extensions/ml/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",

    // JupyterLab extensions
    "sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock"
  ]

  // Add .gitignore excludes to the Apache Rat exclusion list. We re-create the behavior
  // of the Apache Maven Rat plugin since the Apache Ant Rat plugin doesn't do this
  // automatically.
  def gitIgnore = project(':').file('.gitignore')
  if (gitIgnore.exists()) {
    def gitIgnoreExcludes = gitIgnore.readLines().findAll { !it.isEmpty() && !it.startsWith('#') }
    exclusions.addAll(gitIgnoreExcludes)
  }

  failOnError = true
  excludes = exclusions
}
check.dependsOn rat

// Define root pre/post commit tasks simplifying what is needed
// to be specified on the commandline when executing locally.
// This indirection also makes Jenkins use the branch of the PR
// for the test definitions.
task javaPreCommit() {
  // We need to list the model/* builds since sdks/java/core doesn't
  // depend on any of the model.
  dependsOn ":model:pipeline:build"
  dependsOn ":model:job-management:build"
  dependsOn ":model:fn-execution:build"
  dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:build"
  dependsOn ":sdks:java:core:buildNeeded"
  dependsOn ":sdks:java:core:buildDependents"
  dependsOn ":examples:java:preCommit"
  dependsOn ":sdks:java:extensions:sql:jdbc:preCommit"
  dependsOn ":sdks:java:javadoc:allJavadoc"
  dependsOn ":runners:direct-java:needsRunnerTests"
  dependsOn ":sdks:java:container:docker"
}

task sqlPreCommit() {
  dependsOn ":sdks:java:extensions:sql:runBasicExample"
  dependsOn ":sdks:java:extensions:sql:runPojoExample"
  dependsOn ":sdks:java:extensions:sql:build"
  dependsOn ":sdks:java:extensions:sql:buildDependents"
}

task javaPreCommitBeamZetaSQL() {
  dependsOn ":sdks:java:extensions:sql:zetasql:test"
}

task javaPreCommitPortabilityApi() {
  dependsOn ":runners:google-cloud-dataflow-java:worker:build"
}

task javaPostCommit() {
  dependsOn ":runners:google-cloud-dataflow-java:postCommit"
  dependsOn ":sdks:java:extensions:google-cloud-platform-core:postCommit"
  dependsOn ":sdks:java:extensions:zetasketch:postCommit"
  dependsOn ":sdks:java:io:google-cloud-platform:postCommit"
}

task sqlPostCommit() {
  dependsOn ":sdks:java:extensions:sql:postCommit"
  dependsOn ":sdks:java:extensions:sql:jdbc:postCommit"
  dependsOn ":sdks:java:extensions:sql:datacatalog:postCommit"
}

task goPreCommit() {
  dependsOn ":sdks:go:goTest"

  dependsOn ":sdks:go:examples:goBuild"
  dependsOn ":sdks:go:test:goBuild"

  // Ensure all container Go boot code builds as well.
  dependsOn ":sdks:java:container:goBuild"
  dependsOn ":sdks:python:container:goBuild"
  dependsOn ":sdks:go:container:goBuild"
}

task goPostCommit() {
  dependsOn ":goIntegrationTests"
}

task goIntegrationTests() {
  doLast {
    exec {
      executable 'sh'
      args '-c', './sdks/go/test/run_integration_tests.sh'
    }
  }
  dependsOn ":sdks:go:test:build"
  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
}

task pythonPreCommit() {
  dependsOn ":sdks:python:test-suites:tox:pycommon:preCommitPyCommon"
  dependsOn ":sdks:python:test-suites:tox:py2:preCommitPy2"
  dependsOn ":sdks:python:test-suites:tox:py35:preCommitPy35"
  dependsOn ":sdks:python:test-suites:tox:py36:preCommitPy36"
  dependsOn ":sdks:python:test-suites:tox:py37:preCommitPy37"
  dependsOn ":sdks:python:test-suites:tox:py38:preCommitPy38"
  dependsOn ":sdks:python:test-suites:dataflow:py2:preCommitIT"
  dependsOn ":sdks:python:test-suites:dataflow:py2:preCommitIT_V2"
  dependsOn ":sdks:python:test-suites:dataflow:py37:preCommitIT"
  dependsOn ":sdks:python:test-suites:dataflow:py37:preCommitIT_V2"
  // We don't include Py35, Py36 precommit ITs to reduce quota footprint.
  // We can reconsider if we ever see an issue that these suites would
  // have caught. Note that the same tests will still run in postcommit.
}

task pythonDockerBuildPreCommit() {
  dependsOn ":sdks:python:container:py2:docker"
  dependsOn ":sdks:python:container:py35:docker"
  dependsOn ":sdks:python:container:py36:docker"
  dependsOn ":sdks:python:container:py37:docker"
  dependsOn ":sdks:python:container:py38:docker"
}

task pythonLintPreCommit() {
  dependsOn ":sdks:python:test-suites:tox:py2:lint"
  dependsOn ":sdks:python:test-suites:tox:py37:lint"
}

task pythonFormatterPreCommit() {
  dependsOn 'sdks:python:test-suites:tox:py38:formatter'
}

task python2PostCommit() {
  dependsOn ":sdks:python:test-suites:portable:py2:crossLanguagePythonJavaKafkaIOFlink"
  dependsOn ":sdks:python:test-suites:portable:py2:crossLanguageTests"
  dependsOn ":sdks:python:test-suites:dataflow:py2:postCommitIT"
  dependsOn ":sdks:python:test-suites:direct:py2:directRunnerIT"
  dependsOn ":sdks:python:test-suites:direct:py2:hdfsIntegrationTest"
  dependsOn ":sdks:python:test-suites:direct:py2:mongodbioIT"
  dependsOn ":sdks:python:test-suites:portable:py2:postCommitPy2"
}

task python35PostCommit() {
  dependsOn ":sdks:python:test-suites:dataflow:py35:postCommitIT"
  dependsOn ":sdks:python:test-suites:direct:py35:postCommitIT"
  dependsOn ":sdks:python:test-suites:portable:py35:postCommitPy35"
}

task python36PostCommit() {
  dependsOn ":sdks:python:test-suites:dataflow:py36:postCommitIT"
  dependsOn ":sdks:python:test-suites:direct:py36:postCommitIT"
  dependsOn ":sdks:python:test-suites:portable:py36:postCommitPy36"
}

task python37PostCommit() {
  dependsOn ":sdks:python:test-suites:dataflow:py37:postCommitIT"
  dependsOn ":sdks:python:test-suites:direct:py37:postCommitIT"
  dependsOn ":sdks:python:test-suites:portable:py37:postCommitPy37"
}

task python38PostCommit() {
  dependsOn ":sdks:python:test-suites:portable:py38:crossLanguagePythonJavaKafkaIOFlink"
  dependsOn ":sdks:python:test-suites:dataflow:py38:postCommitIT"
  dependsOn ":sdks:python:test-suites:direct:py38:postCommitIT"
  dependsOn ":sdks:python:test-suites:direct:py38:hdfsIntegrationTest"
  dependsOn ":sdks:python:test-suites:portable:py38:postCommitPy38"
}

task portablePythonPreCommit() {
  dependsOn ":sdks:python:test-suites:portable:py2:preCommitPy2"
  dependsOn ":sdks:python:test-suites:portable:py35:preCommitPy35"
  dependsOn ":sdks:python:test-suites:portable:py36:preCommitPy36"
  dependsOn ":sdks:python:test-suites:portable:py37:preCommitPy37"
}

task pythonSparkPostCommit() {
  dependsOn ":sdks:python:test-suites:portable:py2:sparkValidatesRunner"
  dependsOn ":sdks:python:test-suites:portable:py35:sparkValidatesRunner"
  dependsOn ":sdks:python:test-suites:portable:py36:sparkValidatesRunner"
  dependsOn ":sdks:python:test-suites:portable:py37:sparkValidatesRunner"
  dependsOn ":sdks:python:test-suites:portable:py38:sparkValidatesRunner"
}

task websitePreCommit() {
  dependsOn ":website:preCommit"
}

task communityMetricsPreCommit() {
  dependsOn ":beam-test-infra-metrics:preCommit"
}

task communityMetricsProber() {
  dependsOn ":beam-test-infra-metrics:checkProber"
}

task javaExamplesDataflowPrecommit() {
  dependsOn ":runners:google-cloud-dataflow-java:examples:preCommit"
  dependsOn ":runners:google-cloud-dataflow-java:examples-streaming:preCommit"
}

task runBeamDependencyCheck() {
  dependsOn ":dependencyUpdates"
  dependsOn ":sdks:python:dependencyUpdates"
}

// Configure the release plugin to do only local work; the release manager determines what, if
// anything, to push. On failure, the release manager can reset the branch without pushing.
release {
  revertOnFail = false
  tagTemplate = 'v${version}'
  git {
    requireBranch = 'release-.*|master'
    pushToRemote = ''
  }
}

// Reports linkage errors across multiple Apache Beam artifact ids.
//
// To use (from the root of project):
//    ./gradlew -Ppublishing -PjavaLinkageArtifactIds=artifactId1,artifactId2,... :checkJavaLinkage
//
// For example:
//    ./gradlew -Ppublishing -PjavaLinkageArtifactIds=beam-sdks-java-core,beam-sdks-java-io-jdbc :checkJavaLinkage
//
// Note that this task publishes artifacts into your local Maven repository.
if (project.hasProperty('javaLinkageArtifactIds')) {
  if (!project.hasProperty('publishing')) {
    throw new GradleException('You can only check linkage of Java artifacts if you specify -Ppublishing on the command line as well.')
  }

  configurations { linkageCheckerJava }
  dependencies {
    linkageCheckerJava "com.google.cloud.tools:dependencies:1.4.2"
  }

  // We need to evaluate all the projects first so that we can find depend on all the
  // publishMavenJavaPublicationToMavenLocal tasks below.
  for (p in rootProject.subprojects) {
    if (!p.path.equals(project.path)) {
      evaluationDependsOn(p.path)
    }
  }

  project.task('checkJavaLinkage', type: JavaExec) {
    dependsOn project.getTasksByName('publishMavenJavaPublicationToMavenLocal', true /* recursively */)
    classpath = project.configurations.linkageCheckerJava
    main = 'com.google.cloud.tools.opensource.classpath.LinkageCheckerMain'
    def arguments = ['-a', project.javaLinkageArtifactIds.split(',').collect({
      if (it.contains(':')) {
        "${project.ext.mavenGroupId}:${it}"
      } else {
        // specify the version if not provided
        "${project.ext.mavenGroupId}:${it}:${project.version}"
      }
    }).join(',')]
    
    // Exclusion file filters out existing linkage errors before a change
    if (project.hasProperty('javaLinkageWriteBaseline')) {
      arguments += ['--output-exclusion-file', project.getProperty('javaLinkageWriteBaseline')]
    } else if (project.hasProperty('javaLinkageReadBaseline')) {
      arguments += ['--exclusion-file', project.getProperty('javaLinkageReadBaseline')]
    }
    args arguments
    doLast {
      println "NOTE: This task published artifacts into your local Maven repository. You may want to remove them manually."
    }
  }
}
if (project.hasProperty('compileAndRunTestsWithJava11')) {
  project.javaPreCommitPortabilityApi.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'
  project.javaExamplesDataflowPrecommit.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'
  project.javaPreCommitBeamZetaSQL.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'
  project.sqlPreCommit.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'
} else {
  allprojects {
    tasks.withType(Test) {
      exclude '**/JvmVerification.class'
    }
  }
}