Consolidates project

.circleci/config.yml

@@ -33,20 +33,10 @@ commands:
           name: << parameters.args >>
           command: ./gradlew << parameters.args >> --info --max-workers=2 -Dorg.gradle.jvmargs=-Xmx2g -Dorg.gradle.console=plain --continue
           when: << parameters.when >>
-  update-submodule:
-    description: 'Pull Submodules'
-    steps:
-      - run:
-          name: Pull Submodules
-          command: git submodule update --init --recursive
-      - run:
-          name: Ensure submodules point to commit from main branch
-          command: git submodule foreach "git merge-base --is-ancestor HEAD origin/main"
   setup_build_environment:
     description: 'Generates cache key from a hash of all gradle files'
     steps:
       - checkout
-      - update-submodule
       - run:
           name: Generate cache key
           command: find . -type f -name "*.gradle*" -o -name "gradle-wrapper*" -exec shasum {} + | sort > /tmp/checksum.txt && cat /tmp/checksum.txt
@@ -85,29 +75,31 @@ jobs:
           args: jacocoTestReport
       - codecov/upload:
           flags: unit
-  publish:
+  merge-publish:
     executor: gradle_docker
     steps:
       - setup_build_environment
       - setup_remote_docker: *latest_remote_docker
       - docker_login
       - gradle:
-          args: :tag -Prelease
-      - add_ssh_keys:
-          fingerprints:
-            - '76:7a:c8:91:c0:cd:eb:71:96:3c:44:c6:cf:b0:26:83'
-      - run: git push origin $(./gradlew -q :printVersion)
+          args: dockerPushImages
+  release-publish:
+    executor: gradle_docker
+    steps:
+      - setup_build_environment
+      - setup_remote_docker: *latest_remote_docker
+      - docker_login
       - gradle:
           args: dockerPushImages
   validate-charts:
     executor: helm
+    working_directory: ~/repo
     steps:
       - checkout
       - run:
           name: Helm Charts Lint and Template Render
           command: |
-            helm lint --strict ./helm/
-            helm template ./helm/
+            .circleci/helm.sh validate
   snyk-scan:
     executor:
       name: gradle_docker
@@ -117,21 +109,33 @@ jobs:
       - setup_build_environment
       - snyk/scan:
           additional-arguments: --all-sub-projects --policy-path=.snyk
-  package-charts:
+  release-charts:
     executor: helm
+    working_directory: ~/repo
     steps:
       - checkout
+      - run:
+          name: Add release tag
+          command: |
+            git config --global user.email "${CIRCLE_USERNAME}@hypertrace.org"
+            git config --global user.name "$CIRCLE_USERNAME"
+            git tag -am "Released by $CIRCLE_USERNAME" $(git describe --abbrev=0 --tags | sed 's/^release-//')
+      - run:
+          name: Remove trigger tag
+          command: git tag -d release-$(git describe --abbrev=0)
       - run:
           name: Package and Publish Helm Charts
+          # Read the "name:" from Chart.yaml. The chart version is <chart-name>-<semver git tag>
           command: |
-            CHART_VERSION=$(git describe --abbrev=0)
-            CHART_NAME=$(awk '/^name:/ {print $2}' ./helm/Chart.yaml)
-            export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/helm-gcs-key.json
-            echo ${HELM_GCS_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS}
-            helm repo add helm-gcs ${HELM_GCS_REPOSITORY}
-            helm package --version ${CHART_VERSION} --app-version ${CHART_VERSION} ./helm/
-            helm gcs push ${CHART_NAME}-${CHART_VERSION}.tgz helm-gcs --public --retry
-
+            .circleci/helm.sh package
+            .circleci/helm.sh publish
+      - add_ssh_keys:
+          fingerprints:
+            # This ssh key gives write permission needed for the following step.
+            - '76:7a:c8:91:c0:cd:eb:71:96:3c:44:c6:cf:b0:26:83'
+      - run:
+          name: Update remote tags
+          command: git push origin refs/tags/$(git describe --abbrev=0) :refs/tags/release-$(git describe --abbrev=0)
 workflows:
   version: 2
   build-and-publish:
@@ -146,7 +150,7 @@ workflows:
           context:
             - hypertrace-vulnerability-scanning
             - dockerhub-read
-      - publish:
+      - merge-publish:
           context:
             - hypertrace-publishing
             - dockerhub-read
@@ -158,13 +162,23 @@ workflows:
             branches:
               only:
                 - main
-      - package-charts:
+      - release-publish:
+          context:
+            - hypertrace-publishing
+            - dockerhub-read
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^release-.*/
+      - release-charts:
           context:
             - hypertrace-publishing
             - dockerhub-read
           requires:
-            - publish
+            - release-publish
           filters:
             branches:
-              only:
-                - main
+              ignore: /.*/
+            tags:
+              only: /^release-.*/

.circleci/helm.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+set -eu
+
+script=$0
+
+SCRIPT_DIR="$( cd "$( dirname "$script" )" >/dev/null 2>&1 && pwd )"
+ROOT_PROJECT_DIR="$(dirname "${SCRIPT_DIR}")"
+cd $ROOT_PROJECT_DIR
+SUB_PROJECTS_DIRS=$(find . -iname "helm" | sed 's/\(.*\)\/.*/\1/')
+
+subcommand=$1; shift
+case "$subcommand" in
+  validate)
+    for SUB_PROJ_DIR in $SUB_PROJECTS_DIRS; do
+      cd $SUB_PROJ_DIR
+      echo "*******"
+      echo "Validating charts for dir \"$(pwd)\""
+      helm dependency update ./helm/
+      helm lint --strict ./helm/
+      helm template ./helm/
+      cd $ROOT_PROJECT_DIR
+    done
+    ;;
+  package)
+    CHART_VERSION=$(git describe --abbrev=0)
+    for SUB_PROJ_DIR in $SUB_PROJECTS_DIRS; do
+      cd $SUB_PROJ_DIR
+      echo "*******"
+      echo "building charts for:$(pwd)"
+      CHART_NAME=$(awk '/^name:/ {print $2}' ./helm/Chart.yaml)
+      helm dependency update ./helm/
+      helm package --version ${CHART_VERSION} --app-version ${CHART_VERSION} ./helm/
+      cd $ROOT_PROJECT_DIR
+    done
+    ;;
+  publish)
+    CHART_VERSION=$(git describe --abbrev=0)
+    export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/helm-gcs-key.json
+    echo ${HELM_GCS_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS}
+    helm repo add helm-gcs ${HELM_GCS_REPOSITORY}
+    for SUB_PROJ_DIR in $SUB_PROJECTS_DIRS; do
+      cd $SUB_PROJ_DIR
+      echo "*******"
+      echo "publishing charts for:$(pwd)"
+      helm gcs push ${CHART_NAME}-${CHART_VERSION}.tgz helm-gcs --public --retry
+      cd $ROOT_PROJECT_DIR
+    done
+    ;;
+  *)
+    echo "[ERROR] Unknown command: ${subcommand}"
+    echo "usage: $script {validate|package}"
+    ;;
+esac

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# https://help.github.com/articles/about-codeowners/
+# Each line is a file pattern followed by one or more owners.
+
+# global
+* @hypertrace/trace-pipeline-owners

.snyk

@@ -1,5 +1,35 @@
 # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
 version: v1.14.1
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
-ignore: {}
-patch: {}
+ignore:
+## from hypertrace-trace-enricher and span-normalizer
+  SNYK-JAVA-IOGRPC-571957:
+    - '*':
+        reason: No replacement available
+        expires: 2020-12-31T00:00:00.000Z
+## from hypertrace-view-generator
+  SNYK-JAVA-COMMONSHTTPCLIENT-30083:
+    - '*':
+        reason: no available replacement
+        expires: 2020-12-31T00:00:00.000Z
+  SNYK-JAVA-COMMONSHTTPCLIENT-31660:
+    - '*':
+        reason: no available replacement
+        expires: 2020-12-31T00:00:00.000Z
+  SNYK-JAVA-IONETTY-473694:
+    - '*':
+        reason: no available replacement
+        expires: 2020-12-31T00:00:00.000Z
+  SNYK-JAVA-LOG4J-572732:
+    - '*':
+        reason: no available replacement
+        expires: 2020-12-31T00:00:00.000Z
+  SNYK-JAVA-ORGAPACHECOMMONS-559327:
+    - '*':
+        reason: no available replacement
+        expires: 2020-12-31T00:00:00.000Z
+  SNYK-JAVA-ORGCODEHAUSJACKSON-534878:
+    - '*':
+        reason: no available replacement
+        expires: 2020-12-31T00:00:00.000Z
+patch: {}

buf.yaml

@@ -0,0 +1,11 @@
+build:
+  roots:
+    - hypertrace-trace-enricher/enriched-span-constants/src/main/proto
+    - span-normalizer/raw-span-constants/src/main/proto
+lint:
+  use:
+    - DEFAULT
+breaking:
+  use:
+    - PACKAGE
+    - WIRE_JSON

build.gradle.kts

@@ -4,11 +4,19 @@ plugins {
   id("org.hypertrace.docker-java-application-plugin") version "0.7.1" apply false
   id("org.hypertrace.docker-publish-plugin") version "0.7.1" apply false
   id("org.hypertrace.jacoco-report-plugin") version "0.1.1" apply false
+  id("org.hypertrace.publish-plugin") version "0.3.3" apply false
+  id("org.hypertrace.avro-plugin") version "0.2.4" apply false
 }
 
 subprojects {
   group = "org.hypertrace.ingester"
 
+  pluginManager.withPlugin("org.hypertrace.publish-plugin") {
+    configure<org.hypertrace.gradle.publishing.HypertracePublishExtension> {
+      license.set(org.hypertrace.gradle.publishing.License.TRACEABLE_COMMUNITY)
+    }
+  }
+
   pluginManager.withPlugin("java") {
     configure<JavaPluginExtension> {
       sourceCompatibility = JavaVersion.VERSION_11

gradle.properties

@@ -2,4 +2,3 @@ org.gradle.parallel=true
 org.gradle.daemon=true
 org.gradle.caching=true
 org.gradle.configureondemand=true
-

hypertrace-ingester/build.gradle.kts

@@ -7,6 +7,10 @@ plugins {
   id("org.hypertrace.jacoco-report-plugin")
 }
 
+subprojects {
+  group = "org.hypertrace.ingester"
+}
+
 application {
   mainClassName = "org.hypertrace.core.serviceframework.PlatformServiceLauncher"
 }
@@ -25,12 +29,12 @@ dependencies {
   implementation("org.hypertrace.core.serviceframework:platform-service-framework:0.1.9")
   implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.8")
   implementation("org.hypertrace.core.datamodel:data-model:0.1.9")
-
-  implementation("org.hypertrace.core.spannormalizer:span-normalizer")
-  implementation("org.hypertrace.core.rawspansgrouper:raw-spans-grouper")
-  implementation("org.hypertrace.traceenricher:hypertrace-trace-enricher")
-  implementation("org.hypertrace.viewgenerator:hypertrace-view-generator")
   implementation("org.hypertrace.core.viewgenerator:view-generator-framework:0.1.14")
+
+  implementation(project(":span-normalizer:span-normalizer"))
+  implementation(project(":raw-spans-grouper:raw-spans-grouper"))
+  implementation(project(":hypertrace-trace-enricher:hypertrace-trace-enricher"))
+  implementation(project(":hypertrace-view-generator:hypertrace-view-generator"))
 }
 
 // Config for gw run to be able to run this locally. Just execute gw run here on Intellij or on the console.

hypertrace-trace-enricher/build.gradle.kts

@@ -0,0 +1,3 @@
+subprojects {
+  group = "org.hypertrace.traceenricher"
+}