Skip to content

Commit

Permalink
Merge pull request #47 from lensesio/feat/vault-secret-provider-rotation
Browse files Browse the repository at this point in the history
Vault - Secret Provider Rotation
  • Loading branch information
davidsloan authored Mar 17, 2023
2 parents 7523823 + d9085ee commit bcc16e6
Show file tree
Hide file tree
Showing 33 changed files with 1,574 additions and 186 deletions.
26 changes: 23 additions & 3 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
import Settings.modulesSettings
import Settings.secretProviderDeps
import Settings.artifactVersion
import Settings.AssemblyConfigurator
import Settings.testSinkDeps
import Settings.scala213
import sbt.Project.projectToLocalProject

name := "secret-provider"
javacOptions ++= Seq("--release", "11")
javaOptions ++= Seq("-Xms512M", "-Xmx2048M")

lazy val subProjects: Seq[Project] = Seq(
`test-sink`,
`secret-provider`,
)
lazy val subProjectsRefs: Seq[ProjectReference] = subProjects.map(projectToLocalProject)
lazy val subProjectsRefs: Seq[ProjectReference] =
subProjects.map(projectToLocalProject)

lazy val root = (project in file("."))
.settings(
scalaVersion := scala213,
publish := {},
publishArtifact := false,
name := "secret-provider",
Expand All @@ -34,12 +40,26 @@ lazy val `secret-provider` = (project in file("secret-provider"))
),
)
.configureAssembly()
.configs(IntegrationTest)
.settings(Defaults.itSettings: _*)

lazy val `test-sink` = (project in file("test-sink"))
.settings(
modulesSettings ++
Seq(
name := "test-sink",
description := "Kafka Connect compatible connectors to move data between Kafka and popular data stores",
publish / skip := true,
libraryDependencies ++= testSinkDeps,
),
)
.configureAssembly()

addCommandAlias(
"validateAll",
";scalafmtCheck;scalafmtSbtCheck;test:scalafmtCheck;",
";scalafmtCheck;scalafmtSbtCheck;test:scalafmtCheck;it:scalafmtCheck;",
)
addCommandAlias(
"formatAll",
";scalafmt;scalafmtSbt;test:scalafmt;",
";scalafmt;scalafmtSbt;test:scalafmt;it:scalafmt;",
)
60 changes: 41 additions & 19 deletions project/Dependencies.scala
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,22 @@ trait Dependencies {
val vaultVersion = "5.1.0"
val azureKeyVaultVersion = "4.5.2"
val azureIdentityVersion = "1.8.0"
val awsSecretsVersion = "1.12.411"
val awsSecretsVersion = "1.12.420"

//test
val scalaTestVersion = "3.2.15"
val mockitoVersion = "3.2.15.0"
val byteBuddyVersion = "1.14.0"
val slf4jVersion = "2.0.5"
val commonsIOVersion = "1.3.2"
val jettyVersion = "11.0.13"
val testContainersVersion = "1.12.3"
val flexmarkVersion = "0.64.0"
val scalaTestVersion = "3.2.15"
val mockitoVersion = "3.2.15.0"
val byteBuddyVersion = "1.14.1"
val slf4jVersion = "2.0.5"
val commonsIOVersion = "1.3.2"
val jettyVersion = "11.0.13"
val flexmarkVersion = "0.64.0"

val scalaCollectionCompatVersion = "2.8.1"
val jakartaServletVersion = "6.0.0"
val testContainersVersion = "1.17.6"
val json4sVersion = "4.0.6"
val catsEffectVersion = "3.4.8"

}

Expand Down Expand Up @@ -59,6 +61,15 @@ trait Dependencies {
val `jakartaServlet` =
"jakarta.servlet" % "jakarta.servlet-api" % jakartaServletVersion

val `testContainersCore` =
"org.testcontainers" % "testcontainers" % testContainersVersion
val `testContainersKafka` =
"org.testcontainers" % "kafka" % testContainersVersion
val `testContainersVault` =
"org.testcontainers" % "vault" % testContainersVersion
val `json4sNative` = "org.json4s" %% "json4s-native" % json4sVersion
val `json4sJackson` = "org.json4s" %% "json4s-jackson" % json4sVersion
val `cats` = "org.typelevel" %% "cats-effect" % catsEffectVersion
}

import Dependencies._
Expand All @@ -69,16 +80,27 @@ trait Dependencies {
`azure-key-vault`,
`azure-identity` exclude ("javax.activation", "activation"),
`aws-secrets-manager`,
`scalaCollectionCompat`,
`jakartaServlet` % Test,
`mockito` % Test,
`byteBuddy` % Test,
`scalatest` % Test,
`jetty` % Test,
`commons-io` % Test,
`flexmark` % Test,
`slf4j-api` % Test,
`slf4j-simple` % Test,
`jakartaServlet` % Test,
`mockito` % Test,
`byteBuddy` % Test,
`scalatest` % Test,
`jetty` % Test,
`commons-io` % Test,
`flexmark` % Test,
`slf4j-api` % Test,
`slf4j-simple` % Test,
`cats` % IntegrationTest,
`testContainersCore` % IntegrationTest,
`testContainersKafka` % IntegrationTest,
`testContainersVault` % IntegrationTest,
`json4sNative` % IntegrationTest,
)

val testSinkDeps = Seq(
`scala-logging`,
`kafka-connect-api` % Provided,
`vault-java-driver`,
`scalatest` % Test,
)

}
73 changes: 52 additions & 21 deletions project/Settings.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,16 @@
* Copyright (c) 2017-2020 Lenses.io Ltd
*/

import com.eed3si9n.jarjarabrams.ShadeRule
import sbt.Keys._
import sbt.Def
import sbt._
import sbtassembly.AssemblyKeys._
import sbtassembly.MergeStrategy
import sbtassembly.PathList

object Settings extends Dependencies {

val scala212 = "2.12.14"
val scala213 = "2.13.10"
val scala3 = "3.2.2"

Expand All @@ -37,7 +38,7 @@ object Settings extends Dependencies {
Resolver.mavenLocal,
Resolver.mavenCentral,
),
crossScalaVersions := List( /*scala3, */ scala213 /*scala212*/ ),
crossScalaVersions := List( /*scala3, */ scala213),
Compile / scalacOptions ++= Seq(
"-release:11",
"-encoding",
Expand All @@ -58,33 +59,63 @@ object Settings extends Dependencies {
ScalacFlags.WarnUnusedImports212,
ScalacFlags.WarnUnusedImports213,
),
// TODO remove for cross build
scalaVersion := scala213,
Test / fork := true,
)

implicit final class AssemblyConfigurator(project: Project) {

/*
Exclude the jar signing files from dependencies. They come from other jars,
and including these would break assembly (due to duplicates) or classloading
(due to mismatching jar checksum/signature).
*/
val excludeFilePatterns = Set(".MF", ".RSA", ".DSA", ".SF")

def excludeFileFilter(p: String): Boolean =
excludeFilePatterns.exists(p.endsWith)

val excludePatterns = Set(
"kafka-client",
"kafka-connect-json",
"hadoop-yarn",
"org.apache.kafka",
"zookeeper",
"log4j",
"junit",
)

def configureAssembly(): Project =
project.settings(
assembly / assemblyJarName := s"secret-provider_${SemanticVersioning(scalaVersion.value).majorMinor}-${artifactVersion}-all.jar",
assembly / assemblyExcludedJars := {
val cp = (assembly / fullClasspath).value
val excludes = Set(
"org.apache.avro",
"org.apache.kafka",
"io.confluent",
"org.apache.zookeeper",
)
cp filter { f => excludes.exists(f.data.getName.contains(_)) }
},
assembly / assemblyMergeStrategy := {
case "module-info.class" => MergeStrategy.discard
case x if x.contains("io.netty.versions.properties") =>
MergeStrategy.concat
case x =>
val oldStrategy = (assembly / assemblyMergeStrategy).value
oldStrategy(x)
},
Seq(
assembly / assemblyOutputPath := file(
target.value + "/libs/" + (assembly / assemblyJarName).value,
),
assembly / assemblyExcludedJars := {
val cp: Classpath = (assembly / fullClasspath).value
cp filter { f =>
excludePatterns
.exists(f.data.getName.contains) && (!f.data.getName
.contains("slf4j"))
}
},
assembly / assemblyMergeStrategy := {
case PathList("META-INF", "MANIFEST.MF") => MergeStrategy.discard
case p if excludeFileFilter(p) => MergeStrategy.discard
case PathList(ps @ _*) if ps.last == "module-info.class" =>
MergeStrategy.discard
case _ => MergeStrategy.first
},
assembly / assemblyShadeRules ++= Seq(
ShadeRule.rename("io.confluent.**" -> "lshaded.confluent.@1").inAll,
ShadeRule.rename("com.fasterxml.**" -> "lshaded.fasterxml.@1").inAll,
),
),
)

}

lazy val IntegrationTest = config("it") extend Test

}
31 changes: 31 additions & 0 deletions secret-provider/src/it/resources/logback.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>

<!-- -->
<!-- Copyright 2021 Celonis. -->
<!-- -->
<!-- Licensed under the Apache License, Version 2.0 (the "License"); -->
<!-- you may not use this file except in compliance with the License. -->
<!-- You may obtain a copy of the License at -->
<!-- -->
<!-- http://www.apache.org/licenses/LICENSE-2.0 -->
<!-- -->
<!-- Unless required by applicable law or agreed to in writing, software -->
<!-- distributed under the License is distributed on an "AS IS" BASIS, -->
<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
<!-- See the License for the specific language governing permissions and -->
<!-- limitations under the License. -->
<!-- -->
<configuration>
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
<Target>System.out</Target>
<encoder>
<pattern>%d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n</pattern>
</encoder>
</appender>
<logger name="io.lenses" level="INFO"/>
<logger name="com.datamountaineer" level="INFO"/>
<logger name="org.apache.kafka.connect.json.JsonConverterConfig" level="WARN"/>
<root level="INFO">
<appender-ref ref="stdout"/>
</root>
</configuration>
1 change: 1 addition & 0 deletions secret-provider/src/it/resources/openapi.json

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package io.lenses.connect.secrets.integration

import com.bettercloud.vault.rest.RestResponse
import org.json4s.DefaultFormats
import org.json4s._
import org.json4s.native.JsonMethods._

case class DataResponse(
data: Map[String, Any],
metadata: Map[String, Any],
)
case class SecretProviderRestResponse(
requestId: String,
leaseId: String,
renewable: Boolean,
leaseDuration: BigInt,
data: DataResponse,
wrapInfo: Any, // todo what is this?
warnings: Any, // todo
auth: Any, // todo
)

case object SecretProviderRestResponse {

def fromRestResponse(rr: RestResponse) =
fromJson(new String(rr.getBody))

def fromJson(json: String) = {
implicit val formats: DefaultFormats.type = DefaultFormats

parse(json).camelizeKeys.extract[SecretProviderRestResponse]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package io.lenses.connect.secrets.integration

import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers

class SecretProviderRestResponseTest extends AnyFunSuite with Matchers {

private val sampleJson =
"""{"request_id":"41398853-3a2f-ba6b-7c37-23b201941071","lease_id":"","renewable":false,"lease_duration":20,"data": {"data":{"myVaultSecretKey":"myVaultSecretValue"}, "metadata":{"created_time":"2023-02-28T10:49:56.854615Z","custom_metadata":null,"deletion_time":"","destroyed":false,"version":1}},"wrap_info":null,"warnings":null,"auth":null}"""

test("Should unmarshal sample json") {
val sprr = SecretProviderRestResponse.fromJson(sampleJson)

sprr.requestId should be("41398853-3a2f-ba6b-7c37-23b201941071")
sprr.leaseDuration should be(20)
sprr.data.data should be(Map[String, Any]("myVaultSecretKey" -> "myVaultSecretValue"))
sprr.data.metadata.toSet should be(
Set(
"createdTime" -> "2023-02-28T10:49:56.854615Z",
"customMetadata" -> null,
"deletionTime" -> "",
"destroyed" -> false,
"version" -> Integer.valueOf(1),
),
)
}

}
Loading

0 comments on commit bcc16e6

Please sign in to comment.