Merge pull request #22 from 41north/feature/docker-builds

Feature/docker builds

docker-compose.exflo-postgres.yml

@@ -18,7 +18,6 @@ services:
       - 5432:5432
     volumes:
       - postgres:/var/lib/postgresql/data
-      - ./docker/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
     networks:
       - net
     environment:

docker/exflo/Dockerfile

@@ -0,0 +1,13 @@
+FROM gradle:6.3.0-jdk11 AS build
+
+WORKDIR /build
+
+COPY ./ /build
+
+RUN ./gradlew :plugin:build
+
+FROM hyperledger/besu:1.4.4
+
+RUN mkdir -p /opt/besu/plugins
+
+COPY --from=build /build/plugin/build/libs/*.jar /opt/besu/plugins

docker/exflo/docker-compose.yml

@@ -0,0 +1,134 @@
+version: '3.7'
+
+networks:
+  net:
+    driver: bridge
+
+volumes:
+
+  postgres:
+    driver: local
+
+  zookeeper-data:
+    driver: local
+
+  zookeeper-log:
+    driver: local
+
+  kafka-data:
+    driver: local
+
+  besu:
+    driver: local
+
+services:
+
+#  zookeeper:
+#    image: confluentinc/cp-zookeeper:5.3.1
+#    restart: unless-stopped
+#    volumes:
+#      - zookeeper-data:/var/lib/zookeeper/data
+#      - zookeeper-log:/var/lib/zookeeper/log
+#    networks:
+#      - net
+#    ports:
+#      - 2181:2181
+#      - 9585:9585
+#    environment:
+#      ZOOKEEPER_SERVER_ID: 1
+#      ZOOKEEPER_CLIENT_PORT: 2181
+#      ZOOKEEPER_JMX_PORT: 9585
+#
+#  kafka:
+#    image: confluentinc/cp-kafka:5.3.1
+#    restart: unless-stopped
+#    hostname: kafka
+#    depends_on:
+#      - zookeeper
+#    volumes:
+#      - kafka-data:/var/lib/kafka
+#    networks:
+#      - net
+#    ports:
+#      - 9092:9092
+#      - 9586:9586
+#    environment:
+#      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"
+#      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+#      KAFKA_BROKER_ID: 1
+#      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+#      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+#      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+#      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+#      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+#      KAFKA_COMPRESSION_TYPE: "zstd"
+#      KAFKA_PRODUCER_MAX_REQUEST_SIZE: 52428800
+#      KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES: 10485760
+#      KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS: 120000
+#      KAFKA_MESSAGE_MAX_BYTES: 52428800
+#      KAFKA_REPLICA_FETCH_MAX_BYTES: 10485760
+#      KAFKA_TOPIC_DELETE_ENABLE: 'true'
+#      CONFLUENT_METRICS_ENABLE: 'false'
+#      KAFKA_JMX_PORT: 9586
+#
+#  kafkahq:
+#    image: tchiotludo/kafkahq
+#    networks:
+#      - net
+#    ports:
+#      - 8080:8080
+#    links:
+#      - kafka
+#    environment:
+#      KAFKAHQ_CONFIGURATION: |
+#        kafkahq:
+#          connections:
+#            docker-kafka-server:
+#              properties:
+#                bootstrap.servers: "kafka:9092"
+
+
+  postgres:
+    image: postgres:12
+    restart: unless-stopped
+    shm_size: 4g
+    ports:
+      - 5432:5432
+    volumes:
+      - postgres:/var/lib/postgresql/data
+    networks:
+      - net
+    environment:
+      POSTGRES_USER: exflo
+      POSTGRES_PASSWORD: exflo
+      POSTGRES_DB: exflo
+
+  pgweb:
+    image: sosedoff/pgweb:0.11.5
+    restart: unless-stopped
+    networks:
+      - net
+    depends_on:
+      - postgres
+    ports:
+      - 8082:8082
+    command: "/usr/bin/pgweb --bind=0.0.0.0 --listen=8082 --url postgres://exflo:exflo@postgres:5432/exflo?sslmode=disable"
+
+  besu:
+    image: 41north/exflo
+    networks:
+      - net
+    depends_on:
+      - postgres
+    volumes:
+      - besu:/opt/besu/data
+    environment:
+      BESU_LOGGING: info
+      BESU_NETWORK: ROPSTEN
+      BESU_BOOT_NODES: "enode://20c9ad97c081d63397d7b685a412227a40e23c8bdc6688c6f37e97cfbc22d2b4d1db1510d8f61e6a8866ad7f0e17c02b14182d37ea7c3c8b9c2683aeb6b733a1@52.169.14.227:30303,enode://6ce05930c72abc632c58e2e4324f7c7ea478cec0ed4fa2528982cf34483094e9cbc9216e7aa349691242576d552a2a56aaeae426c5303ded677ce455ba1acd9d@13.84.180.240:30303"
+      BESU_SYNC_MODE: FULL
+      BESU_DATA_PATH: /opt/besu/data
+      BESU_PLUGIN_EXFLO_KAFKA_ENABLED: 'false'
+      BESU_PLUGIN_EXFLO_KAFKA_BOOTSTRAP_SERVERS: kafka:9092
+      BESU_PLUGIN_EXFLO_POSTGRES_ENABLED: 'true'
+      BESU_PLUGIN_EXFLO_POSTGRES_JDBC_URL: jdbc:postgresql://postgres/exflo?user=exflo&password=exflo

ingestion/base/build.gradle.kts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import dev.north.fortyone.gradle.solidity.EVMVersion.ISTANBUL
 import dev.north.fortyone.gradle.solidity.OutputComponent
 import io.exflo.gradle.tasks.ClassOutput
@@ -24,9 +23,8 @@ import io.exflo.gradle.tasks.Web3KtCodegenTask
 plugins {
     `java-library`
     kotlin("jvm")
-    id("com.github.johnrengelman.shadow")
     id("org.jlleitschuh.gradle.ktlint")
-    id("dev.north.fortyone.solidity") version "0.1.0"
+    id("dev.north.fortyone.solidity") version "0.1.1"
 }
 
 dependencies {
@@ -71,14 +69,12 @@ ktlint {
 }
 
 solidity {
+    attachToBuild.set(false)
     dockerSolidityImage.set("ethereum/solc:0.5.13")
     evmVersion.set(ISTANBUL)
     outputComponents.set(listOf(OutputComponent.BIN_RUNTIME, OutputComponent.ABI))
 }
 
-val build: DefaultTask by tasks
-build.dependsOn(tasks.shadowJar)
-
 tasks {
     register<Web3KtCodegenTask>("generateContractWrappers") {
         dependsOn(project.tasks["compileSolidity"])
@@ -98,16 +94,6 @@ tasks {
         )
     }
 
-    withType<ShadowJar> {
-        archiveBaseName.set(project.name)
-        archiveClassifier.set("")
-        minimize()
-    }
-
-    withType<Jar> {
-        enabled = false
-    }
-
     withType<Test> {
         useJUnitPlatform()
     }

ingestion/base/src/main/kotlin/io/exflo/ingestion/ExfloPlugin.kt

@@ -20,10 +20,13 @@ import io.exflo.ingestion.KoinModules.eventsModule
 import io.exflo.ingestion.KoinModules.stateModule
 import io.exflo.ingestion.KoinModules.storageModule
 import io.exflo.ingestion.extensions.reflektField
+import io.exflo.ingestion.storage.InterceptingKeyValueStorageFactory
+import io.exflo.ingestion.storage.InterceptingPrivacyKeyValueStorageFactory
 import io.exflo.ingestion.tokens.precompiled.PrecompiledContractsFactory
 import io.exflo.ingestion.tracker.BlockWriter
 import io.exflo.ingestion.tracker.ChainTracker
 import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
 import org.hyperledger.besu.cli.BesuCommand
 import org.hyperledger.besu.cli.config.EthNetworkConfig
 import org.hyperledger.besu.config.GenesisConfigFile
@@ -33,6 +36,8 @@ import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions
 import org.hyperledger.besu.plugin.BesuContext
 import org.hyperledger.besu.plugin.BesuPlugin
 import org.hyperledger.besu.plugin.services.PicoCLIOptions
+import org.hyperledger.besu.plugin.services.StorageService
+import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory
 import org.koin.core.KoinApplication
 import org.koin.core.context.startKoin
 import org.koin.core.context.stopKoin
@@ -43,12 +48,12 @@ import picocli.CommandLine
 @Suppress("MemberVisibilityCanBePrivate")
 abstract class ExfloPlugin<T : ExfloCliOptions> : BesuPlugin {
 
-    private val log = LogManager.getLogger()
-
     protected abstract val name: String
 
     protected abstract val options: T
 
+    private lateinit var log: Logger
+
     protected lateinit var context: BesuContext
 
     private lateinit var blockWriter: BlockWriter
@@ -57,13 +62,10 @@ abstract class ExfloPlugin<T : ExfloCliOptions> : BesuPlugin {
 
     private lateinit var besuCommand: BesuCommand
 
-    private val rocksDBPlugin = ExfloRocksDBPlugin()
-
     override fun register(context: BesuContext) {
 
-        rocksDBPlugin.register(context)
+        log = LogManager.getLogger(name)
 
-        log.debug("Registering plugin")
         this.context = context
 
         val cmdlineOptions = context.getService(PicoCLIOptions::class.java)
@@ -76,18 +78,61 @@ abstract class ExfloPlugin<T : ExfloCliOptions> : BesuPlugin {
         commandLine = reflektField(cliOptions, "commandLine")
         besuCommand = commandLine.commandSpec.userObject() as BesuCommand
 
+        registerStorageInterceptors()
+
         log.info("Plugin registered")
     }
 
+    private fun registerStorageInterceptors() {
+
+        val storageService = context
+            .getService(StorageService::class.java)
+            .get()
+
+        // get the original factories registered by the rocks db plugin
+
+        val keyValueStorageFactory = storageService
+            .getByName("rocksdb")
+            .get()
+
+        val privacyKeyValueStorageFactory = storageService
+            .getByName("rocksdb-privacy")
+            .get()
+
+        // determine if another exflo plugin has already decorated the base storage factories and capture their
+        // references, otherwise register our intercepting factories
+
+        if (
+            keyValueStorageFactory is InterceptingKeyValueStorageFactory &&
+            privacyKeyValueStorageFactory is InterceptingPrivacyKeyValueStorageFactory
+        ) {
+            // we have already setup the interceptors in another exflo plugin
+            return
+        }
+
+        val interceptingKeyValueStorageFactory =
+            InterceptingKeyValueStorageFactory(keyValueStorageFactory)
+
+        val interceptingPrivacyKeyValueStorageFactory =
+            InterceptingPrivacyKeyValueStorageFactory(privacyKeyValueStorageFactory as PrivacyKeyValueStorageFactory)
+
+        storageService.registerKeyValueStorage(interceptingKeyValueStorageFactory)
+        storageService.registerKeyValueStorage(interceptingPrivacyKeyValueStorageFactory)
+    }
+
     protected abstract fun implKoinModules(): List<Module>
 
     protected open fun implStart(koinApp: KoinApplication) {}
 
     override fun start() {
+
+        if (!options.enabled) {
+            return
+        }
+
         log.debug("Starting plugin")
 
         try {
-            rocksDBPlugin.start()
 
             val networkConfig = reflektField<EthNetworkConfig>(besuCommand, "ethNetworkConfig")
             val genesisConfigFile = GenesisConfigFile.fromConfig(networkConfig.genesisConfig)
@@ -105,6 +150,9 @@ abstract class ExfloPlugin<T : ExfloCliOptions> : BesuPlugin {
             // create a module for injecting various basic context objects
             val contextModule = module {
                 single { context }
+                // we capture the classloader as plugins are executed under a custom classloader and we need to
+                // specify this in some places to ensure behaviour
+                single<ClassLoader> { ExfloPlugin::class.java.classLoader }
                 single { networkConfig }
                 single { protocolSchedule }
                 single { genesisState }
@@ -139,10 +187,14 @@ abstract class ExfloPlugin<T : ExfloCliOptions> : BesuPlugin {
     }
 
     override fun stop() {
+
+        if (!options.enabled) {
+            return
+        }
+
         log.debug("Stopping plugin")
         blockWriter.stop()
         stopKoin()
-        rocksDBPlugin.stop()
     }
 }
 
@@ -151,6 +203,8 @@ abstract class ExfloPlugin<T : ExfloCliOptions> : BesuPlugin {
  */
 interface ExfloCliOptions {
 
+    var enabled: Boolean
+
     var startBlockOverride: Long?
 
     var maxForkSize: Int