Merge pull request #18 from Bisnode/update_kafka_2.13-2.7.0

Update to Kafka 2.13-2.7.0

.github/workflows/build.yaml

@@ -15,6 +15,8 @@ jobs:
         run: wget -O opa https://openpolicyagent.org/downloads/latest/opa_linux_amd64
       - name: Install opa
         run: sudo mv opa /usr/local/bin/ && sudo chmod +x /usr/local/bin/opa
+      - name: Test example rego
+        run: opa test ./src/main/rego/policy.rego ./src/test/rego/policy_test.rego
       - name: Build plugin
         run: ./gradlew check shadowJar
       - name: Jacoco test report
@@ -33,5 +35,3 @@ jobs:
           token: ${{secrets.CODECOV_TOKEN}}
           file: ./build/reports/opa/opa-codecov-coverage.json
           name: opa-policies
-      - name: Test example rego domain_based_policy
-        run: opa test ./examples/rego/domain_based_policy/policy.rego ./examples/rego/domain_based_policy/test/

.github/workflows/release.yaml

@@ -20,4 +20,3 @@ jobs:
           OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
         with:
           arguments: publishAllPublicationsToOSSRHRepository -PossrhUsername=$OSSRH_USERNAME -PossrhPassword=$OSSRH_PASSWORD -Dorg.gradle.internal.publish.checksums.insecure=true
-

.gitignore

@@ -17,3 +17,6 @@ atlassian-ide-plugin.xml
 *.rar
 hs_err_pid*
 
+.metals
+.project
+.bloop

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+# To use in local development:
+# - Run `pip3 install pre-commit` (or `brew install pre-commit`)
+# - In the project root directory (where this file resides) run: `pre-commit install`
+# - Done! The below hooks should now automatically run before commit (where applicable).
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.3.0
+  hooks:
+    - id: no-commit-to-branch # Defaults to refuse commits to master
+    - id: trailing-whitespace
+    - id: check-added-large-files
+    - id: check-merge-conflict
+    - id: check-json
+    - id: check-xml
+    - id: check-yaml
+    - id: detect-private-key
+    - id: detect-aws-credentials
+    - id: mixed-line-ending
+    - id: end-of-file-fixer
+
+- repo: https://github.com/anderseknert/pre-commit-opa
+  rev: v1.4.0
+  hooks:
+  - id: opa-fmt

CHANGELOG.md

@@ -4,7 +4,66 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Unreleased 
+## [1.0.0] - 2021-03-XX
+
+### Changes
+
+#### Breaking changes:
+
+- Update to use Scala 2.13
+  - Requires a Kafka cluster running 2.13
+- Update to Kafka library 2.7.0
+  - Requires Kafka 2.7.X
+- New input structure to OPA
+  - You will need to adjust policies to work with the new input structure. See an example of the new structure down below. We suggest to update your policies before upgrading, to work with both the old and the new structure. Then upgrade the plugin and then remove the old policies.`
+
+New input structure:
+```json
+{
+    "action": {
+        "logIfAllowed": true,
+        "logIfDenied": true,
+        "operation": "DESCRIBE",
+        "resourcePattern": {
+            "name": "alice-topic",
+            "patternType": "LITERAL",
+            "resourceType": "TOPIC",
+            "unknown": false
+        },
+        "resourceReferenceCount": 1
+    },
+    "requestContext": {
+        "clientAddress": "192.168.64.1",
+        "clientInformation": {
+            "softwareName": "unknown",
+            "softwareVersion": "unknown"
+        },
+        "connectionId": "192.168.64.4:9092-192.168.64.1:58864-0",
+        "header": {
+            "data": {
+                "clientId": "rdkafka",
+                "correlationId": 5,
+                "requestApiKey": 3,
+                "requestApiVersion": 2
+            },
+            "headerVersion": 1
+        },
+        "listenerName": "SASL_PLAINTEXT",
+        "principal": {
+            "name": "alice-consumer",
+            "principalType": "User"
+        },
+        "securityProtocol": "SASL_PLAINTEXT"
+    }
+}
+```
+
+#### Other changes
+
+- Include `guava` and `paranamer` in the shadowJar since it's been excluded from the Kafka installation
+- Update to use the new Kafka libraries to use the new API
+- Update OPA policy and tests to work with the new input structure
+- Update version on various dependencies
 - Add Maven information to README
 - Update changelog
 

README.md

@@ -1,4 +1,4 @@
-#  Open Policy Agent plugin for Kafka authorization 
+#  Open Policy Agent plugin for Kafka authorization
 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.bisnode.kafka.authorization/opa-authorizer/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.bisnode.kafka.authorization/opa-authorizer)
 ![](https://github.com/Bisnode/opa-kafka-plugin/workflows/build/badge.svg)
 [![codecov](https://codecov.io/gh/Bisnode/opa-kafka-plugin/branch/master/graph/badge.svg)](https://codecov.io/gh/Bisnode/opa-kafka-plugin)
@@ -13,11 +13,11 @@ Open Policy Agent (OPA) plugin for Kafka authorization.
 
 ## Installation
 
-### 
+###
 
 Download the latest OPA authorizer plugin jar from [Releases](https://github.com/Bisnode/opa-kafka-plugin/releases/) (or [Maven Central](https://search.maven.org/artifact/com.bisnode.kafka.authorization/opa-authorizer)) and put the
 file (`opa-authorizer-{$VERSION}.jar`) somewhere Kafka recognizes it - this could be directly  in Kafkas `libs` directory
-or in a separate plugin directory pointed out to Kafka at startup, e.g: 
+or in a separate plugin directory pointed out to Kafka at startup, e.g:
 
 `CLASSPATH=/usr/local/share/kafka/plugins/*`
 
@@ -34,46 +34,82 @@ The plugin supports the following properties:
 | `opa.authorizer.cache.initial.capacity` | `5000` | `5000` | Initial decision cache size. |
 | `opa.authorizer.cache.maximum.size` | `50000` | `50000` | Max decision cache size. |
 | `opa.authorizer.cache.expire.after.seconds` | `3600` | `3600` | Decision cache expiry in seconds. |
-| `super.users` | `User:alice;User:bob` | `` | Super users which are always allowed. |
+| `super.users` | `User:alice;User:bob` |  | Super users which are always allowed. |
 
 ## Usage
 
 Example structure of input data provided from opa-kafka-plugin to Open Policy Agent.
 ```
 {
-  "operation": {
-    "name": "Write"
-  },
-  "resource": {
-    "resourceType": {
-      "name": "Topic"
+    "action": {
+        "logIfAllowed": true,
+        "logIfDenied": true,
+        "operation": "DESCRIBE",
+        "resourcePattern": {
+            "name": "alice-topic",
+            "patternType": "LITERAL",
+            "resourceType": "TOPIC",
+            "unknown": false
+        },
+        "resourceReferenceCount": 1
     },
-    "name": "alice-topic1"
-  },
-  "session": {
-    "principal": {
-      "principalType": "alice-producer"
-    },
-    "clientAddress": "172.21.0.5",
-    "sanitizedUser": "alice-producer"
-  }
+    "requestContext": {
+        "clientAddress": "192.168.64.1",
+        "clientInformation": {
+            "softwareName": "unknown",
+            "softwareVersion": "unknown"
+        },
+        "connectionId": "192.168.64.4:9092-192.168.64.1:58864-0",
+        "header": {
+            "data": {
+                "clientId": "rdkafka",
+                "correlationId": 5,
+                "requestApiKey": 3,
+                "requestApiVersion": 2
+            },
+            "headerVersion": 1
+        },
+        "listenerName": "SASL_PLAINTEXT",
+        "principal": {
+            "name": "alice-consumer",
+            "principalType": "User"
+        },
+        "securityProtocol": "SASL_PLAINTEXT"
+    }
 }
 ```
 
 The following table summarizes the supported resource types and operation names.
 
-| `input.resourceType.name` | `input.operation.name` |
+| `input.action.resourcePattern.resourceType` | `input.action.operation` |
 | --- | --- |
-| `Cluster` | `ClusterAction` |
-| `Cluster` | `Create` |
-| `Cluster` | `Describe` |
-| `Group` | `Read` |
-| `Group` | `Describe` |
-| `Topic` | `Alter` |
-| `Topic` | `Delete` |
-| `Topic` | `Describe` |
-| `Topic` | `Read` |
-| `Topic` | `Write` |
+| `CLUSTER` | `CLUSTER_ACTION` |
+| `CLUSTER` | `CREATE` |
+| `CLUSTER` | `DESCRIBE` |
+| `GROUP` | `READ` |
+| `GROUP` | `DESCRIPTION` |
+| `TOPIC` | `ALTER` |
+| `TOPIC` | `DELETE` |
+| `TOPIC` | `DESCRIBE` |
+| `TOPIC` | `READ` |
+| `TOPIC` | `WRITE` |
+
+It's likely possible to use all different resource types and operations described in the Kafka API docs:
+https://kafka.apache.org/24/javadoc/org/apache/kafka/common/acl/AclOperation.html
+https://kafka.apache.org/24/javadoc/org/apache/kafka/common/resource/ResourceType.html
+
+### Security protocols:
+
+| Protocol | Decsription |
+|---|---|
+| `PLAINTEXT` | Un-authenticated, non-encrypted channel |
+| `SASL_PLAINTEXT` | authenticated, non-encrypted channel |
+| `SASL` | authenticated, SSL channel |
+| `SSL` | SSL channel |
+
+More info:
+
+https://kafka.apache.org/24/javadoc/org/apache/kafka/common/security/auth/SecurityProtocol.html
 
 ### Policy sample
 
@@ -92,47 +128,6 @@ User `alice-consumer` will be...
 
 [See sample rego](src/main/rego/README.md)
 
-## Performance
-Performance results of opa-kafka-plugin compared with ACL's and even unauthorized
-access to Kafka shows that there is a very little trade off when it comes to
-performance when using either this opa-kafka-plugin or ACL's.
-
-The tests were made with the following setup:
-
-|Resources|#|
-|---|---|
-| Brokers | 3 |
-| Partitions per topic | 10 |
-| Replicas per topic | 3 |
-| Zookeeper nodes | 3 |
-
-Background noise on one topic with a producer producing 5000 msgs/s with message
-size of 512b were used in all tests.
-
-### opa-kafka-plugin test results
-
-|Test #|Records sent|Payload(b)|records/s|MB/sec|avg latency (ms)|latency avg 50th perc (ms)|latency avg 95th perc (ms)|latency avg 99th perc (ms)|latency avg 99,99 perc (ms)|
-|---|---|---|---|---|---|---|---|---|---|
-|1|102000|666|170|0.11|4.07|1|2|18|748|
-|2|102000|330|170|0.05|2.03|1|2|18|754|
-|3|102000|1200|170|0.19|2.09|1|2|19|946|
-
-### ACL test results
-
-|Test #|Records sent|Payload(b)|records/s|MB/sec|avg latency (ms)|latency avg 50th perc (ms)|latency avg 95th perc (ms)|latency avg 99th perc (ms)|latency avg 99,99 perc (ms)|
-|---|---|---|---|---|---|---|---|---|---|
-|1|102000|666|170|0.11|3.45|1|4|21|604|
-|2|102000|330|170|0.05|3.21|1|3|22|566|
-|3|102000|1200|170|0.19|3.01|1|3|19|504|
-
-### No authorization test results
-
-|Test #|Records sent|Payload(b)|records/s|MB/sec|avg latency (ms)|latency avg 50th perc (ms)|latency avg 95th perc (ms)|latency avg 99th perc (ms)|latency avg 99,99 perc (ms)|
-|---|---|---|---|---|---|---|---|---|---|
-|1|102000|666|170| 1.11|1.70|1|2|19|128|
-|2|102000|330|170| 0.05|2.03|1|2|18|298|
-|3|102000|1200|170| 0.19|2.09|1|2|18|332|
-
 ## Build from source
 
 Using gradle wrapper: `./gradlew clean test shadowJar`
@@ -146,4 +141,3 @@ Set log level `log4j.logger.com.bisnode=INFO` in `config/log4j.properties`
 Use DEBUG or TRACE for debugging.
 
 In a busy Kafka cluster it might be good to tweak the cache since it may produce a lot of log entries in Open Policy Agent, especially if decision logs are turned on. If the policy isn't dynamically updated very often it's recommended to cache a lot to improve performance and reduce the amount of log entries.
-

build.gradle

@@ -3,12 +3,12 @@ plugins {
     id 'jacoco'
     id 'signing'
     id 'maven-publish'
-    id 'com.github.johnrengelman.shadow' version '5.1.0'
-    id 'com.bisnode.opa' version '0.3.0'
+    id 'com.github.johnrengelman.shadow' version '6.1.0'
+    id 'com.bisnode.opa' version '0.3.1'
 }
 
 group 'com.bisnode.kafka.authorization'
-version '0.4.2'
+version '1.0.0'
 
 java {
     sourceCompatibility = JavaVersion.VERSION_11
@@ -21,15 +21,17 @@ repositories {
     mavenCentral()
 }
 
+// See versions used in Kafka here https://github.com/apache/kafka/blob/2.7.0/gradle/dependencies.gradle
 dependencies {
-    implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.12', version: '2.10.0'
-    implementation group: 'org.apache.kafka', name: 'kafka_2.12', version: '2.3.1'
-    implementation group: 'com.google.guava', name: 'guava', version: '30.0-jre'
+    implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.13', version: '2.10.5'
+    implementation group: 'org.apache.kafka', name: 'kafka_2.13', version: '2.7.0'
+    implementation group: 'com.google.guava', name: 'guava', version: '30.1-jre'
     implementation group: 'com.google.code.findbugs', name: 'jsr305', version: '3.0.2'
 
-    testImplementation group: 'org.scalatest', name: 'scalatest_2.12', version: '3.0.0'
+    testImplementation group: 'org.scalatest', name: 'scalatest_2.13', version: '3.2.5'
+    testImplementation group: 'org.scalatestplus', name: 'junit-4-13_2.13', version: '3.2.5.0'
     testImplementation group: 'junit', name: 'junit', version: '4.12'
-    testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.12.1'
+    testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.14.0'
 }
 
 shadowJar {
@@ -38,7 +40,8 @@ shadowJar {
             !(it.moduleGroup in [
                     'com.bisnode.kafka.authorization',
                     'com.fasterxml.jackson.module',
-                    'com.thoughtworks.paranamer'
+                    'com.thoughtworks.paranamer',
+                    'com.google.guava'
             ])
         })
     }
@@ -51,12 +54,18 @@ jacocoTestReport {
     }
 }
 
+test {
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+}
+
 publishing {
     publications {
         mavenJava(MavenPublication) {
             groupId ='com.bisnode.kafka.authorization'
             artifactId = 'opa-authorizer'
-            version = '0.4.2'
+            version = '1.0.0'
 
             from components.java