[feat][io] AzureDataExplorer/Kusto Sink for Pulsar (apache#22006)

- https://learn.microsoft.com/en-us/azure/data-explorer/ Co-authored-by: Ramachandran A G <[email protected]>
hanmz · Feb 12, 2025 · 2d0b0d2 · 2d0b0d2
1 parent 86303dc
commit 2d0b0d2
Show file tree

Hide file tree

Showing 16 changed files with 1,025 additions and 1 deletion.
diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml
@@ -81,5 +81,6 @@
     <file><source>${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/alluxio/target/pulsar-io-alluxio-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/azure-data-explorer/target/pulsar-io-azuredataexplorer-${project.version}.nar</source></file>
   </files>
 </assembly>
diff --git a/pulsar-io/azure-data-explorer/docker-compose.yml b/pulsar-io/azure-data-explorer/docker-compose.yml
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+version: '3.0'
+
+services:
+  pulsar-server:
+    command: bin/pulsar standalone
+    image: apachepulsar/pulsar-all:latest
+    container_name: pulsar-server
+    hostname: pulsar-server
+    volumes:
+      - ./target/pulsar-io-azuredataexplorer-3.2.0-SNAPSHOT.nar:/pulsar/connectors/pulsar-io-azuredataexplorer-3.2.0-SNAPSHOT.nar
+    ports:
+      - 8080:8080
+      - 6650:6650
+    networks:
+      - custom-sink-connector
+    healthcheck:
+      interval: 10s
+      retries: 20
+      test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8080/admin/v2/clusters/standalone
+
+volumes:
+  custom-sink-connector:
+    driver: local
+
+networks:
+  custom-sink-connector:
+    driver: bridge
diff --git a/pulsar-io/azure-data-explorer/pom.xml b/pulsar-io/azure-data-explorer/pom.xml
@@ -0,0 +1,83 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.pulsar</groupId>
+        <artifactId>pulsar-io</artifactId>
+        <version>3.3.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>pulsar-io-azuredataexplorer</artifactId>
+    <name>Pulsar IO :: AzureDataExplorer</name>
+
+    <properties>
+        <kusto.sdk.version>5.0.4</kusto.sdk.version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>pulsar-io-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>pulsar-io-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.pulsar</groupId>
+            <artifactId>pulsar-functions-instance</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.microsoft.azure.kusto/kusto-data -->
+        <dependency>
+            <groupId>com.microsoft.azure.kusto</groupId>
+            <artifactId>kusto-data</artifactId>
+            <version>${kusto.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.microsoft.azure.kusto/kusto-ingest -->
+        <dependency>
+            <groupId>com.microsoft.azure.kusto</groupId>
+            <artifactId>kusto-ingest</artifactId>
+            <version>${kusto.sdk.version}</version>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.nifi</groupId>
+                <artifactId>nifi-nar-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/...re-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXPulsarEvent.java b/...re-data-explorer/src/main/java/org/apache/pulsar/io/azuredataexplorer/ADXPulsarEvent.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.azuredataexplorer;
+
+import java.time.Instant;
+import lombok.Data;
+import lombok.experimental.Accessors;
+
+@Data
+@Accessors(chain = true)
+public class ADXPulsarEvent {
+    private String key;
+    private String value;
+    private String properties;
+    private String producerName;
+    private long sequenceId;
+    private Instant eventTime;
+}