Merge remote-tracking branch 'apache/master' into air-compressor

README.md

@@ -19,7 +19,7 @@
 
 -->
 
-![logo](site/img/pulsar.png)
+![logo](site2/website/static/img/pulsar.svg)
 
 Pulsar is a distributed pub-sub messaging platform with a very
 flexible messaging model and an intuitive client API.
@@ -50,7 +50,7 @@ Learn more about Pulsar at https://pulsar.apache.org
 
 Requirements:
  * Java JDK 1.8
- * Maven
+ * Maven 3.3.9+
 
 Compile and install:
 

bin/pulsar

@@ -44,7 +44,13 @@ FUNCTIONS_EXTRA_DEPS_DIR=${PULSAR_FUNCTIONS_EXTRA_DEPS_DIR:-"${DEFAULT_FUNCTIONS
 SQL_HOME=$PULSAR_HOME/pulsar-sql
 PRESTO_HOME=${PULSAR_HOME}/lib/presto
 
+# Check bookkeeper env and load bkenv.sh
+if [ -f "$PULSAR_HOME/conf/bkenv.sh" ]
+then
+    . "$PULSAR_HOME/conf/bkenv.sh"
+fi
 
+# Check pulsar env and load pulser_env.sh
 if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ]
 then
     . "$PULSAR_HOME/conf/pulsar_env.sh"
@@ -135,6 +141,7 @@ where command is one of:
     standalone          Run a broker server with local bookies and local zookeeper
 
     initialize-cluster-metadata     One-time metadata initialization
+    initialize-transaction-coordinator-metadata     One-time transaction coordinator metadata initialization
     compact-topic       Run compaction against a topic
     zookeeper-shell     Open a ZK shell client
     tokens              Utility to create authentication tokens
@@ -328,6 +335,8 @@ elif [ $COMMAND == "standalone" ]; then
     exec $JAVA $OPTS $ASPECTJ_AGENT ${ZK_OPTS} -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarStandaloneStarter --config $PULSAR_STANDALONE_CONF $@
 elif [ $COMMAND == "initialize-cluster-metadata" ]; then
     exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataSetup $@
+elif [ $COMMAND == "initialize-transaction-coordinator-metadata" ]; then
+    exec $JAVA $OPTS org.apache.pulsar.PulsarTransactionCoordinatorMetadataSetup $@
 elif [ $COMMAND == "zookeeper-shell" ]; then
     exec $JAVA $OPTS org.apache.zookeeper.ZooKeeperMain $@
 elif [ $COMMAND == "compact-topic" ]; then

bin/pulsar-daemon

@@ -40,6 +40,12 @@ EOF
 BINDIR=$(dirname "$0")
 PULSAR_HOME=$(cd $BINDIR/..;pwd)
 
+# Check bookkeeper env and load bkenv.sh
+if [ -f "$PULSAR_HOME/conf/bkenv.sh" ]
+then
+    . "$PULSAR_HOME/conf/bkenv.sh"
+fi
+
 if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ]
 then
     . "$PULSAR_HOME/conf/pulsar_env.sh"

bin/pulsar-perf

@@ -24,6 +24,12 @@ PULSAR_HOME=`cd $BINDIR/..;pwd`
 DEFAULT_CLIENT_CONF=${PULSAR_CLIENT_CONF:-"$PULSAR_HOME/conf/client.conf"}
 DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
 
+# Check bookkeeper env and load bkenv.sh
+if [ -f "$PULSAR_HOME/conf/bkenv.sh" ]
+then
+    . "$PULSAR_HOME/conf/bkenv.sh"
+fi
+
 if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ]
 then
     . "$PULSAR_HOME/conf/pulsar_env.sh"

buildtools/src/main/java/org/apache/pulsar/tests/PulsarTestListener.java

@@ -29,20 +29,20 @@ public class PulsarTestListener implements ITestListener {
 
     @Override
     public void onTestStart(ITestResult result) {
-        System.out.format("------- Starting test %s.%s(%s)-------\n", result.getTestClass(), result.getTestName(),
-                Arrays.toString(result.getParameters()));
+        System.out.format("------- Starting test %s.%s(%s)-------\n", result.getTestClass(),
+                result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
     }
 
     @Override
     public void onTestSuccess(ITestResult result) {
-        System.out.format("------- SUCCESS -- %s.%s(%s)-------", result.getTestClass(), result.getTestName(),
-                Arrays.toString(result.getParameters()));
+        System.out.format("------- SUCCESS -- %s.%s(%s)-------", result.getTestClass(),
+                result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
     }
 
     @Override
     public void onTestFailure(ITestResult result) {
-        System.out.format("!!!!!!!!! FAILURE-- %s.%s(%s)-------\n", result.getTestClass(), result.getTestName(),
-                Arrays.toString(result.getParameters()));
+        System.out.format("!!!!!!!!! FAILURE-- %s.%s(%s)-------\n", result.getTestClass(),
+                result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
 
         if (result.getThrowable() instanceof ThreadTimeoutException) {
             System.out.println("====== THREAD DUMPS ======");
@@ -52,8 +52,8 @@ public void onTestFailure(ITestResult result) {
 
     @Override
     public void onTestSkipped(ITestResult result) {
-        System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------", result.getTestClass(), result.getTestName(),
-                Arrays.toString(result.getParameters()));
+        System.out.format("~~~~~~~~~ SKIPPED -- %s.%s(%s)-------", result.getTestClass(),
+                result.getMethod().getMethodName(), Arrays.toString(result.getParameters()));
     }
 
     @Override
@@ -68,6 +68,5 @@ public void onStart(ITestContext context) {
 
     @Override
     public void onFinish(ITestContext context) {
-
     }
 }

conf/broker.conf

@@ -175,6 +175,10 @@ maxUnackedMessagesPerBroker=0
 # limit/2 messages
 maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16
 
+# Tick time to schedule task that checks publish rate limiting across all topics  
+# (Disable publish throttling with value 0)
+publisherThrottlingTickTimeMillis=10
+
 # Too many subscribe requests from a consumer can cause broker rewinding consumer cursors and loading data from bookies,
 # hence causing high network bandwidth usage
 # When the positive value is set, broker will throttle the subscribe requests for one consumer.
@@ -795,3 +799,9 @@ globalZookeeperServers=
 
 # Deprecated - Enable TLS when talking with other clusters to replicate messages
 replicationTlsEnabled=false
+
+### --- Transaction config variables --- ###
+
+# Enable transaction coordinator in broker
+transactionCoordinatorEnabled=true
+transactionMetadataStoreProviderClassName=org.apache.pulsar.transaction.coordinator.impl.InMemTransactionMetadataStoreProvider

conf/functions_worker.yml

@@ -92,17 +92,33 @@ topicCompactionFrequencySec: 1800
 # Function Runtime Management
 ###############################
 
-#threadContainerFactory:
+#### Process Runtime  ####
+# Pulsar function instances are launched as processes
+
+functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory
+functionRuntimeFactoryConfigs:
+    # location of log files for functions
+    logDirectory: /tmp
+    # change the jar location only when you put the java instance jar in a different location
+    javaInstanceJarLocation:
+    # change the python instance location only when you put the python instance jar in a different location
+    pythonInstanceLocation:
+    # change the extra dependencies location:
+    extraFunctionDependenciesDir:
+
+#### Thread Runtime ####
+# Pulsar function instances are run as threads
+
+#functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory
+#functionRuntimeFactoryConfigs:
+#  # thread group name
 #  threadGroupName: "Thread Function Container Group"
-processContainerFactory:
-  logDirectory:
-  # change the jar location only when you put the java instance jar in a different location
-  javaInstanceJarLocation:
-  # change the python instance location only when you put the python instance jar in a different location
-  pythonInstanceLocation:
-  # change the extra dependencies location:
-  extraFunctionDependenciesDir:
-#kubernetesContainerFactory:
+
+#### Kubernetes Runtime ####
+# Pulsar function are deployed to Kubernetes
+
+#functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory
+#functionRuntimeFactoryConfigs:
 #  # uri to kubernetes cluster, leave it to empty and it will use the kubernetes settings in function worker
 #  k8Uri:
 #  # the kubernetes namespace to run the function instances. it is `default`, if this setting is left to be empty
@@ -116,7 +132,7 @@ processContainerFactory:
 #  # setting this to true is let function worker to submit functions to the same k8s cluster as function worker
 #  # is running. setting this to false if your function worker is not running as a k8 pod.
 #  submittingInsidePod: false
-#  # setting the pulsar service url that pulsar function should use to connect to pulsar 
+#  # setting the pulsar service url that pulsar function should use to connect to pulsar
 #  # if it is not set, it will use the pulsar service url configured in worker service
 #  pulsarServiceUrl:
 #  # setting the pulsar admin url that pulsar function should use to connect to pulsar

conf/pulsar_env.sh

@@ -59,7 +59,3 @@ PULSAR_EXTRA_OPTS=${PULSAR_EXTRA_OPTS:-" -Dpulsar.allocator.exit_on_oom=true -Di
 #Wait time before forcefully kill the pulser server instance, if the stop is not successful
 #PULSAR_STOP_TIMEOUT=
 
-# Set BOOKIE_EXTRA_OPTS option here to ensure that all pulsar scripts can work seamless with bookkeeper
-
-# Extra options to be passed to the jvm
-BOOKIE_EXTRA_OPTS="${BOOKIE_EXTRA_OPTS} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024"

conf/standalone.conf

@@ -143,6 +143,11 @@ maxUnackedMessagesPerBroker=0
 # limit/2 messages
 maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16
 
+# Tick time to schedule task that checks publish rate limiting across all topics
+# Reducing to lower value can give more accuracy while throttling publish but 
+# it uses more CPU to perform frequent check. (Disable publish throttling with value 0)
+publisherThrottlingTickTimeMillis=2
+
 # Default messages per second dispatch throttling-limit for every topic. Using a value of 0, is disabling default
 # message dispatch-throttling
 dispatchThrottlingRatePerTopicInMsg=0
@@ -322,9 +327,9 @@ managedLedgerDefaultWriteQuorum=1
 # Number of guaranteed copies (acks to wait before write is complete)
 managedLedgerDefaultAckQuorum=1
 
-# Default type of checksum to use when writing to BookKeeper. Default is "CRC32"
-# Other possible options are "CRC32C" (which is faster), "MAC" or "DUMMY" (no checksum).
-managedLedgerDigestType=CRC32
+# Default type of checksum to use when writing to BookKeeper. Default is "CRC32C"
+# Other possible options are "CRC32", "MAC" or "DUMMY" (no checksum).
+managedLedgerDigestType=CRC32C
 
 # Number of threads to be used for managed ledger tasks dispatching
 managedLedgerNumWorkerThreads=4
@@ -602,4 +607,7 @@ allowAutoTopicCreation=true
 allowAutoTopicCreationType=non-partitioned
 
 # The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned.
-defaultNumPartitions=1
+defaultNumPartitions=1
+
+### --- Transaction config variables --- ###
+transactionMetadataStoreProviderClassName=org.apache.pulsar.transaction.coordinator.impl.InMemTransactionMetadataStoreProvider

dashboard/Dockerfile

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-FROM python:3.7
+FROM python:3.7-stretch
 
 MAINTAINER Pulsar
 

dashboard/django/collector.py

@@ -79,7 +79,7 @@ def _fetch_broker_stats(cluster, broker_host_port, timestamp):
     active_broker.save()
 
     # Get topics stats
-    topics_stats = get(broker_url, '/admin/broker-stats/destinations')
+    topics_stats = get(broker_url, '/admin/v2/broker-stats/topics')
 
     clusters = dict((cluster.name, cluster) for cluster in Cluster.objects.all())
 

deployment/kubernetes/README.md

@@ -22,5 +22,5 @@
 This directory contains the Kubernetes services definitions for all
 the components required to do a complete Pulsar deployment.
 
-Refer to [Kubernetes.md](../../site/docs/latest/deployment/Kubernetes.md) document for instructions on
+Refer to [Kubernetes.md](../../site2/docs/deploy-kubernetes.md) document for instructions on
 how to deploy Pulsar on a Kubernetes cluster.

deployment/kubernetes/aws/README.md

@@ -22,5 +22,5 @@
 This directory contains the Kubernetes services definitions for all the
 components required to run Pulsar and use ebs volume as bookie storage.
 
-Refer to [Kubernetes.md](../site/docs/latest/deployment/Kubernetes.md) document for instructions on
+Refer to [Kubernetes.md](../../../site2/docs/deploy-kubernetes.md) document for instructions on
 how to deploy Pulsar on a Kubernetes cluster.

deployment/kubernetes/aws/zookeeper.yaml

@@ -60,7 +60,7 @@ spec:
             annotations:
                 pod.alpha.kubernetes.io/initialized: "true"
                 prometheus.io/scrape: "true"
-                prometheus.io/port: "8080"
+                prometheus.io/port: "8000"
 
         spec:
             # Make sure multiple pods of ZK don't get scheduled on the

deployment/kubernetes/generic/k8s-1-9-and-above/zookeeper.yaml

@@ -86,7 +86,7 @@ spec:
             annotations:
                 pod.alpha.kubernetes.io/initialized: "true"
                 prometheus.io/scrape: "true"
-                prometheus.io/port: "8080"
+                prometheus.io/port: "8000"
 
         spec:
             # Make sure multiple pods of ZK don't get scheduled on the

deployment/kubernetes/generic/original/zookeeper.yaml

@@ -60,7 +60,7 @@ spec:
             annotations:
                 pod.alpha.kubernetes.io/initialized: "true"
                 prometheus.io/scrape: "true"
-                prometheus.io/port: "8080"
+                prometheus.io/port: "8000"
 
         spec:
             # Make sure multiple pods of ZK don't get scheduled on the

deployment/kubernetes/google-kubernetes-engine/zookeeper.yaml

@@ -71,7 +71,7 @@ spec:
             annotations:
                 pod.alpha.kubernetes.io/initialized: "true"
                 prometheus.io/scrape: "true"
-                prometheus.io/port: "8080"
+                prometheus.io/port: "8000"
 
         spec:
             # Make sure multiple pods of ZK don't get scheduled on the

distribution/io/src/assemble/io.xml

@@ -66,6 +66,7 @@
     <file><source>${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar</source></file>

distribution/server/licenses/LICENSE-jcip.txt

@@ -0,0 +1,7 @@
+Copyright (c) 2005 Brian Goetz and Tim Peierls
+Released under the Creative Commons Attribution License
+  (http://creativecommons.org/licenses/by/2.5)
+Official home: http://www.jcip.net
+
+Any republication or derived work distributed in source code form
+must include this copyright and license notice.