diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh
index fc89f25d9d255..81b423afa1ec9 100755
--- a/bin/kafka-run-class.sh
+++ b/bin/kafka-run-class.sh
@@ -73,28 +73,50 @@ do
fi
done
-for file in "$base_dir"/clients/build/libs/kafka-clients*.jar;
-do
- if should_include_file "$file"; then
- CLASSPATH="$CLASSPATH":"$file"
- fi
-done
+if [ -z "$UPGRADE_KAFKA_STREAMS_TEST_VERSION" ]; then
+ clients_lib_dir=$(dirname $0)/../clients/build/libs
+ streams_lib_dir=$(dirname $0)/../streams/build/libs
+ rocksdb_lib_dir=$(dirname $0)/../streams/build/dependant-libs-${SCALA_VERSION}
+else
+ clients_lib_dir=/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs
+ streams_lib_dir=$clients_lib_dir
+ rocksdb_lib_dir=$streams_lib_dir
+fi
+
-for file in "$base_dir"/streams/build/libs/kafka-streams*.jar;
+for file in "$clients_lib_dir"/kafka-clients*.jar;
do
if should_include_file "$file"; then
CLASSPATH="$CLASSPATH":"$file"
fi
done
-for file in "$base_dir"/streams/examples/build/libs/kafka-streams-examples*.jar;
+for file in "$streams_lib_dir"/kafka-streams*.jar;
do
if should_include_file "$file"; then
CLASSPATH="$CLASSPATH":"$file"
fi
done
-for file in "$base_dir"/streams/build/dependant-libs-${SCALA_VERSION}/rocksdb*.jar;
+if [ -z "$UPGRADE_KAFKA_STREAMS_TEST_VERSION" ]; then
+ for file in "$base_dir"/streams/examples/build/libs/kafka-streams-examples*.jar;
+ do
+ if should_include_file "$file"; then
+ CLASSPATH="$CLASSPATH":"$file"
+ fi
+ done
+else
+ VERSION_NO_DOTS=`echo $UPGRADE_KAFKA_STREAMS_TEST_VERSION | sed 's/\.//g'`
+ SHORT_VERSION_NO_DOTS=${VERSION_NO_DOTS:0:((${#VERSION_NO_DOTS} - 1))} # remove last char, ie, bug-fix number
+ for file in "$base_dir"/streams/upgrade-system-tests-$SHORT_VERSION_NO_DOTS/build/libs/kafka-streams-upgrade-system-tests*.jar;
+ do
+ if should_include_file "$file"; then
+ CLASSPATH="$CLASSPATH":"$file"
+ fi
+ done
+fi
+
+for file in "$rocksdb_lib_dir"/rocksdb*.jar;
do
CLASSPATH="$CLASSPATH":"$file"
done
diff --git a/build.gradle b/build.gradle
index 7a3b4119f81cf..727bdeae8a526 100644
--- a/build.gradle
+++ b/build.gradle
@@ -961,6 +961,54 @@ project(':streams:examples') {
}
}
+project(':streams:upgrade-system-tests-0100') {
+ archivesBaseName = "kafka-streams-upgrade-system-tests-0100"
+
+ dependencies {
+ testCompile libs.kafkaStreams_0100
+ }
+
+ systemTestLibs {
+ dependsOn testJar
+ }
+}
+
+project(':streams:upgrade-system-tests-0101') {
+ archivesBaseName = "kafka-streams-upgrade-system-tests-0101"
+
+ dependencies {
+ testCompile libs.kafkaStreams_0101
+ }
+
+ systemTestLibs {
+ dependsOn testJar
+ }
+}
+
+project(':streams:upgrade-system-tests-0102') {
+ archivesBaseName = "kafka-streams-upgrade-system-tests-0102"
+
+ dependencies {
+ testCompile libs.kafkaStreams_0102
+ }
+
+ systemTestLibs {
+ dependsOn testJar
+ }
+}
+
+project(':streams:upgrade-system-tests-0110') {
+ archivesBaseName = "kafka-streams-upgrade-system-tests-0110"
+
+ dependencies {
+ testCompile libs.kafkaStreams_0110
+ }
+
+ systemTestLibs {
+ dependsOn testJar
+ }
+}
+
project(':jmh-benchmarks') {
apply plugin: 'com.github.johnrengelman.shadow'
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index acff3f625bcb3..bc7a2bd5779fa 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -181,7 +181,7 @@
files="SmokeTestDriver.java"/>
- If you want to upgrade from 0.11.0.x to 1.0.0 you don't need to do any code changes as the public API is fully backward compatible. + If you want to upgrade from 0.10.2.x or 0.11.0.x to 1.0.x you don't need to do any code changes as the public API is fully backward compatible. However, some public APIs were deprecated and thus it is recommended to update your code eventually to allow for future upgrades. - See below a complete list of 1.0.0 API and semantic changes that allow you to advance your application and/or simplify your code base, including the usage of new features. + See below a complete list of 1.0 and 0.11.0 API + and semantic changes that allow you to advance your application and/or simplify your code base, including the usage of new features. + Additionally, Streams API 1.0.x requires broker on-disk message format version 0.10 or higher; thus, you need to make sure that the message + format is configured correctly before you upgrade your Kafka Streams application.
- If you want to upgrade from 0.10.2.x to 0.11.0 you don't need to do any code changes as the public API is fully backward compatible. - However, some configuration parameters were deprecated and thus it is recommended to update your code eventually to allow for future upgrades. - See below a complete list of 0.11.0 API and semantic changes that allow you to advance your application and/or simplify your code base, including the usage of new features. + If you want to upgrade from 0.10.1.x to 1.0.x see the Upgrade Sections for 0.10.2, + 0.11.0, and + 1.0. + Note, that a brokers on-disk message format must be on version 0.10 or higher to run a Kafka Streams application version 1.0 or higher. + See below a complete list of 0.10.2, 0.11.0, + and 1.0 API and semantical changes that allow you to advance your application and/or simplify your code base, including the usage of new features.
- If you want to upgrade from 0.10.1.x to 0.10.2, see the Upgrade Section for 0.10.2. - It highlights incompatible changes you need to consider to upgrade your code and application. - See below a complete list of 0.10.2 API and semantic changes that allow you to advance your application and/or simplify your code base, including the usage of new features. -
- -
- If you want to upgrade from 0.10.0.x to 0.10.1, see the Upgrade Section for 0.10.1.
- It highlights incompatible changes you need to consider to upgrade your code and application.
- See below a complete list of 0.10.1 API changes that allow you to advance your application and/or simplify your code base, including the usage of new features.
+ Upgrading from 0.10.0.x to 1.0.x directly is also possible.
+ Note, that a brokers must be on version 0.10.1 or higher and on-disk message format must be on version 0.10 or higher
+ to run a Kafka Streams application version 1.0 or higher.
+ See Streams API changes in 0.10.1, Streams API changes in 0.10.2,
+ Streams API changes in 0.11.0, and Streams API changes in 1.0
+ for a complete list of API changes.
+ Upgrading to 1.0.2 requires two rolling bounces with config upgrade.from="0.10.0" set for first upgrade phase
+ (cf. KIP-268).
+ As an alternative, an offline upgrade is also possible.
upgrade.from is set to "0.10.0" for new version 1.0.2upgrade.mode Upgrading from 0.10.0.x to 1.0.0 or 1.0.1 requires an offline upgrade (rolling bounce upgrade is not supported)
+ProcessorContext#schedule(), Processor#punctuate() and KStreamBuilder, TopologyBuilder are being deprecated by new APIs.
- We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade.
+ We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade.
ProcessorContext#schedule(), Processor#punctuate() and KStreamBuilder, TopologyBuilder are being deprecated by new APIs.
+ We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade.
+ key.serde, value.serde and timestamp.extractor in configs, it is recommended to use their replaced configure parameter as these configs are deprecated. ProcessorContext#schedule(), Processor#punctuate() and KStreamBuilder, TopologyBuilder are being deprecated by new APIs.
+ We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade.
+ key.serde, value.serde and timestamp.extractor in configs, it is recommended to use their replaced configure parameter as these configs are deprecated. TimestampExtractor interface was changed. StreamsMetric interface was changed. upgrade.from="0.10.0" set for first upgrade phase
+ (cf. KIP-268).
+ As an alternative, an offline upgrade is also possible.
+ upgrade.from is set to "0.10.0" for new version 0.11.0.3 upgrade.mode upgrade.from added that allows rolling bounce upgrade from version 0.10.0.x Kafka 0.11.0.0 introduces a new message format version as well as wire protocol changes. By following the recommended rolling upgrade plan below, you guarantee no downtime during the upgrade. However, please review the notable changes in 0.11.0.0 before upgrading. @@ -188,11 +248,55 @@
key.serde, value.serde and timestamp.extractor in configs, it is recommended to use their replaced configure parameter as these configs are deprecated. key.serde, value.serde and timestamp.extractor in configs, it is recommended to use their replaced configure parameter as these configs are deprecated. TimestampExtractor interface was changed. StreamsMetric interface was changed. upgrade.from="0.10.0" set for first upgrade phase
+ (cf. KIP-268).
+ As an alternative, an offline upgrade is also possible.
+ upgrade.from is set to "0.10.0" for new version 0.11.0.3 upgrade.mode upgrade.from added that allows rolling bounce upgrade from version 0.10.0.x upgrade.from="0.10.0" set for first upgrade phase
+ (cf. KIP-268).
+ As an alternative, an offline upgrade is also possible.
+ upgrade.from is set to "0.10.0" for new version 0.10.2.2 upgrade.mode upgrade.from added that allows rolling bounce upgrade from version 0.10.0.x retries default value was changed from 0 to 10. The internal Kafka Streams consumer max.poll.interval.ms default value was changed from 300000 to Integer.MAX_VALUE.
@@ -421,6 +554,23 @@ upgrade.from="0.10.0" set for first upgrade phase
+ (cf. KIP-268).
+ As an alternative, an offline upgrade is also possible.
+ upgrade.from is set to "0.10.0" for new version 0.10.1.2 upgrade.mode org.apache.kafka.streams.processor.TimestampExtractor interface. This config is deprecated, use " + DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG + " instead";
+ /** {@code upgrade.from} */
+ public static final String UPGRADE_FROM_CONFIG = "upgrade.from";
+ public static final String UPGRADE_FROM_DOC = "Allows upgrading from version 0.10.0 to version 0.10.1 (or newer) in a backward compatible way. " +
+ "Default is null. Accepted values are \"" + UPGRADE_FROM_0100 + "\" (for upgrading from 0.10.0.x).";
+
/**
* {@code value.serde}
* @deprecated Use {@link #DEFAULT_VALUE_SERDE_CLASS_CONFIG} instead.
@@ -491,6 +501,12 @@ public class StreamsConfig extends AbstractConfig {
10 * 60 * 1000,
Importance.LOW,
STATE_CLEANUP_DELAY_MS_DOC)
+ .define(UPGRADE_FROM_CONFIG,
+ ConfigDef.Type.STRING,
+ null,
+ in(null, UPGRADE_FROM_0100),
+ Importance.LOW,
+ UPGRADE_FROM_DOC)
.define(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG,
Type.LONG,
24 * 60 * 60 * 1000,
@@ -712,6 +728,7 @@ public Map