diff --git a/release.py b/release.py index b008794e733d8..0917184a44b5b 100755 --- a/release.py +++ b/release.py @@ -20,14 +20,31 @@ """ Utility for creating release candidates and promoting release candidates to a final relase. -Usage: release.py +Usage: release.py [subcommand] -The utility is interactive; you will be prompted for basic release information and guided through the process. +release.py stage + + Builds and stages an RC for a release. + + The utility is interactive; you will be prompted for basic release information and guided through the process. + + This utility assumes you already have local a kafka git folder and that you + have added remotes corresponding to both: + (i) the github apache kafka mirror and + (ii) the apache kafka git repo. + +release.py stage-docs [kafka-site-path] + + Builds the documentation and stages it into an instance of the Kafka website repository. + + This is meant to automate the integration between the main Kafka website repository (https://github.com/apache/kafka-site) + and the versioned documentation maintained in the main Kafka repository. This is useful both for local testing and + development of docs (follow the instructions here: https://cwiki.apache.org/confluence/display/KAFKA/Setup+Kafka+Website+on+Local+Apache+Server) + as well as for committers to deploy docs (run this script, then validate, commit, and push to kafka-site). + + With no arguments this script assumes you have the Kafka repository and kafka-site repository checked out side-by-side, but + you can specify a full path to the kafka-site repository if this is not the case. -This utility assumes you already have local a kafka git folder and that you -have added remotes corresponding to both: -(i) the github apache kafka mirror and -(ii) the apache kafka git repo. """ from __future__ import print_function @@ -145,11 +162,113 @@ def get_pref(prefs, name, request_fn): prefs[name] = val return val -# Load saved preferences -prefs = {} -if os.path.exists(PREFS_FILE): - with open(PREFS_FILE, 'r') as prefs_fp: - prefs = json.load(prefs_fp) +def load_prefs(): + """Load saved preferences""" + prefs = {} + if os.path.exists(PREFS_FILE): + with open(PREFS_FILE, 'r') as prefs_fp: + prefs = json.load(prefs_fp) + return prefs + +def save_prefs(prefs): + """Save preferences""" + print("Saving preferences to %s" % PREFS_FILE) + with open(PREFS_FILE, 'w') as prefs_fp: + prefs = json.dump(prefs, prefs_fp) + +def get_jdk(prefs, version): + """ + Get settings for the specified JDK version. + """ + jdk_java_home = get_pref(prefs, 'jdk%d' % version, lambda: raw_input("Enter the path for JAVA_HOME for a JDK%d compiler (blank to use default JAVA_HOME): " % version)) + jdk_env = dict(os.environ) if jdk_java_home.strip() else None + if jdk_env is not None: jdk_env['JAVA_HOME'] = jdk_java_home + if "1.%d.0" % version not in cmd_output("java -version", env=jdk_env): + fail("JDK %s is required" % version) + return jdk_env + +def get_version(repo=REPO_HOME): + """ + Extracts the full version information as a str from gradle.properties + """ + with open(os.path.join(repo, 'gradle.properties')) as fp: + for line in fp: + parts = line.split('=') + if parts[0].strip() != 'version': continue + return parts[1].strip() + fail("Couldn't extract version from gradle.properties") + +def docs_version(version): + """ + Detects the major/minor version and converts it to the format used for docs on the website, e.g. gets 0.10.2.0-SNAPSHOT + from gradle.properties and converts it to 0102 + """ + version_parts = version.strip().split('.') + # 1.0+ will only have 3 version components as opposed to pre-1.0 that had 4 + major_minor = version_parts[0:3] if version_parts[0] == '0' else version_parts[0:2] + return ''.join(major_minor) + +def docs_release_version(version): + """ + Detects the version from gradle.properties and converts it to a release version number that should be valid for the + current release branch. For example, 0.10.2.0-SNAPSHOT would remain 0.10.2.0-SNAPSHOT (because no release has been + made on that branch yet); 0.10.2.1-SNAPSHOT would be converted to 0.10.2.0 because 0.10.2.1 is still in development + but 0.10.2.0 should have already been released. Regular version numbers (e.g. as encountered on a release branch) + will remain the same. + """ + version_parts = version.strip().split('.') + if '-SNAPSHOT' in version_parts[-1]: + bugfix = int(version_parts[-1].split('-')[0]) + if bugfix > 0: + version_parts[-1] = str(bugfix - 1) + return '.'.join(version_parts) + +def command_stage_docs(): + kafka_site_repo_path = sys.argv[2] if len(sys.argv) > 2 else os.path.join(REPO_HOME, '..', 'kafka-site') + if not os.path.exists(kafka_site_repo_path) or not os.path.exists(os.path.join(kafka_site_repo_path, 'powered-by.html')): + sys.exit("%s doesn't exist or does not appear to be the kafka-site repository" % kafka_site_repo_path) + + prefs = load_prefs() + jdk8_env = get_jdk(prefs, 8) + save_prefs(prefs) + + version = get_version() + # We explicitly override the version of the project that we normally get from gradle.properties since we want to be + # able to run this from a release branch where we made some updates, but the build would show an incorrect SNAPSHOT + # version due to already having bumped the bugfix version number. + gradle_version_override = docs_release_version(version) + + cmd("Building docs", "./gradlew -Pversion=%s clean releaseTarGzAll aggregatedJavadoc" % gradle_version_override, cwd=REPO_HOME, env=jdk8_env) + + docs_tar = os.path.join(REPO_HOME, 'core', 'build', 'distributions', 'kafka_2.11-%s-site-docs.tgz' % gradle_version_override) + + versioned_docs_path = os.path.join(kafka_site_repo_path, docs_version(version)) + if not os.path.exists(versioned_docs_path): + os.mkdir(versioned_docs_path, 0755) + + # The contents of the docs jar are site-docs/. We need to get rid of the site-docs prefix and dump everything + # inside it into the docs version subdirectory in the kafka-site repo + cmd('Extracting site-docs', 'tar xf %s --strip-components 1' % docs_tar, cwd=versioned_docs_path) + + javadocs_src_dir = os.path.join(REPO_HOME, 'build', 'docs', 'javadoc') + + cmd('Copying javadocs', 'cp -R %s %s' % (javadocs_src_dir, versioned_docs_path)) + + sys.exit(0) + + +# Dispatch to subcommand +subcommand = sys.argv[1] if len(sys.argv) > 1 else None +if subcommand == 'stage-docs': + command_stage_docs() +elif not (subcommand is None or subcommand == 'stage'): + fail("Unknown subcommand: %s" % subcommand) +# else -> default subcommand stage + + +## Default 'stage' subcommand implementation isn't isolated to its own function yet for historical reasons + +prefs = load_prefs() if not user_ok("""Requirements: 1. Updated docs to reference the new release version where appropriate. @@ -221,7 +340,7 @@ def get_pref(prefs, name, request_fn): rc = raw_input("Release candidate number: ") dev_branch = '.'.join(release_version_parts[:2]) -docs_version = ''.join(release_version_parts[:2]) +docs_release_version = docs_version(release_version[:2]) # Validate that the release doesn't already exist and that the cmd("Fetching tags from upstream", 'git fetch --tags %s' % PUSH_REMOTE_NAME) @@ -244,18 +363,8 @@ def get_pref(prefs, name, request_fn): # Prereq checks apache_id = get_pref(prefs, 'apache_id', lambda: raw_input("Enter your apache username: ")) - -jdk7_java_home = get_pref(prefs, 'jdk7', lambda: raw_input("Enter the path for JAVA_HOME for a JDK7 compiler (blank to use default JAVA_HOME): ")) -jdk7_env = dict(os.environ) if jdk7_java_home.strip() else None -if jdk7_env is not None: jdk7_env['JAVA_HOME'] = jdk7_java_home -if "1.7.0" not in cmd_output("java -version", env=jdk7_env): - fail("You must be able to build artifacts with JDK7 for Scala 2.10 and 2.11 artifacts") - -jdk8_java_home = get_pref(prefs, 'jdk8', lambda: raw_input("Enter the path for JAVA_HOME for a JDK8 compiler (blank to use default JAVA_HOME): ")) -jdk8_env = dict(os.environ) if jdk8_java_home.strip() else None -if jdk8_env is not None: jdk8_env['JAVA_HOME'] = jdk8_java_home -if "1.8.0" not in cmd_output("java -version", env=jdk8_env): - fail("You must be able to build artifacts with JDK8 for Scala 2.12 artifacts") +jdk7_env = get_jdk(prefs, 7) +jdk8_env = get_jdk(prefs, 8) def select_gpg_key(): @@ -275,10 +384,7 @@ def select_gpg_key(): gpg_test_tempfile.write("abcdefg") cmd("Testing GPG key & passphrase", ["gpg", "--batch", "--pinentry-mode", "loopback", "--passphrase-fd", "0", "-u", key_name, "--armor", "--output", gpg_test_tempfile.name + ".asc", "--detach-sig", gpg_test_tempfile.name], stdin=gpg_passphrase) -# Save preferences -print("Saving preferences to %s" % PREFS_FILE) -with open(PREFS_FILE, 'w') as prefs_fp: - prefs = json.dump(prefs, prefs_fp) +save_prefs(prefs) # Generate RC try: @@ -400,7 +506,7 @@ def select_gpg_key(): 'rc_tag': rc_tag, 'rc_githash': rc_githash, 'dev_branch': dev_branch, - 'docs_version': docs_version, + 'docs_version': docs_release_version, 'apache_id': apache_id, }