MINOR: Extend release.py with a subcommand for staging docs into the kafka-site repo

release.py

@@ -20,14 +20,31 @@
 """
 Utility for creating release candidates and promoting release candidates to a final relase.
 
-Usage: release.py
+Usage: release.py [subcommand]
 
-The utility is interactive; you will be prompted for basic release information and guided through the process.
+release.py stage
+
+  Builds and stages an RC for a release.
+
+  The utility is interactive; you will be prompted for basic release information and guided through the process.
+
+  This utility assumes you already have local a kafka git folder and that you
+  have added remotes corresponding to both:
+  (i) the github apache kafka mirror and
+  (ii) the apache kafka git repo.
+
+release.py stage-docs [kafka-site-path]
+
+  Builds the documentation and stages it into an instance of the Kafka website repository.
+
+  This is meant to automate the integration between the main Kafka website repository (https://github.com/apache/kafka-site)
+  and the versioned documentation maintained in the main Kafka repository. This is useful both for local testing and
+  development of docs (follow the instructions here: https://cwiki.apache.org/confluence/display/KAFKA/Setup+Kafka+Website+on+Local+Apache+Server)
+  as well as for committers to deploy docs (run this script, then validate, commit, and push to kafka-site).
+
+  With no arguments this script assumes you have the Kafka repository and kafka-site repository checked out side-by-side, but
+  you can specify a full path to the kafka-site repository if this is not the case.
 
-This utility assumes you already have local a kafka git folder and that you
-have added remotes corresponding to both:
-(i) the github apache kafka mirror and
-(ii) the apache kafka git repo.
 """
 
 from __future__ import print_function
@@ -145,11 +162,113 @@ def get_pref(prefs, name, request_fn):
         prefs[name] = val
     return val
 
-# Load saved preferences
-prefs = {}
-if os.path.exists(PREFS_FILE):
-    with open(PREFS_FILE, 'r') as prefs_fp:
-        prefs = json.load(prefs_fp)
+def load_prefs():
+    """Load saved preferences"""
+    prefs = {}
+    if os.path.exists(PREFS_FILE):
+        with open(PREFS_FILE, 'r') as prefs_fp:
+            prefs = json.load(prefs_fp)
+    return prefs
+
+def save_prefs(prefs):
+    """Save preferences"""
+    print("Saving preferences to %s" % PREFS_FILE)
+    with open(PREFS_FILE, 'w') as prefs_fp:
+        prefs = json.dump(prefs, prefs_fp)
+
+def get_jdk(prefs, version):
+    """
+    Get settings for the specified JDK version.
+    """
+    jdk_java_home = get_pref(prefs, 'jdk%d' % version, lambda: raw_input("Enter the path for JAVA_HOME for a JDK%d compiler (blank to use default JAVA_HOME): " % version))
+    jdk_env = dict(os.environ) if jdk_java_home.strip() else None
+    if jdk_env is not None: jdk_env['JAVA_HOME'] = jdk_java_home
+    if "1.%d.0" % version not in cmd_output("java -version", env=jdk_env):
+        fail("JDK %s is required" % version)
+    return jdk_env
+
+def get_version(repo=REPO_HOME):
+    """
+    Extracts the full version information as a str from gradle.properties
+    """
+    with open(os.path.join(repo, 'gradle.properties')) as fp:
+        for line in fp:
+            parts = line.split('=')
+            if parts[0].strip() != 'version': continue
+            return parts[1].strip()
+    fail("Couldn't extract version from gradle.properties")
+
+def docs_version(version):
+    """
+    Detects the major/minor version and converts it to the format used for docs on the website, e.g. gets 0.10.2.0-SNAPSHOT
+    from gradle.properties and converts it to 0102
+    """
+    version_parts = version.strip().split('.')
+    # 1.0+ will only have 3 version components as opposed to pre-1.0 that had 4
+    major_minor = version_parts[0:3] if version_parts[0] == '0' else version_parts[0:2]
+    return ''.join(major_minor)
+
+def docs_release_version(version):
+    """
+    Detects the version from gradle.properties and converts it to a release version number that should be valid for the
+    current release branch. For example, 0.10.2.0-SNAPSHOT would remain 0.10.2.0-SNAPSHOT (because no release has been
+    made on that branch yet); 0.10.2.1-SNAPSHOT would be converted to 0.10.2.0 because 0.10.2.1 is still in development
+    but 0.10.2.0 should have already been released. Regular version numbers (e.g. as encountered on a release branch)
+    will remain the same.
+    """
+    version_parts = version.strip().split('.')
+    if '-SNAPSHOT' in version_parts[-1]:
+        bugfix = int(version_parts[-1].split('-')[0])
+        if bugfix > 0:
+            version_parts[-1] = str(bugfix - 1)
+    return '.'.join(version_parts)
+
+def command_stage_docs():
+    kafka_site_repo_path = sys.argv[2] if len(sys.argv) > 2 else os.path.join(REPO_HOME, '..', 'kafka-site')
+    if not os.path.exists(kafka_site_repo_path) or not os.path.exists(os.path.join(kafka_site_repo_path, 'powered-by.html')):
+        sys.exit("%s doesn't exist or does not appear to be the kafka-site repository" % kafka_site_repo_path)
+
+    prefs = load_prefs()
+    jdk8_env = get_jdk(prefs, 8)
+    save_prefs(prefs)
+
+    version = get_version()
+    # We explicitly override the version of the project that we normally get from gradle.properties since we want to be
+    # able to run this from a release branch where we made some updates, but the build would show an incorrect SNAPSHOT
+    # version due to already having bumped the bugfix version number.
+    gradle_version_override = docs_release_version(version)
+
+    cmd("Building docs", "./gradlew -Pversion=%s clean releaseTarGzAll aggregatedJavadoc" % gradle_version_override, cwd=REPO_HOME, env=jdk8_env)
+
+    docs_tar = os.path.join(REPO_HOME, 'core', 'build', 'distributions', 'kafka_2.11-%s-site-docs.tgz' % gradle_version_override)
+
+    versioned_docs_path = os.path.join(kafka_site_repo_path, docs_version(version))
+    if not os.path.exists(versioned_docs_path):
+        os.mkdir(versioned_docs_path, 0755)
+
+    # The contents of the docs jar are site-docs/<docs dir>. We need to get rid of the site-docs prefix and dump everything
+    # inside it into the docs version subdirectory in the kafka-site repo
+    cmd('Extracting site-docs', 'tar xf %s --strip-components 1' % docs_tar, cwd=versioned_docs_path)
+
+    javadocs_src_dir = os.path.join(REPO_HOME, 'build', 'docs', 'javadoc')
+
+    cmd('Copying javadocs', 'cp -R %s %s' % (javadocs_src_dir, versioned_docs_path))
+
+    sys.exit(0)
+
+
+# Dispatch to subcommand
+subcommand = sys.argv[1] if len(sys.argv) > 1 else None
+if subcommand == 'stage-docs':
+    command_stage_docs()
+elif not (subcommand is None or subcommand == 'stage'):
+    fail("Unknown subcommand: %s" % subcommand)
+# else -> default subcommand stage
+
+
+## Default 'stage' subcommand implementation isn't isolated to its own function yet for historical reasons
+
+prefs = load_prefs()
 
 if not user_ok("""Requirements:
 1. Updated docs to reference the new release version where appropriate.
@@ -221,7 +340,7 @@ def get_pref(prefs, name, request_fn):
 rc = raw_input("Release candidate number: ")
 
 dev_branch = '.'.join(release_version_parts[:2])
-docs_version = ''.join(release_version_parts[:2])
+docs_release_version = docs_version(release_version[:2])
 
 # Validate that the release doesn't already exist and that the
 cmd("Fetching tags from upstream", 'git fetch --tags %s' % PUSH_REMOTE_NAME)
@@ -244,18 +363,8 @@ def get_pref(prefs, name, request_fn):
 # Prereq checks
 apache_id = get_pref(prefs, 'apache_id', lambda: raw_input("Enter your apache username: "))
 
-
-jdk7_java_home = get_pref(prefs, 'jdk7', lambda: raw_input("Enter the path for JAVA_HOME for a JDK7 compiler (blank to use default JAVA_HOME): "))
-jdk7_env = dict(os.environ) if jdk7_java_home.strip() else None
-if jdk7_env is not None: jdk7_env['JAVA_HOME'] = jdk7_java_home
-if "1.7.0" not in cmd_output("java -version", env=jdk7_env):
-    fail("You must be able to build artifacts with JDK7 for Scala 2.10 and 2.11 artifacts")
-
-jdk8_java_home = get_pref(prefs, 'jdk8', lambda: raw_input("Enter the path for JAVA_HOME for a JDK8 compiler (blank to use default JAVA_HOME): "))
-jdk8_env = dict(os.environ) if jdk8_java_home.strip() else None
-if jdk8_env is not None: jdk8_env['JAVA_HOME'] = jdk8_java_home
-if "1.8.0" not in cmd_output("java -version", env=jdk8_env):
-    fail("You must be able to build artifacts with JDK8 for Scala 2.12 artifacts")
+jdk7_env = get_jdk(prefs, 7)
+jdk8_env = get_jdk(prefs, 8)
 
 
 def select_gpg_key():
@@ -275,10 +384,7 @@ def select_gpg_key():
     gpg_test_tempfile.write("abcdefg")
     cmd("Testing GPG key & passphrase", ["gpg", "--batch", "--pinentry-mode", "loopback", "--passphrase-fd", "0", "-u", key_name, "--armor", "--output", gpg_test_tempfile.name + ".asc", "--detach-sig", gpg_test_tempfile.name], stdin=gpg_passphrase)
 
-# Save preferences
-print("Saving preferences to %s" % PREFS_FILE)
-with open(PREFS_FILE, 'w') as prefs_fp:
-    prefs = json.dump(prefs, prefs_fp)
+save_prefs(prefs)
 
 # Generate RC
 try:
@@ -400,7 +506,7 @@ def select_gpg_key():
                                'rc_tag': rc_tag,
                                'rc_githash': rc_githash,
                                'dev_branch': dev_branch,
-                               'docs_version': docs_version,
+                               'docs_version': docs_release_version,
                                'apache_id': apache_id,
                                }