Skip to content

KAFKA-10712; Update release scripts to Python3 #11538

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dajac merged 1 commit into from
Nov 29, 2021
Merged

KAFKA-10712; Update release scripts to Python3 #11538

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 23 additions & 24 deletions release.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@

"""

from __future__ import print_function

import datetime
from getpass import getpass
import json
Expand Down Expand Up @@ -99,25 +97,25 @@ def print_output(output):
print(">", line)

def cmd(action, cmd_arg, *args, **kwargs):
if isinstance(cmd_arg, basestring) and not kwargs.get("shell", False):
if isinstance(cmd_arg, str) and not kwargs.get("shell", False):
cmd_arg = cmd_arg.split()
allow_failure = kwargs.pop("allow_failure", False)
num_retries = kwargs.pop("num_retries", 0)

stdin_log = ""
if "stdin" in kwargs and isinstance(kwargs["stdin"], basestring):
if "stdin" in kwargs and isinstance(kwargs["stdin"], str):
stdin_log = "--> " + kwargs["stdin"]
stdin = tempfile.TemporaryFile()
stdin.write(kwargs["stdin"])
stdin.write(kwargs["stdin"].encode('utf-8'))
stdin.seek(0)
kwargs["stdin"] = stdin

print(action, cmd_arg, stdin_log)
try:
output = subprocess.check_output(cmd_arg, *args, stderr=subprocess.STDOUT, **kwargs)
print_output(output)
print_output(output.decode('utf-8'))
except subprocess.CalledProcessError as e:
print_output(e.output)
print_output(e.output.decode('utf-8'))

if num_retries > 0:
kwargs['num_retries'] = num_retries - 1
Expand All @@ -137,9 +135,9 @@ def cmd(action, cmd_arg, *args, **kwargs):


def cmd_output(cmd, *args, **kwargs):
if isinstance(cmd, basestring):
if isinstance(cmd, str):
cmd = cmd.split()
return subprocess.check_output(cmd, *args, stderr=subprocess.STDOUT, **kwargs)
return subprocess.check_output(cmd, *args, stderr=subprocess.STDOUT, **kwargs).decode('utf-8')

def replace(path, pattern, replacement):
updated = []
Expand All @@ -162,7 +160,7 @@ def regexReplace(path, pattern, replacement):
f.write(line)

def user_ok(msg):
ok = raw_input(msg)
ok = input(msg)
return ok.strip().lower() == 'y'

def sftp_mkdir(dir):
Expand Down Expand Up @@ -211,13 +209,14 @@ def get_jdk(prefs, version):
"""
Get settings for the specified JDK version.
"""
jdk_java_home = get_pref(prefs, 'jdk%d' % version, lambda: raw_input("Enter the path for JAVA_HOME for a JDK%d compiler (blank to use default JAVA_HOME): " % version))
jdk_java_home = get_pref(prefs, 'jdk%d' % version, lambda: input("Enter the path for JAVA_HOME for a JDK%d compiler (blank to use default JAVA_HOME): " % version))
jdk_env = dict(os.environ) if jdk_java_home.strip() else None
if jdk_env is not None: jdk_env['JAVA_HOME'] = jdk_java_home
java_version = cmd_output("%s/bin/java -version" % jdk_java_home, env=jdk_env)
if version == 8 and "1.8.0" not in java_version:
fail("JDK 8 is required")
elif "%d.0" % version not in java_version or '"%d"' % version not in java_version:
if version == 8:
if "1.8.0" not in java_version:
fail("JDK 8 is required")
elif "%d.0" % version not in java_version and '"%d"' % version not in java_version:

@dajac dajac Nov 25, 2021

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We had a bug here. The script should fail if both conditions are not met.

fail("JDK %s is required" % version)
return jdk_env

Expand Down Expand Up @@ -278,7 +277,7 @@ def command_stage_docs():

versioned_docs_path = os.path.join(kafka_site_repo_path, docs_version(version))
if not os.path.exists(versioned_docs_path):
os.mkdir(versioned_docs_path, 0755)
os.mkdir(versioned_docs_path, 755)

# The contents of the docs jar are site-docs/<docs dir>. We need to get rid of the site-docs prefix and dump everything
# inside it into the docs version subdirectory in the kafka-site repo
Expand Down Expand Up @@ -316,16 +315,16 @@ def command_release_announcement_email():
release_tags = sorted([t for t in tags if re.match(release_tag_pattern, t)])
release_version_num = release_tags[-1]
if not user_ok("""Is the current release %s ? (y/n): """ % release_version_num):
release_version_num = raw_input('What is the current release version:')
release_version_num = input('What is the current release version:')
validate_release_num(release_version_num)
previous_release_version_num = release_tags[-2]
if not user_ok("""Is the previous release %s ? (y/n): """ % previous_release_version_num):
previous_release_version_num = raw_input('What is the previous release version:')
previous_release_version_num = input('What is the previous release version:')
validate_release_num(previous_release_version_num)
if release_version_num < previous_release_version_num :
fail("Current release version number can't be less than previous release version number")
number_of_contributors = int(subprocess.check_output('git shortlog -sn --no-merges %s..%s | wc -l' % (previous_release_version_num, release_version_num) , shell=True))
contributors = subprocess.check_output("git shortlog -sn --no-merges %s..%s | cut -f2 | sort --ignore-case" % (previous_release_version_num, release_version_num), shell=True)
number_of_contributors = int(subprocess.check_output('git shortlog -sn --no-merges %s..%s | wc -l' % (previous_release_version_num, release_version_num) , shell=True).decode('utf-8'))
contributors = subprocess.check_output("git shortlog -sn --no-merges %s..%s | cut -f2 | sort --ignore-case" % (previous_release_version_num, release_version_num), shell=True).decode('utf-8')
release_announcement_data = {
'number_of_contributors': number_of_contributors,
'contributors': ', '.join(str(x) for x in filter(None, contributors.split('\n'))),
Expand Down Expand Up @@ -488,10 +487,10 @@ def command_release_announcement_email():
cmd("Verifying that you have no unstaged git changes", 'git diff --exit-code --quiet')
cmd("Verifying that you have no staged git changes", 'git diff --cached --exit-code --quiet')

release_version = raw_input("Release version (without any RC info, e.g. 1.0.0): ")
release_version = input("Release version (without any RC info, e.g. 1.0.0): ")
release_version_parts = get_release_version_parts(release_version)

rc = raw_input("Release candidate number: ")
rc = input("Release candidate number: ")

dev_branch = '.'.join(release_version_parts[:2])
docs_release_version = docs_version(release_version)
Expand All @@ -515,7 +514,7 @@ def command_release_announcement_email():
sys.exit(0)

# Prereq checks
apache_id = get_pref(prefs, 'apache_id', lambda: raw_input("Enter your apache username: "))
apache_id = get_pref(prefs, 'apache_id', lambda: input("Enter your apache username: "))

jdk8_env = get_jdk(prefs, 8)
jdk17_env = get_jdk(prefs, 17)
Expand All @@ -524,7 +523,7 @@ def select_gpg_key():
print("Here are the available GPG keys:")
available_keys = cmd_output("gpg --list-secret-keys")
print(available_keys)
key_name = raw_input("Which user name (enter the user name without email address): ")
key_name = input("Which user name (enter the user name without email address): ")
if key_name not in available_keys:
fail("Couldn't find the requested key.")
return key_name
Expand All @@ -534,7 +533,7 @@ def select_gpg_key():
gpg_passphrase = get_pref(prefs, 'gpg-pass', lambda: getpass("Passphrase for this GPG key: "))
# Do a quick validation so we can fail fast if the password is incorrect
with tempfile.NamedTemporaryFile() as gpg_test_tempfile:
gpg_test_tempfile.write("abcdefg")
gpg_test_tempfile.write("abcdefg".encode('utf-8'))
cmd("Testing GPG key & passphrase", ["gpg", "--batch", "--pinentry-mode", "loopback", "--passphrase-fd", "0", "-u", key_name, "--armor", "--output", gpg_test_tempfile.name + ".asc", "--detach-sig", gpg_test_tempfile.name], stdin=gpg_passphrase)

save_prefs(prefs)
Expand Down
29 changes: 15 additions & 14 deletions release_notes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import itertools, sys

if len(sys.argv) < 2:
print >>sys.stderr, "Usage: release_notes.py <version>"
print("Usage: release_notes.py <version>", file=sys.stderr)
sys.exit(1)

version = sys.argv[1]
Expand Down Expand Up @@ -58,7 +58,7 @@ def issue_link(issue):
apache = JIRA(JIRA_BASE_URL)
issues = get_issues(apache, 'project=KAFKA and fixVersion=%s' % version)
if not issues:
print >>sys.stderr, "Didn't find any issues for the target fix version"
print("Didn't find any issues for the target fix version", file=sys.stderr)
sys.exit(1)

# Some resolutions, including a lack of resolution, indicate that the bug hasn't actually been addressed and we shouldn't even be able to create a release until they are fixed
Expand All @@ -78,11 +78,11 @@ def issue_link(issue):
]
unresolved_issues = [issue for issue in issues if issue.fields.resolution in UNRESOLVED_RESOLUTIONS or issue.fields.resolution.name in UNRESOLVED_RESOLUTIONS]
if unresolved_issues:
print >>sys.stderr, "The release is not completed since unresolved issues or improperly resolved issues were found still tagged with this release as the fix version:"
print("The release is not completed since unresolved issues or improperly resolved issues were found still tagged with this release as the fix version:", file=sys.stderr)
for issue in unresolved_issues:
print >>sys.stderr, "Unresolved issue: %15s %20s %s" % (issue.key, issue.fields.resolution, issue_link(issue))
print >>sys.stderr
print >>sys.stderr, "Note that for some resolutions, you should simply remove the fix version as they have not been truly fixed in this release."
print("Unresolved issue: %15s %20s %s" % (issue.key, issue.fields.resolution, issue_link(issue)), file=sys.stderr)
print("", file=sys.stderr)
print("Note that for some resolutions, you should simply remove the fix version as they have not been truly fixed in this release.", file=sys.stderr)
sys.exit(1)

# Get list of (issue type, [issues]) sorted by the issue ID type, with each subset of issues sorted by their key so they
Expand All @@ -93,11 +93,12 @@ def issue_type_key(issue):
return -2
if issue.fields.issuetype.name == 'Improvement':
return -1
return issue.fields.issuetype.id
return int(issue.fields.issuetype.id)

by_group = [(k,sorted(g, key=lambda issue: issue.id)) for k,g in itertools.groupby(sorted(issues, key=issue_type_key), lambda issue: issue.fields.issuetype.name)]

print "<h1>Release Notes - Kafka - Version %s</h1>" % version
print """<p>Below is a summary of the JIRA issues addressed in the %(version)s release of Kafka. For full documentation of the
print("<h1>Release Notes - Kafka - Version %s</h1>" % version)
print("""<p>Below is a summary of the JIRA issues addressed in the %(version)s release of Kafka. For full documentation of the
release, a guide to get started, and information about the project, see the <a href="https://kafka.apache.org/">Kafka
project site</a>.</p>

Expand All @@ -107,10 +108,10 @@ def issue_type_key(issue):
changes, performance changes, and any other changes that might impact your production deployment of Kafka.</p>

<p>The documentation for the most recent release can be found at
<a href="https://kafka.apache.org/documentation.html">https://kafka.apache.org/documentation.html</a>.</p>""" % { 'version': version, 'minor': minor_version_dotless }
<a href="https://kafka.apache.org/documentation.html">https://kafka.apache.org/documentation.html</a>.</p>""" % { 'version': version, 'minor': minor_version_dotless })
for itype, issues in by_group:
print "<h2>%s</h2>" % itype
print "<ul>"
print("<h2>%s</h2>" % itype)
print("<ul>")
for issue in issues:
print '<li>[<a href="%(link)s">%(key)s</a>] - %(summary)s</li>' % {'key': issue.key, 'link': issue_link(issue), 'summary': issue.fields.summary}
print "</ul>"
print('<li>[<a href="%(link)s">%(key)s</a>] - %(summary)s</li>' % {'key': issue.key, 'link': issue_link(issue), 'summary': issue.fields.summary})
print("</ul>")