Skip to content

[SPARK-34551][INFRA] Fix credit related scripts to recover, drop Python 2 and work with Python 3 #31660

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
HyukjinKwon wants to merge 1 commit into from
Closed

[SPARK-34551][INFRA] Fix credit related scripts to recover, drop Python 2 and work with Python 3 #31660

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions dev/create-release/generate-contributors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import re
import sys

from releaseutils import tag_exists, raw_input, get_commits, yesOrNoPrompt, get_date, \
from releaseutils import tag_exists, get_commits, yesOrNoPrompt, get_date, \
is_valid_author, capitalize_author, JIRA, find_components, translate_issue_type, \
translate_component, CORE_COMPONENT, contributors_file_name, nice_join

Expand All @@ -33,10 +33,10 @@

# If the release tags are not provided, prompt the user to provide them
while not tag_exists(RELEASE_TAG):
RELEASE_TAG = raw_input("Please provide a valid release tag: ")
RELEASE_TAG = input("Please provide a valid release tag: ")
while not tag_exists(PREVIOUS_RELEASE_TAG):
print("Please specify the previous release tag.")
PREVIOUS_RELEASE_TAG = raw_input(
PREVIOUS_RELEASE_TAG = input(
"For instance, if you are releasing v1.2.0, you should specify v1.1.0: ")

# Gather commits found in the new tag but not in the old tag.
Expand Down Expand Up @@ -236,7 +236,7 @@ def populate(issue_type, components):
# e.g. * Andrew Or -- Bug fixes in Windows, Core, and Web UI; improvements in Core
# e.g. * Tathagata Das -- Bug fixes and new features in Streaming
contributors_file = open(contributors_file_name, "w")
authors = author_info.keys()
authors = list(author_info.keys())
authors.sort()
for author in authors:
contribution = ""
Expand Down
15 changes: 3 additions & 12 deletions dev/create-release/releaseutils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,6 @@
print("Install using 'sudo pip install PyGithub'")
sys.exit(-1)

try:
import unidecode
except ImportError:
print("This tool requires the unidecode library to decode obscure github usernames")
print("Install using 'sudo pip install unidecode'")
sys.exit(-1)


# Contributors list file name
contributors_file_name = "contributors.txt"
Expand All @@ -64,11 +57,11 @@ def yesOrNoPrompt(msg):

# Utility functions run git commands (written with Git 1.8.5)
def run_cmd(cmd):
return Popen(cmd, stdout=PIPE).communicate()[0]
return Popen(cmd, stdout=PIPE).communicate()[0].decode("utf8")


def run_cmd_error(cmd):
return Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()[1]
return Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()[1].decode("utf8")


def get_date(commit_hash):
Expand Down Expand Up @@ -149,9 +142,7 @@ def get_commits(tag):
# username so we can translate it properly later
if not is_valid_author(author):
author = github_username
# Guard against special characters
author = str(author)
author = unidecode.unidecode(author).strip()
author = author.strip()
commit = Commit(_hash, author, title, pr_number)
commits.append(commit)
return commits
Expand Down
24 changes: 5 additions & 19 deletions dev/create-release/translate-contributors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,7 @@
import sys

from releaseutils import JIRA, JIRAError, get_jira_name, Github, get_github_name, \
contributors_file_name, is_valid_author, raw_input, capitalize_author, yesOrNoPrompt

try:
import unidecode
except ImportError:
print("This tool requires the unidecode library to decode obscure github usernames")
Copy link
Member Author

@HyukjinKwon HyukjinKwon Feb 26, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to clarify, I think now most of release docs, notes, and everywhere else can handle UFT-8 properly, and we don't have to force and mangle other people's name into ASCII. I believe this was initially introduced due to unicode and Python 2's string handling. If there's an issue, we believe we should try to fix it instead of manipulating author's names.

print("Install using 'sudo pip install unidecode'")
sys.exit(-1)
contributors_file_name, is_valid_author, capitalize_author, yesOrNoPrompt

# You must set the following before use!
JIRA_API_BASE = os.environ.get("JIRA_API_BASE", "https://issues.apache.org/jira")
Expand Down Expand Up @@ -139,15 +132,8 @@ def generate_candidates(author, issues):
(NOT_FOUND, "No full name found for %s assignee %s" % (issue, user_name)))
else:
candidates.append((NOT_FOUND, "No assignee found for %s" % issue))
# Guard against special characters in candidate names
# Note that the candidate name may already be in unicode (JIRA returns this)
for i, (candidate, source) in enumerate(candidates):
try:
candidate = unicode(candidate, "UTF-8") # noqa: F821
except TypeError:
# already in unicode
pass
candidate = unidecode.unidecode(candidate).strip()
candidate = candidate.strip()
candidates[i] = (candidate, source)
return candidates

Expand Down Expand Up @@ -209,13 +195,13 @@ def generate_candidates(author, issues):
if INTERACTIVE_MODE:
print(" [%d] %s - Raw GitHub username" % (raw_index, author))
print(" [%d] Custom" % custom_index)
response = raw_input(" Your choice: ")
response = input(" Your choice: ")
last_index = custom_index
while not response.isdigit() or int(response) > last_index:
response = raw_input(" Please enter an integer between 0 and %d: " % last_index)
response = input(" Please enter an integer between 0 and %d: " % last_index)
response = int(response)
if response == custom_index:
new_author = raw_input(" Please type a custom name for this author: ")
new_author = input(" Please type a custom name for this author: ")
elif response != raw_index:
new_author = candidate_names[response]
# In non-interactive mode, just pick the first candidate
Expand Down
1 change: 0 additions & 1 deletion dev/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
flake8==3.5.0
jira==1.0.3
PyGithub==1.26.0
Unidecode==0.04.19
sphinx
pydata_sphinx_theme
ipython
Expand Down