From 35a752a54017ac2242a7d7bd3b2367b274a94a4f Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 2 Dec 2019 12:58:21 -0500 Subject: [PATCH 1/3] tools: splitting flags and proto scripts into separae files Signed-off-by: Alyssa Wilk --- GOVERNANCE.md | 3 + .../deprecate_features/deprecate_features.py | 49 ++------------ tools/deprecate_features/requirements.txt | 3 +- tools/deprecate_protos/deprecate_protos.py | 67 +++++++++++++++++++ tools/deprecate_protos/deprecate_protos.sh | 7 ++ tools/deprecate_protos/requirements.txt | 1 + 6 files changed, 83 insertions(+), 47 deletions(-) create mode 100644 tools/deprecate_protos/deprecate_protos.py create mode 100644 tools/deprecate_protos/deprecate_protos.sh create mode 100644 tools/deprecate_protos/requirements.txt diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 6b7fb0e16a581..98e50f317a775 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -104,6 +104,9 @@ or you can subscribe to the iCal feed [here](https://app.opsgenie.com/webcal/get * Run the deprecate_versions.py script (e.g. `sh tools/deprecate_version/deprecate_version.sh`) to file tracking issues for code which can be removed. * Run the deprecate_features.py script (e.g. `sh tools/deprecate_features/deprecate_features.sh`) + to flip any false runtime guards to true. Submit the resultant PR and send + an email to envoy-announce. +* Run the deprecate_protos.py script (e.g. `tools/deprecate_protos/deprecate_protos.sh`) to make the last release's deprecated features fatal-by-default. Submit the resultant PR and send an email to envoy-announce. diff --git a/tools/deprecate_features/deprecate_features.py b/tools/deprecate_features/deprecate_features.py index 203d4d2a99dbf..7e12ffa9d2307 100644 --- a/tools/deprecate_features/deprecate_features.py +++ b/tools/deprecate_features/deprecate_features.py @@ -1,4 +1,4 @@ -# A simple script to snag deprecated proto fields and add them to runtime_features.h +# A simple script to flip runtime flags true. from __future__ import print_function import re @@ -7,45 +7,6 @@ from six.moves import input -# Sorts out the list of deprecated proto fields which should be disallowed and returns a tuple of -# email and code changes. -def deprecate_proto(): - grep_output = subprocess.check_output('grep -r "deprecated = true" api/*', shell=True) - - filenames_and_fields = set() - - # Compile the set of deprecated fields and the files they're in, deduping via set. - deprecated_regex = re.compile(r'.*\/([^\/]*.proto):[^=]* ([^= ]+) =.*') - for line in grep_output.splitlines(): - match = deprecated_regex.match(line) - if match: - filenames_and_fields.add(tuple([match.group(1), match.group(2)])) - else: - print('no match in ' + line + ' please address manually!') - - # Now discard any deprecated features already listed in runtime_features - exiting_deprecated_regex = re.compile(r'.*"envoy.deprecated_features.(.*):(.*)",.*') - with open('source/common/runtime/runtime_features.cc', 'r') as features: - for line in features.readlines(): - match = exiting_deprecated_regex.match(line) - if match: - filenames_and_fields.discard(tuple([match.group(1), match.group(2)])) - - # Finally sort out the code to add to runtime_features.cc and a canned email for envoy-announce. - code_snippets = [] - email_snippets = [] - for (filename, field) in filenames_and_fields: - code_snippets.append(' "envoy.deprecated_features.' + filename + ':' + field + '",\n') - email_snippets.append(field + ' from ' + filename + '\n') - code = ''.join(code_snippets) - email = '' - if email_snippets: - email = ('\nThe following deprecated configuration fields will be disallowed by default:\n' + - ''.join(email_snippets)) - - return email, code - - # Sorts out the list of features which should be default enabled and returns a tuple of # email and code changes. def flip_runtime_features(): @@ -56,7 +17,8 @@ def flip_runtime_features(): # Compile the set of features to flip, deduping via set. deprecated_regex = re.compile(r'.*"(envoy.reloadable_features\.[^"]+)".*') - for line in grep_output.splitlines(): + for byte_line in grep_output.splitlines(): + line = byte_line.decode('utf-8') match = deprecated_regex.match(line) if match: features_to_flip.add(match.group(1)) @@ -82,10 +44,9 @@ def flip_runtime_features(): # Gather code and suggested email changes. runtime_email, runtime_features_code = flip_runtime_features() -deprecate_email, deprecate_code = deprecate_proto() email = ('The Envoy maintainer team is cutting the next Envoy release. In the new release ' + - runtime_email + deprecate_email) + runtime_email) print('\n\nSuggested envoy-announce email: \n') print(email) @@ -96,8 +57,6 @@ def flip_runtime_features(): for line in fileinput.FileInput('source/common/runtime/runtime_features.cc', inplace=1): if 'envoy.reloadable_features.test_feature_true' in line: line = line.replace(line, line + runtime_features_code) - if 'envoy.deprecated_features.deprecated.proto:is_deprecated_fatal' in line: - line = line.replace(line, line + deprecate_code) print(line, end='') print('\nChanges applied. Please send the email above to envoy-announce.\n') diff --git a/tools/deprecate_features/requirements.txt b/tools/deprecate_features/requirements.txt index dc2a917a768ee..b1013c19345c4 100644 --- a/tools/deprecate_features/requirements.txt +++ b/tools/deprecate_features/requirements.txt @@ -1,2 +1 @@ -GitPython==3.0.0 -PyGithub==1.43.8 +six==1.12.0 diff --git a/tools/deprecate_protos/deprecate_protos.py b/tools/deprecate_protos/deprecate_protos.py new file mode 100644 index 0000000000000..057095635517d --- /dev/null +++ b/tools/deprecate_protos/deprecate_protos.py @@ -0,0 +1,67 @@ +# A simple script to snag deprecated proto ields and add them to runtime_features.cc + +from __future__ import print_function +import re +import subprocess +import fileinput +from six.moves import input + + +# Sorts out the list of deprecated proto fields which should be disallowed and returns a tuple of +# email and code changes. +def deprecate_proto(): + grep_output = subprocess.check_output('grep -r "deprecated = true" api/*', shell=True) + + filenames_and_fields = set() + + # Compile the set of deprecated fields and the files they're in, deduping via set. + deprecated_regex = re.compile(r'.*\/([^\/]*.proto):[^=]* ([^= ]+) =.*') + for byte_line in grep_output.splitlines(): + line = byte_line.decode('utf-8') + match = deprecated_regex.match(line) + if match: + filenames_and_fields.add(tuple([match.group(1), match.group(2)])) + else: + print('no match in ' + line + ' please address manually!') + + # Now discard any deprecated features already listed in runtime_features + exiting_deprecated_regex = re.compile(r'.*"envoy.deprecated_features.(.*):(.*)",.*') + with open('source/common/runtime/runtime_features.cc', 'r') as features: + for line in features.readlines(): + match = exiting_deprecated_regex.match(line) + if match: + filenames_and_fields.discard(tuple([match.group(1), match.group(2)])) + + # Finally sort out the code to add to runtime_features.cc and a canned email for envoy-announce. + code_snippets = [] + email_snippets = [] + for (filename, field) in filenames_and_fields: + code_snippets.append(' "envoy.deprecated_features.' + filename + ':' + field + '",\n') + email_snippets.append(field + ' from ' + filename + '\n') + code = ''.join(code_snippets) + email = '' + if email_snippets: + email = ('\nThe following deprecated configuration fields will be disallowed by default:\n' + + ''.join(email_snippets)) + + return email, code + + +# Gather code and suggested email changes. +deprecate_email, deprecate_code = deprecate_proto() + +email = ('The Envoy maintainer team is cutting the next Envoy release. In the new release ' + + deprecate_email) + +print('\n\nSuggested envoy-announce email: \n') +print(email) + +if not input('Apply relevant runtime changes? [yN] ').strip().lower() in ('y', 'yes'): + exit(1) + +for line in fileinput.FileInput('source/common/runtime/runtime_features.cc', inplace=1): + if 'envoy.deprecated_features.deprecated.proto:is_deprecated_fatal' in line: + line = line.replace(line, line + deprecate_code) + print(line, end='') + +print('\nChanges applied. Please send the email above to envoy-announce.\n') diff --git a/tools/deprecate_protos/deprecate_protos.sh b/tools/deprecate_protos/deprecate_protos.sh new file mode 100644 index 0000000000000..d245bcb99bee1 --- /dev/null +++ b/tools/deprecate_protos/deprecate_protos.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. tools/shell_utils.sh + +set -e + +python_venv deprecate_protos diff --git a/tools/deprecate_protos/requirements.txt b/tools/deprecate_protos/requirements.txt new file mode 100644 index 0000000000000..b1013c19345c4 --- /dev/null +++ b/tools/deprecate_protos/requirements.txt @@ -0,0 +1 @@ +six==1.12.0 From a459602d7774dabadc9cef965fd77011f4f518f2 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 2 Dec 2019 16:05:04 -0500 Subject: [PATCH 2/3] actually, killing off flag flipping entirely Signed-off-by: Alyssa Wilk --- GOVERNANCE.md | 3 - .../deprecate_features/deprecate_features.py | 45 +++++++------ tools/deprecate_protos/deprecate_protos.py | 67 ------------------- tools/deprecate_protos/deprecate_protos.sh | 7 -- tools/deprecate_protos/requirements.txt | 1 - 5 files changed, 25 insertions(+), 98 deletions(-) delete mode 100644 tools/deprecate_protos/deprecate_protos.py delete mode 100644 tools/deprecate_protos/deprecate_protos.sh delete mode 100644 tools/deprecate_protos/requirements.txt diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 98e50f317a775..6b7fb0e16a581 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -104,9 +104,6 @@ or you can subscribe to the iCal feed [here](https://app.opsgenie.com/webcal/get * Run the deprecate_versions.py script (e.g. `sh tools/deprecate_version/deprecate_version.sh`) to file tracking issues for code which can be removed. * Run the deprecate_features.py script (e.g. `sh tools/deprecate_features/deprecate_features.sh`) - to flip any false runtime guards to true. Submit the resultant PR and send - an email to envoy-announce. -* Run the deprecate_protos.py script (e.g. `tools/deprecate_protos/deprecate_protos.sh`) to make the last release's deprecated features fatal-by-default. Submit the resultant PR and send an email to envoy-announce. diff --git a/tools/deprecate_features/deprecate_features.py b/tools/deprecate_features/deprecate_features.py index 7e12ffa9d2307..2ea14d4c3db0d 100644 --- a/tools/deprecate_features/deprecate_features.py +++ b/tools/deprecate_features/deprecate_features.py @@ -1,4 +1,4 @@ -# A simple script to flip runtime flags true. +# A simple script to snag deprecated proto fields and add them to runtime_features.cc from __future__ import print_function import re @@ -7,46 +7,51 @@ from six.moves import input -# Sorts out the list of features which should be default enabled and returns a tuple of +# Sorts out the list of deprecated proto fields which should be disallowed and returns a tuple of # email and code changes. -def flip_runtime_features(): - grep_output = subprocess.check_output('grep -r "envoy.reloadable_features\." source/*', - shell=True) +def deprecate_proto(): + grep_output = subprocess.check_output('grep -r "deprecated = true" api/*', shell=True) - features_to_flip = set() + filenames_and_fields = set() - # Compile the set of features to flip, deduping via set. - deprecated_regex = re.compile(r'.*"(envoy.reloadable_features\.[^"]+)".*') + # Compile the set of deprecated fields and the files they're in, deduping via set. + deprecated_regex = re.compile(r'.*\/([^\/]*.proto):[^=]* ([^= ]+) =.*') for byte_line in grep_output.splitlines(): line = byte_line.decode('utf-8') match = deprecated_regex.match(line) if match: - features_to_flip.add(match.group(1)) + filenames_and_fields.add(tuple([match.group(1), match.group(2)])) else: print('no match in ' + line + ' please address manually!') - # Exempt the two test flags. - features_to_flip.remove('envoy.reloadable_features.my_feature_name') - features_to_flip.remove('envoy.reloadable_features.test_feature_true') + # Now discard any deprecated features already listed in runtime_features + exiting_deprecated_regex = re.compile(r'.*"envoy.deprecated_features.(.*):(.*)",.*') + with open('source/common/runtime/runtime_features.cc', 'r') as features: + for line in features.readlines(): + match = exiting_deprecated_regex.match(line) + if match: + filenames_and_fields.discard(tuple([match.group(1), match.group(2)])) + # Finally sort out the code to add to runtime_features.cc and a canned email for envoy-announce. code_snippets = [] email_snippets = [] - for (feature) in features_to_flip: - code_snippets.append(' "' + feature + '",\n') - email_snippets.append(feature + '\n') + for (filename, field) in filenames_and_fields: + code_snippets.append(' "envoy.deprecated_features.' + filename + ':' + field + '",\n') + email_snippets.append(field + ' from ' + filename + '\n') code = ''.join(code_snippets) email = '' if email_snippets: - email = 'the following features will be defaulted to true:\n' + ''.join(email_snippets) + email = ('\nThe following deprecated configuration fields will be disallowed by default:\n' + + ''.join(email_snippets)) return email, code # Gather code and suggested email changes. -runtime_email, runtime_features_code = flip_runtime_features() +deprecate_email, deprecate_code = deprecate_proto() email = ('The Envoy maintainer team is cutting the next Envoy release. In the new release ' + - runtime_email) + deprecate_email) print('\n\nSuggested envoy-announce email: \n') print(email) @@ -55,8 +60,8 @@ def flip_runtime_features(): exit(1) for line in fileinput.FileInput('source/common/runtime/runtime_features.cc', inplace=1): - if 'envoy.reloadable_features.test_feature_true' in line: - line = line.replace(line, line + runtime_features_code) + if 'envoy.deprecated_features.deprecated.proto:is_deprecated_fatal' in line: + line = line.replace(line, line + deprecate_code) print(line, end='') print('\nChanges applied. Please send the email above to envoy-announce.\n') diff --git a/tools/deprecate_protos/deprecate_protos.py b/tools/deprecate_protos/deprecate_protos.py deleted file mode 100644 index 057095635517d..0000000000000 --- a/tools/deprecate_protos/deprecate_protos.py +++ /dev/null @@ -1,67 +0,0 @@ -# A simple script to snag deprecated proto ields and add them to runtime_features.cc - -from __future__ import print_function -import re -import subprocess -import fileinput -from six.moves import input - - -# Sorts out the list of deprecated proto fields which should be disallowed and returns a tuple of -# email and code changes. -def deprecate_proto(): - grep_output = subprocess.check_output('grep -r "deprecated = true" api/*', shell=True) - - filenames_and_fields = set() - - # Compile the set of deprecated fields and the files they're in, deduping via set. - deprecated_regex = re.compile(r'.*\/([^\/]*.proto):[^=]* ([^= ]+) =.*') - for byte_line in grep_output.splitlines(): - line = byte_line.decode('utf-8') - match = deprecated_regex.match(line) - if match: - filenames_and_fields.add(tuple([match.group(1), match.group(2)])) - else: - print('no match in ' + line + ' please address manually!') - - # Now discard any deprecated features already listed in runtime_features - exiting_deprecated_regex = re.compile(r'.*"envoy.deprecated_features.(.*):(.*)",.*') - with open('source/common/runtime/runtime_features.cc', 'r') as features: - for line in features.readlines(): - match = exiting_deprecated_regex.match(line) - if match: - filenames_and_fields.discard(tuple([match.group(1), match.group(2)])) - - # Finally sort out the code to add to runtime_features.cc and a canned email for envoy-announce. - code_snippets = [] - email_snippets = [] - for (filename, field) in filenames_and_fields: - code_snippets.append(' "envoy.deprecated_features.' + filename + ':' + field + '",\n') - email_snippets.append(field + ' from ' + filename + '\n') - code = ''.join(code_snippets) - email = '' - if email_snippets: - email = ('\nThe following deprecated configuration fields will be disallowed by default:\n' + - ''.join(email_snippets)) - - return email, code - - -# Gather code and suggested email changes. -deprecate_email, deprecate_code = deprecate_proto() - -email = ('The Envoy maintainer team is cutting the next Envoy release. In the new release ' + - deprecate_email) - -print('\n\nSuggested envoy-announce email: \n') -print(email) - -if not input('Apply relevant runtime changes? [yN] ').strip().lower() in ('y', 'yes'): - exit(1) - -for line in fileinput.FileInput('source/common/runtime/runtime_features.cc', inplace=1): - if 'envoy.deprecated_features.deprecated.proto:is_deprecated_fatal' in line: - line = line.replace(line, line + deprecate_code) - print(line, end='') - -print('\nChanges applied. Please send the email above to envoy-announce.\n') diff --git a/tools/deprecate_protos/deprecate_protos.sh b/tools/deprecate_protos/deprecate_protos.sh deleted file mode 100644 index d245bcb99bee1..0000000000000 --- a/tools/deprecate_protos/deprecate_protos.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -. tools/shell_utils.sh - -set -e - -python_venv deprecate_protos diff --git a/tools/deprecate_protos/requirements.txt b/tools/deprecate_protos/requirements.txt deleted file mode 100644 index b1013c19345c4..0000000000000 --- a/tools/deprecate_protos/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -six==1.12.0 From fd3bd1782630a2ecf0bcaead4d8afc79e546d128 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Mon, 2 Dec 2019 16:47:03 -0500 Subject: [PATCH 3/3] cleanup Signed-off-by: Alyssa Wilk --- tools/deprecate_features/deprecate_features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deprecate_features/deprecate_features.py b/tools/deprecate_features/deprecate_features.py index 2ea14d4c3db0d..3da00632ba9f5 100644 --- a/tools/deprecate_features/deprecate_features.py +++ b/tools/deprecate_features/deprecate_features.py @@ -17,7 +17,7 @@ def deprecate_proto(): # Compile the set of deprecated fields and the files they're in, deduping via set. deprecated_regex = re.compile(r'.*\/([^\/]*.proto):[^=]* ([^= ]+) =.*') for byte_line in grep_output.splitlines(): - line = byte_line.decode('utf-8') + line = str(byte_line) match = deprecated_regex.match(line) if match: filenames_and_fields.add(tuple([match.group(1), match.group(2)]))