From 4c1c987b845071c748c2e9c2c7074a1d4a5d5b09 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Wed, 30 Jan 2019 14:32:15 -0800 Subject: [PATCH 1/8] build: add pedantic spell check to build Adds support for fixing errors via the pedantic spell checker. Runs pedantic comment spelling checking during builds. Adds the location where python deps are installed during do_ci.sh (to avoid spurrious format check errors). *Risk Level*: low *Testing*: checks comment spelling *Docs Changes*: n/a *Release Notes*: n/a Signed-off-by: Stephan Zuercher --- .circleci/config.yml | 3 +- ci/do_ci.sh | 7 +- tools/check_format.py | 2 +- tools/check_spelling_pedantic.py | 408 ++++++++++++++++++++++--------- 4 files changed, 297 insertions(+), 123 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index decf0b22f3471..290475d9b52a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ executors: ubuntu-build: description: "A regular build executor based on ubuntu image" docker: - - image: envoyproxy/envoy-build:111781fa2823535762e9c514db0c5c41a119b4b1 + - image: envoyproxy/envoy-build:698009170e362f9ca0594f2b1927fbbee199bf98 resource_class: xlarge working_directory: /source @@ -127,6 +127,7 @@ jobs: - run: ci/do_circle_ci.sh check_format - run: ci/do_circle_ci.sh check_repositories - run: ci/do_circle_ci.sh check_spelling + - run: ci/do_circle_ci.sh check_spelling_pedantic build_image: docker: - image: circleci/python:3.7 diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 7f4cb02c461b4..5a8085a46fa11 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -7,7 +7,7 @@ set -e build_setup_args="" if [[ "$1" == "fix_format" || "$1" == "check_format" || "$1" == "check_repositories" || \ "$1" == "check_spelling" || "$1" == "fix_spelling" || "$1" == "bazel.clang_tidy" || \ - "$1" == "check_spelling_pedantic" ]]; then + "$1" == "check_spelling_pedantic" || "$1" == "fix_spelling_pedantic" ]]; then build_setup_args="-nofetch" fi @@ -305,6 +305,11 @@ elif [[ "$1" == "check_spelling_pedantic" ]]; then echo "check_spelling_pedantic..." ./tools/check_spelling_pedantic.py check exit 0 +elif [[ "$1" == "fix_spelling_pedantic" ]]; then + cd "${ENVOY_SRCDIR}" + echo "fix_spelling_pedantic..." + ./tools/check_spelling_pedantic.py fix + exit 0 elif [[ "$1" == "docs" ]]; then echo "generating docs..." docs/build.sh diff --git a/tools/check_format.py b/tools/check_format.py index 6468a275a97a7..f03defe713ee5 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -16,7 +16,7 @@ EXCLUDED_PREFIXES = ("./generated/", "./thirdparty/", "./build", "./.git/", "./bazel-", "./.cache", "./source/extensions/extensions_build_config.bzl", - "./tools/testdata/check_format/") + "./tools/testdata/check_format/", "./tools/pyformat/") SUFFIXES = (".cc", ".h", "BUILD", "WORKSPACE", ".bzl", ".md", ".rst", ".proto") DOCS_SUFFIX = (".md", ".rst") PROTO_SUFFIX = (".proto") diff --git a/tools/check_spelling_pedantic.py b/tools/check_spelling_pedantic.py index 428cfe308bfec..59cf7bccfd7e6 100755 --- a/tools/check_spelling_pedantic.py +++ b/tools/check_spelling_pedantic.py @@ -3,12 +3,19 @@ from __future__ import print_function import argparse +import math import os import re import subprocess import sys -# TODO(zuercher): provide support for fixing errors +from functools import partial + +# handle function rename between python 2/3 +try: + input = raw_input +except NameError: + pass TOOLS_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -73,17 +80,130 @@ def debug(s): print(s) +class SpellChecker: + """Aspell-based spell checker.""" + + def __init__(self, dictionary_file): + self.dictionary_file = dictionary_file + self.aspell = None + + def start(self): + words = self.load_dictionary() + + # Generate aspell personal dictionary. + pws = os.path.join(TOOLS_DIR, '.aspell.en.pws') + with open(pws, 'w') as f: + f.write("personal_ws-1.1 en %d\n" % (len(words))) + for word in words: + f.write(word) + + # Start an aspell process. + aspell_args = ["aspell", "pipe", "--run-together", "--encoding=utf-8", "--personal=" + pws] + self.aspell = subprocess.Popen( + aspell_args, + bufsize=4096, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + + # Read the version line that aspell emits on startup. + self.aspell.stdout.readline() + + def stop(self): + if not self.aspell: + return + + self.aspell.stdin.close() + self.aspell.wait() + self.aspell = None + + def check(self, line): + self.aspell.poll() + if self.aspell.returncode is not None: + print("aspell quit unexpectedly: return code %d" % (self.aspell.returncode)) + sys.exit(1) + + debug("ASPELL< %s" % (line)) + + self.aspell.stdin.write(line + os.linesep) + self.aspell.stdin.flush() + + errors = [] + while True: + result = self.aspell.stdout.readline().strip() + debug("ASPELL> %s" % (result)) + + if result == "": + break # handled all results + + t = result[0] + if t == "*" or t == "-" or t == "+": + # *: found in dictionary + # -: found run-together words in dictionary + # +: found root word in dictionary + continue + + # & : m1, m2, ... mN, g1, g2, ... + # ? 0 : g1, g2, .... + # # + original, rem = result[2:].split(" ", 1) + + if t == "#": + # Not in dictionary, but no suggestions + errors.append((original, int(rem), [])) + elif t == '&' or t == '?': + # Near misses and/or guesses + _, rem = rem.split(" ", 1) # drop N (or 0) + o, rem = rem.split(": ", 1) # o is offset from start of line + suggestions = rem.split(", ") + + errors.append((original, int(o), suggestions)) + else: + print("aspell produced unexpected output: %s" % (result)) + sys.exit(1) + + return errors + + def load_dictionary(self): + # Read the custom dictionary + words = [] + with open(self.dictionary_file, 'r') as f: + words = f.readlines() + + # Strip comments and blank lines. + words = [w for w in words if len(w) > 0 and w[0] != "#"] + + # Allow acronyms and abbreviations to be spelled in lowercase. + # (e.g. Convert "HTTP"" into "HTTP" and "http" which also matches + # "Http"). + for word in words: + if word.isupper(): + words += word.lower() + + return words + + def add_words(self, additions): + additions = [w + os.linesep for w in additions] + + with open(self.dictionary_file, 'a') as f: + f.writelines(additions) + + self.stop() + self.start() + + # Split camel case words and run them through the dictionary. Returns # True if they are all spelled correctly, False if word is not camel # case or has a misspelled sub-word. -def check_camel_case(aspell, word): +def check_camel_case(checker, word): # Words is not camel case: the previous result stands. parts = re.findall(CAMEL_CASE, word) if len(parts) <= 1: return False for part in parts: - if check_comment(aspell, 0, part): + if check_comment(checker, 0, part): # Part of camel case word is misspelled, the result stands. return False @@ -105,10 +225,10 @@ def mask_with_regex(comment, regex, group, secondary=None): return comment -# Checks the comment at offset against the aspell pipe. Result is an array +# Checks the comment at offset against the spell checker. Result is an array # of tuples where each tuple is the misspelled word, it's offset from the # start of the line, and an array of possible replacements. -def check_comment(aspell, offset, comment): +def check_comment(checker, offset, comment): # Replace TODO comments with spaces to preserve string offsets. comment = mask_with_regex(comment, TODO, 0) @@ -139,77 +259,165 @@ def check_comment(aspell, offset, comment): if comment == "" or comment.strip() == "": return [] - # aspell does not like leading punctuation + # Mask leading punctuation. if not comment[0].isalnum(): comment = ' ' + comment[1:] - errors = [] + errors = checker.check(comment) - aspell.poll() - if aspell.returncode is not None: - print("aspell quit unexpectedly: return code %d" % (aspell.returncode)) - sys.exit(1) + # Fix up offsets relative to the start of the line vs start of the comment. + errors = [(w, o + offset, s) for (w, o, s) in errors] - debug("ASPELL< %s" % (comment)) + # Retry camel case words after splitting them. + errors = [err for err in errors if not check_camel_case(checker, err[0])] + return errors - aspell.stdin.write(comment + os.linesep) - aspell.stdin.flush() - while True: - result = aspell.stdout.readline().strip() - debug("ASPELL> %s" % (result)) - if result == "": - break # handled all results +def print_error(file, line_offset, lines, errors): + # Highlight misspelled words. + line = lines[line_offset] + prefix = "%s:%d:" % (file, line_offset + 1) + for (word, offset, suggestions) in reversed(errors): + line = line[:offset] + red(word) + line[offset + len(word):] - t = result[0] - if t == "*" or t == "-" or t == "+": - # *: found in dictionary - # -: found run-together words in dictionary - # +: found root word in dictionary + print("%s%s" % (prefix, line.rstrip())) + + if MARK: + # Print a caret at the start of each misspelled word. + marks = ' ' * len(prefix) + last = 0 + for (word, offset, suggestions) in errors: + marks += (' ' * (offset - last)) + '^' + last = offset + 1 + print(marks) + + +def print_fix_options(word, suggestions): + print("%s:" % (word)) + print(" a: accept and add to dictionary") + print(" A: accept and add to dictionary as ALLCAPS (for acronyms)") + print(" i: ignore") + print(" r : replace with given word and add to dictionary") + print(" R : replace with given word and add to dictionary as ALLCAPS (for acronyms)") + print(" x: abort") + + col_width = max(len(word) for word in suggestions) + opt_width = int(math.log(len(suggestions), 10)) + 1 + padding = 2 # two spaces of padding + delim = 2 # colon and space after number + num_cols = int(78 / (col_width + padding + opt_width + delim)) + num_rows = int(len(suggestions) / num_cols + 1) + rows = [""] * num_rows + + indent = " " * padding + for idx, sugg in enumerate(suggestions): + row = idx % len(rows) + row_data = "%d: %s" % (idx, sugg) + + rows[row] += indent + row_data.ljust(col_width + opt_width + delim) + + for row in rows: + print(row) + + +def fix_error(checker, file, line_offset, lines, errors): + print_error(file, line_offset, lines, errors) + + fixed = {} + replacements = [] + additions = [] + for (word, offset, suggestions) in errors: + if word in fixed: + # Same typo was repeated in a line, so just reuse the previous choice. + replacements += [fixed[word]] continue - # & : m1, m2, ... mN, g1, g2, ... - # ? 0 : g1, g2, .... - # # - original, rem = result[2:].split(" ", 1) - - if t == "#": - # Not in dictionary, but no suggestions - errors.append((original, int(rem) + offset, [])) - elif t == '&' or t == '?': - # Near misses and/or guesses - _, rem = rem.split(" ", 1) # drop N (or 0) - o, rem = rem.split(": ", 1) # o is offset from start of comment - suggestions = rem.split(", ") - - errors.append((original, int(o) + offset, suggestions)) - else: - print("aspell produced unexpected output: %s" % (result)) - sys.exit(2) + print_fix_options(word, suggestions) + + replacement = "" + while replacement == "": + try: + choice = input("> ") + except EOFError: + choice = "x" + + add = None + if choice == "x": + print("Spell checking aborted.") + sys.exit(2) + elif choice == "a": + replacement = word + add = word + elif choice == "A": + replacement = word + add = word.upper() + elif choice == "i": + replacement = word + elif choice[:1] == "r" or choice[:1] == "R": + replacement = choice[1:].strip() + if replacement == "": + print("Invalid choice: '%s'. Must specify a replacement (e.g. 'r corrected')." % (choice)) + continue + + if choice[:1] == "R": + if replacement.upper() not in suggestions: + add = replacement.upper() + elif replacement not in suggestions: + add = replacement + elif choice == 's': + for idx, sugg in enumerate(suggestions[:10]): + print("\t%d: %s" % (idx, sugg)) + else: + try: + idx = int(choice) + except ValueError: + idx = -1 + if idx >= 0 and idx < len(suggestions): + replacement = suggestions[idx] + else: + print("Invalid choice: '%s'" % (choice)) + + fixed[word] = replacement + replacements += [replacement] + if add: + additions += [add] + + if len(errors) != len(replacements): + print("Internal error %d errors with %d replacements" % (len(errors), len(replacements))) + sys.exit(1) - # Retry camel case words after splitting them. - errors = [err for err in errors if not check_camel_case(aspell, err[0])] - return errors + # Perform replacements on the line + line = lines[line_offset] + for idx in range(len(replacements) - 1, -1, -1): + word, offset, _ = errors[idx] + replacement = replacements[idx] + if word == replacement: + continue + line = line[:offset] + replacement + line[offset + len(word):] + lines[line_offset] = line -def check_file(aspell, file, lines): + # Update dictionary + checker.add_words(additions) + + +def check_file(checker, file, lines, error_handler): in_comment = False - line_num = 0 - num = 0 - for line in lines: - line_num += 1 + num_comments = 0 + num_errors = 0 + for line_idx, line in enumerate(lines): errors = [] last = 0 if in_comment: mc_end = MULTI_COMMENT_END.search(line) if mc_end is None: # Full line is within a multi-line comment. - errors += check_comment(aspell, 0, line) - num += 1 + errors += check_comment(checker, 0, line) + num_comments += 1 else: # Start of line is the end of a multi-line comment. - errors += check_comment(aspell, 0, mc_end.group(1)) - num += 1 + errors += check_comment(checker, 0, mc_end.group(1)) + num_comments += 1 last = mc_end.end() in_comment = False @@ -217,88 +425,48 @@ def check_file(aspell, file, lines): for inline in INLINE_COMMENT.finditer(line, last): # Single-line comment m = inline.lastindex # 1 or 2 depending on group matched - errors += check_comment(aspell, inline.start(m), inline.group(m)) - num += 1 + errors += check_comment(checker, inline.start(m), inline.group(m)) + num_comments += 1 last = inline.end(m) if last < len(line): mc_start = MULTI_COMMENT_START.search(line, last) if mc_start is not None: # New multi-lie comment starts at end of line. - errors += check_comment(aspell, mc_start.start(1), mc_start.group(1)) - num += 1 + errors += check_comment(checker, mc_start.start(1), mc_start.group(1)) + num_comments += 1 in_comment = True if errors: - # Highlight misspelled words. - prefix = "%s:%d:" % (file, line_num) - for (word, offset, suggestions) in reversed(errors): - line = line[:offset] + red(word) + line[offset + len(word):] - - print("%s%s" % (prefix, line.rstrip())) - - if MARK: - # Print a caret at the start of each misspelled word. - marks = ' ' * len(prefix) - last = 0 - for (word, offset, suggestions) in errors: - marks += (' ' * (offset - last)) + '^' - last = offset + 1 - print(marks) - - return num - - -def start_aspell(dictionary): - # Read the custom dictionary - words = [] - with open(dictionary, 'r') as f: - words = f.readlines() - - # Strip comments. - words = [w for w in words if len(w) > 0 and w[0] != "#"] - - # Allow acronyms and abbreviations to be spelled in lowercase. - # (e.g. Convert "HTTP"" into "HTTP" and "http" which also matches - # "Http"). - for word in words: - if word.isupper(): - words += word.lower() - - # Generate aspell personal dictionary. - pws = os.path.join(TOOLS_DIR, '.aspell.en.pws') - with open(pws, 'w') as f: - f.write("personal_ws-1.1 en %d\n" % (len(words))) - for word in words: - f.write(word) + num_errors += len(errors) + error_handler(file, line_idx, lines, errors) - # Start an aspell process. - aspell_args = ["aspell", "pipe", "--run-together", "--encoding=utf-8", "--personal=" + pws] - aspell = subprocess.Popen( - aspell_args, - bufsize=4096, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) + return (num_comments, num_errors) - # Read the version line that aspell emits on startup. - aspell.stdout.readline() - return aspell +def execute(files, dictionary_file, fix): + checker = SpellChecker(dictionary_file) + checker.start() -def execute(files, dictionary): - aspell = start_aspell(dictionary) + handler = print_error + if fix: + handler = partial(fix_error, checker) - num = 0 + total_comments = 0 + total_errors = 0 for path in files: with open(path, 'r') as f: lines = f.readlines() - num += check_file(aspell, path, lines) + (num_comments, num_errors) = check_file(checker, path, lines, handler) + total_comments += num_comments + total_errors += num_errors + + if fix and num_errors > 0: + with open(path, 'w') as f: + f.writelines(lines) - aspell.stdin.close() - aspell.wait() + checker.stop() - print("Checked %d lines of comments" % (num)) + print("Checked %d comments, found %d errors." % (total_comments, total_errors)) if __name__ == "__main__": @@ -308,7 +476,7 @@ def execute(files, dictionary): parser.add_argument( 'operation_type', type=str, - choices=['check'], + choices=['check', 'fix'], help="specify if the run should 'check' or 'fix' spelling.") parser.add_argument( 'target_paths', type=str, nargs="*", help="specify the files for the script to process.") @@ -341,4 +509,4 @@ def execute(files, dictionary): for root, _, files in os.walk(p): target_paths += [os.path.join(root, f) for f in files if os.path.splitext(f)[1] in exts] - execute(target_paths, args.dictionary) + execute(target_paths, args.dictionary, args.operation_type == 'fix') From c98e03b95e62e3bb60be71104e0cfae29c50e5f9 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Wed, 30 Jan 2019 15:19:15 -0800 Subject: [PATCH 2/8] set exit code on check Signed-off-by: Stephan Zuercher --- tools/check_spelling_pedantic.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/check_spelling_pedantic.py b/tools/check_spelling_pedantic.py index 59cf7bccfd7e6..3e8443909ee96 100755 --- a/tools/check_spelling_pedantic.py +++ b/tools/check_spelling_pedantic.py @@ -122,7 +122,7 @@ def check(self, line): self.aspell.poll() if self.aspell.returncode is not None: print("aspell quit unexpectedly: return code %d" % (self.aspell.returncode)) - sys.exit(1) + sys.exit(2) debug("ASPELL< %s" % (line)) @@ -161,7 +161,7 @@ def check(self, line): errors.append((original, int(o), suggestions)) else: print("aspell produced unexpected output: %s" % (result)) - sys.exit(1) + sys.exit(2) return errors @@ -384,7 +384,7 @@ def fix_error(checker, file, line_offset, lines, errors): if len(errors) != len(replacements): print("Internal error %d errors with %d replacements" % (len(errors), len(replacements))) - sys.exit(1) + sys.exit(2) # Perform replacements on the line line = lines[line_offset] @@ -468,6 +468,8 @@ def execute(files, dictionary_file, fix): print("Checked %d comments, found %d errors." % (total_comments, total_errors)) + return total_errors == 0 + if __name__ == "__main__": default_dictionary = os.path.join(TOOLS_DIR, 'spelling_dictionary.txt') @@ -509,4 +511,11 @@ def execute(files, dictionary_file, fix): for root, _, files in os.walk(p): target_paths += [os.path.join(root, f) for f in files if os.path.splitext(f)[1] in exts] - execute(target_paths, args.dictionary, args.operation_type == 'fix') + rv = execute(target_paths, args.dictionary, args.operation_type == 'fix') + + if args.operation_type == 'check': + if not rv: + print("ERROR: spell check failed. Run 'tool/check_spelling_pedantic.py fix'") + sys.exit(1) + + print("PASS") From 8596bc11252698ced0810e24ec03af9887e0eb60 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Wed, 30 Jan 2019 15:31:22 -0800 Subject: [PATCH 3/8] fix the latest typos Signed-off-by: Stephan Zuercher --- test/common/network/udp_listener_impl_test.cc | 4 ++-- tools/spelling_dictionary.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index 3251d1b9775ed..d9262e2dc052b 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -256,7 +256,7 @@ TEST_P(ListenerImplTest, UdpEcho) { getSocketAddressInfo(*client_socket.get(), server_ip->port(), server_addr, addr_len); ASSERT_GT(addr_len, 0); - // We send 2 packets and exptect it to echo. + // We send 2 packets and expect it to echo. const std::string first("first"); const std::string second("second"); @@ -456,7 +456,7 @@ TEST_P(ListenerImplTest, UdpListenerEnableDisable) { } /** - * Tests UDP listebe's error callback. + * Tests UDP listener's error callback. */ TEST_P(ListenerImplTest, UdpListenerRecvFromError) { // Setup server socket diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 5122aeee620a4..dc1162faa6134 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -738,3 +738,5 @@ xyz zag zig zlib +recvmmsg +OS From 7187bbf91e027b252b7fd2d823f5f0507a3895d8 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Wed, 30 Jan 2019 15:39:11 -0800 Subject: [PATCH 4/8] one more Signed-off-by: Stephan Zuercher --- include/envoy/stats/symbol_table.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/envoy/stats/symbol_table.h b/include/envoy/stats/symbol_table.h index 3bc6ee4069835..b0efd1cbd75d1 100644 --- a/include/envoy/stats/symbol_table.h +++ b/include/envoy/stats/symbol_table.h @@ -87,7 +87,7 @@ class SymbolTable { * with a corrupt stats set. * * @param stat_name the stat name. - * @return std::string stringifiied stat_name. + * @return std::string stringified stat_name. */ virtual std::string toString(const StatName& stat_name) const PURE; @@ -122,7 +122,7 @@ class SymbolTable { * * This is intended for use doing cached name lookups of scoped stats, where * the scope prefix and the names to combine it with are already in StatName - * form. Using this class, they can be combined without acessingm the + * form. Using this class, they can be combined without accessing the * SymbolTable or, in particular, taking its lock. * * @param stat_names the names to join. From 2f28461493870b97d6b7b4be4cf196a964468e1a Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Thu, 31 Jan 2019 14:32:19 -0800 Subject: [PATCH 5/8] review comments Signed-off-by: Stephan Zuercher --- tools/check_spelling_pedantic.py | 2 +- tools/spelling_dictionary.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/check_spelling_pedantic.py b/tools/check_spelling_pedantic.py index 3e8443909ee96..c1f940db8202d 100755 --- a/tools/check_spelling_pedantic.py +++ b/tools/check_spelling_pedantic.py @@ -175,7 +175,7 @@ def load_dictionary(self): words = [w for w in words if len(w) > 0 and w[0] != "#"] # Allow acronyms and abbreviations to be spelled in lowercase. - # (e.g. Convert "HTTP"" into "HTTP" and "http" which also matches + # (e.g. Convert "HTTP" into "HTTP" and "http" which also matches # "Http"). for word in words: if word.isupper(): diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index dc1162faa6134..c6ab0836e7fa3 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -148,6 +148,7 @@ Nilsson OCSP OK OOM +OS OSI OSS OSX @@ -587,6 +588,7 @@ rebalancing rebuffer recurse recv +recvmmsg redis redistributions refactored @@ -738,5 +740,3 @@ xyz zag zig zlib -recvmmsg -OS From a25ae7efe3e09b0c2a76fd14fe255f33f3d843bc Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Fri, 1 Feb 2019 09:45:47 -0800 Subject: [PATCH 6/8] fix comments; sorted insert into dictionary Signed-off-by: Stephan Zuercher --- tools/check_spelling_pedantic.py | 87 ++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/tools/check_spelling_pedantic.py b/tools/check_spelling_pedantic.py index c1f940db8202d..aeb831afc0f62 100755 --- a/tools/check_spelling_pedantic.py +++ b/tools/check_spelling_pedantic.py @@ -11,7 +11,7 @@ from functools import partial -# handle function rename between python 2/3 +# Handle function rename between python 2/3. try: input = raw_input except NameError: @@ -19,19 +19,19 @@ TOOLS_DIR = os.path.dirname(os.path.realpath(__file__)) -# e.g., // comment OR /* comment */ (single line) +# Single line comments: // comment OR /* comment */ # Limit the characters that may precede // to help filter out some code # mistakenly processed as a comment. INLINE_COMMENT = re.compile(r'(?:^|[^:"])//(.*?)$|/\*+(.*?)\*+/') -# e.g., /* comment */ (multiple lines) +# Multi-line comments: /* comment */ (multiple lines) MULTI_COMMENT_START = re.compile(r'/\*(.*?)$') MULTI_COMMENT_END = re.compile(r'^(.*?)\*/') -# e.g., TODO(username): blah +# Envoy TODO comment style. TODO = re.compile(r'(TODO|NOTE)\s*\(@?[A-Za-z0-9-]+\):?') -# e.g., ignore parameter names in doxygen comments +# Ignore parameter names in doxygen comments. METHOD_DOC = re.compile('@(param\s+\w+|return(\s+const)?\s+\w+)') # Camel Case splitter @@ -45,7 +45,7 @@ # Hex: match 1) longish strings of hex digits (to avoid matching "add" and # other simple words that happen to look like hex), 2) 2 or more two digit # hex numbers separated by colons, 3) "0x" prefixed hex numbers of any length, -# or 4) UUIDs +# or 4) UUIDs. HEX = re.compile(r'(?:^|\s|[(])([A-Fa-f0-9]{8,})(?:$|\s|[.,)])') HEX_SIG = re.compile(r'\W([A-Fa-f0-9]{2}(:[A-Fa-f0-9]{2})+)\W') PREFIXED_HEX = re.compile(r'0x[A-Fa-f0-9]+') @@ -55,13 +55,13 @@ # aspell ignores that anyway. IPV6_ADDR = re.compile(r'\W([A-Fa-f0-9]+:[A-Fa-f0-9:]+/[0-9]{1,3})\W') -# Quoted words: "word", 'word', or *word* +# Quoted words: "word", 'word', or *word*. QUOTED_WORD = re.compile(r'(["\'*])[A-Za-z0-9]+(\1)') -# Command flags (e.g. "-rf") and percent specifiers +# Command flags (e.g. "-rf") and percent specifiers. FLAG = re.compile(r'\W([-%][A-Za-z]+)') -# Bare github users (e.g. @user) +# Bare github users (e.g. @user). USER = re.compile(r'\W(@[A-Za-z0-9-]+)') DEBUG = False @@ -94,8 +94,7 @@ def start(self): pws = os.path.join(TOOLS_DIR, '.aspell.en.pws') with open(pws, 'w') as f: f.write("personal_ws-1.1 en %d\n" % (len(words))) - for word in words: - f.write(word) + f.writelines(words) # Start an aspell process. aspell_args = ["aspell", "pipe", "--run-together", "--encoding=utf-8", "--personal=" + pws] @@ -134,14 +133,15 @@ def check(self, line): result = self.aspell.stdout.readline().strip() debug("ASPELL> %s" % (result)) + # Check for end of results. if result == "": - break # handled all results + break t = result[0] if t == "*" or t == "-" or t == "+": - # *: found in dictionary - # -: found run-together words in dictionary - # +: found root word in dictionary + # *: found in dictionary. + # -: found run-together words in dictionary. + # +: found root word in dictionary. continue # & : m1, m2, ... mN, g1, g2, ... @@ -150,12 +150,12 @@ def check(self, line): original, rem = result[2:].split(" ", 1) if t == "#": - # Not in dictionary, but no suggestions + # Not in dictionary, but no suggestions. errors.append((original, int(rem), [])) elif t == '&' or t == '?': - # Near misses and/or guesses - _, rem = rem.split(" ", 1) # drop N (or 0) - o, rem = rem.split(": ", 1) # o is offset from start of line + # Near misses and/or guesses. + _, rem = rem.split(" ", 1) # Drop N (may be 0). + o, rem = rem.split(": ", 1) # o is offset from start of line. suggestions = rem.split(", ") errors.append((original, int(o), suggestions)) @@ -166,13 +166,13 @@ def check(self, line): return errors def load_dictionary(self): - # Read the custom dictionary + # Read the custom dictionary. words = [] with open(self.dictionary_file, 'r') as f: words = f.readlines() # Strip comments and blank lines. - words = [w for w in words if len(w) > 0 and w[0] != "#"] + words = [w for w in words if len(w.strip()) > 0 and w[0] != "#"] # Allow acronyms and abbreviations to be spelled in lowercase. # (e.g. Convert "HTTP" into "HTTP" and "http" which also matches @@ -184,10 +184,32 @@ def load_dictionary(self): return words def add_words(self, additions): - additions = [w + os.linesep for w in additions] + lines = [] + with open(self.dictionary_file, 'r') as f: + lines = f.readlines() - with open(self.dictionary_file, 'a') as f: - f.writelines(additions) + additions = [w + os.linesep for w in additions] + additions.sort() + + # Insert additions into the lines ignoring comments and blank lines. + idx = 0 + add_idx = 0 + while idx < len(lines) and add_idx < len(additions): + line = lines[idx] + if len(line.strip()) != 0 and line[0] != "#": + c = cmp(additions[add_idx], line) + if c < 0: + lines.insert(idx, additions[add_idx]) + add_idx += 1 + elif c == 0: + add_idx += 1 + idx += 1 + + # Append any remaining additions. + lines += additions[add_idx:] + + with open(self.dictionary_file, 'w') as f: + f.writelines(lines) self.stop() self.start() @@ -197,8 +219,9 @@ def add_words(self, additions): # True if they are all spelled correctly, False if word is not camel # case or has a misspelled sub-word. def check_camel_case(checker, word): - # Words is not camel case: the previous result stands. parts = re.findall(CAMEL_CASE, word) + + # Word is not camel case: the previous result stands. if len(parts) <= 1: return False @@ -255,7 +278,7 @@ def check_comment(checker, offset, comment): # Github user refs: comment = mask_with_regex(comment, USER, 1) - # Everything got stripped, return early. + # Everything got masked, return early. if comment == "" or comment.strip() == "": return [] @@ -303,8 +326,8 @@ def print_fix_options(word, suggestions): col_width = max(len(word) for word in suggestions) opt_width = int(math.log(len(suggestions), 10)) + 1 - padding = 2 # two spaces of padding - delim = 2 # colon and space after number + padding = 2 # Two spaces of padding. + delim = 2 # Colon and space after number. num_cols = int(78 / (col_width + padding + opt_width + delim)) num_rows = int(len(suggestions) / num_cols + 1) rows = [""] * num_rows @@ -386,7 +409,7 @@ def fix_error(checker, file, line_offset, lines, errors): print("Internal error %d errors with %d replacements" % (len(errors), len(replacements))) sys.exit(2) - # Perform replacements on the line + # Perform replacements on the line. line = lines[line_offset] for idx in range(len(replacements) - 1, -1, -1): word, offset, _ = errors[idx] @@ -397,7 +420,7 @@ def fix_error(checker, file, line_offset, lines, errors): line = line[:offset] + replacement + line[offset + len(word):] lines[line_offset] = line - # Update dictionary + # Update the dictionary. checker.add_words(additions) @@ -423,8 +446,8 @@ def check_file(checker, file, lines, error_handler): if not in_comment: for inline in INLINE_COMMENT.finditer(line, last): - # Single-line comment - m = inline.lastindex # 1 or 2 depending on group matched + # Single-line comment. + m = inline.lastindex # 1 or 2 depending on group matched. errors += check_comment(checker, inline.start(m), inline.group(m)) num_comments += 1 last = inline.end(m) From e34b3bb3112a6b22b526576dcda8e7298810a291 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Fri, 1 Feb 2019 10:15:46 -0800 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=98=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stephan Zuercher --- tools/check_spelling_pedantic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check_spelling_pedantic.py b/tools/check_spelling_pedantic.py index aeb831afc0f62..3a6a23938d484 100755 --- a/tools/check_spelling_pedantic.py +++ b/tools/check_spelling_pedantic.py @@ -154,7 +154,7 @@ def check(self, line): errors.append((original, int(rem), [])) elif t == '&' or t == '?': # Near misses and/or guesses. - _, rem = rem.split(" ", 1) # Drop N (may be 0). + _, rem = rem.split(" ", 1) # Drop N (may be 0). o, rem = rem.split(": ", 1) # o is offset from start of line. suggestions = rem.split(", ") From bc31df21d8c3dd55eaf6c4487529c82c9a7c64ff Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Fri, 1 Feb 2019 11:20:30 -0800 Subject: [PATCH 8/8] fix new spelling errors from master Signed-off-by: Stephan Zuercher --- include/envoy/stats/histogram.h | 2 +- source/extensions/common/tap/tap_matcher.h | 2 +- tools/spelling_dictionary.txt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/envoy/stats/histogram.h b/include/envoy/stats/histogram.h index ffd76f274b248..bebba3ff976e5 100644 --- a/include/envoy/stats/histogram.h +++ b/include/envoy/stats/histogram.h @@ -45,7 +45,7 @@ class HistogramStatistics { virtual const std::vector& supportedBuckets() const PURE; /** - * Returns computed bucket values during the period. The vector contains an appoximation + * Returns computed bucket values during the period. The vector contains an approximation * of samples below each quantile bucket defined in supportedBuckets(). This vector is * guaranteed to be the same length as supportedBuckets(). */ diff --git a/source/extensions/common/tap/tap_matcher.h b/source/extensions/common/tap/tap_matcher.h index 0380967c14ddb..c2d68fa0bc83f 100644 --- a/source/extensions/common/tap/tap_matcher.h +++ b/source/extensions/common/tap/tap_matcher.h @@ -46,7 +46,7 @@ class Matcher { size_t index() { return my_index_; } /** - * Update match status when a stream is created. This might be an HTTP stream, a TCP connectin, + * Update match status when a stream is created. This might be an HTTP stream, a TCP connection, * etc. This allows any matchers to flip to an initial state of true if applicable. */ virtual bool onNewStream(std::vector& statuses) const PURE; diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index c6ab0836e7fa3..f7a4911917f8a 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -591,6 +591,7 @@ recv recvmmsg redis redistributions +reentrant refactored refactoring referencee