diff --git a/.github/problem-matcher-lint-check-licences.json b/.github/problem-matcher-lint-check-licences.json new file mode 100644 index 0000000000..33b3fcfb2f --- /dev/null +++ b/.github/problem-matcher-lint-check-licences.json @@ -0,0 +1,16 @@ +{ + "problemMatcher": [ + { + "owner": "lint-check-licences", + "pattern": [ + { + "regexp": "^(notice|error)(:(file|dir):([^:]+)(:lines? (\\d+)(-(\\d+))?)?)?: (.+)$", + "severity": 1, + "file": 4, + "line": 6, + "message": 9 + } + ] + } + ] +} diff --git a/.github/problem-matcher-lint-cpplint.json b/.github/problem-matcher-lint-cpplint.json new file mode 100644 index 0000000000..0ca1d6199c --- /dev/null +++ b/.github/problem-matcher-lint-cpplint.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "lint-cpplint", + "severity": "error", + "pattern": [ + { + "regexp": "^(.+):(\\d+):\\s+((.+)\\s+\\[(.*)\\]\\s\\[(.*)\\])$", + "file": 1, + "line": 2, + "message": 3, + "code": 5 + } + ] + } + ] +} diff --git a/.github/problem-matcher-lint-generic-nolints.json b/.github/problem-matcher-lint-generic-nolints.json new file mode 100644 index 0000000000..114c9b1663 --- /dev/null +++ b/.github/problem-matcher-lint-generic-nolints.json @@ -0,0 +1,16 @@ +{ + "problemMatcher": [ + { + "owner": "lint-generic-nolints", + "pattern": [ + { + "regexp": "^(notice|error)(:(file|dir):([^:]+)(:lines? (\\d+)(-(\\d+))?)?)?: (.+)$", + "severity": 1, + "file": 4, + "line": 6, + "message": 9 + } + ] + } + ] +} diff --git a/.github/workflows/annotation.yml b/.github/workflows/annotation.yml index 357d18975e..4b52634e45 100644 --- a/.github/workflows/annotation.yml +++ b/.github/workflows/annotation.yml @@ -12,13 +12,3 @@ jobs: - name: Flake8 with annotations - Python 2 # Using the flake8 options in the .flake8 file uses: TrueBrain/actions-flake8@main - flake8-annotation-python3: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - name: Flake8 with annotations - Python 3 - # Using the flake8 options in the .flake8 file - uses: TrueBrain/actions-flake8@main diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000000..c0ac97b39b --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,196 @@ +name: lint +on: [push, pull_request] +jobs: + build: + name: Build for Lint Tasks + runs-on: ubuntu-latest + container: debian:stable + steps: + - name: Update package database + run: apt-get update -y + # See comments beginning at + # https://github.com/actions/runner/issues/763#issuecomment-1435474884 + # Without Git, actions/checkout@v3 will resort to REST and will not + # create a .git folder or .git.config. The Problem Matcher looks for + # .git/config to find where the root of the repo is, so it must be + # present. + - name: Install Git + run: apt-get -y install git + - uses: actions/checkout@v3 + - name: Install build tools + shell: bash + run: | + apt-get -y install pkg-config libtool autoconf \ + automake g++ bison flex make bash-completion dh-autoreconf \ + debhelper devscripts wget python3-pip + - name: Install Python lint tools + run: python3 -m pip install --no-input cpplint flake8 + - name: Install build dependencies + shell: bash + run: | + apt-get -y install libcppunit-dev uuid-dev libncurses5-dev \ + libmicrohttpd-dev protobuf-compiler python3-protobuf \ + libprotobuf-dev libprotoc-dev zlib1g-dev libftdi-dev \ + libusb-1.0-0-dev liblo-dev libavahi-client-dev python3-numpy + - name: Autoreconf + run: autoreconf -i + - name: Configure + run: ./configure --enable-rdm-tests --enable-ja-rule --enable-e133 + - name: Build builtfiles + run: make builtfiles VERBOSE=1 + - name: Display structure of the built files + if: env.ACTIONS_STEP_DEBUG == 'true' + run: ls -alR + - name: Archive artifacts to speed up slow GH Actions upload/download + shell: bash + # If the file does not exist when tar excludes it, then it will not + # actually exclude it, so it must first be touched + run: | + touch ola-debian-stable-built-source-tree.tar.gz + tar --exclude=ola-debian-stable-built-source-tree.tar.gz -cvzf ola-debian-stable-built-source-tree.tar.gz . + - name: SHA256 artifact archive + run: sha256sum ola-debian-stable-built-source-tree.tar.gz + - uses: actions/upload-artifact@v3 + with: + name: ola-debian-stable-built-source-tree + path: ola-debian-stable-built-source-tree.tar.gz + check-licences: + name: Check Licences + runs-on: ubuntu-latest + container: debian:stable + needs: build + steps: + - name: Download built source tree archive + uses: actions/download-artifact@v3 + with: + name: ola-debian-stable-built-source-tree + path: . + - name: SHA256 artifact archive + run: sha256sum ola-debian-stable-built-source-tree.tar.gz + - name: Unarchive artifacts and delete archive + shell: bash + run: | + tar -xvzf ola-debian-stable-built-source-tree.tar.gz . + rm ola-debian-stable-built-source-tree.tar.gz + - name: Display structure of extracted files + if: env.ACTIONS_STEP_DEBUG == 'true' + run: ls -alR + - name: Update package database + run: apt-get update -y + - name: Install Python + run: apt-get -y install python3 python-is-python3 + - name: Enable Problem Matcher for GitHub annotations + run: echo "::add-matcher::.github/problem-matcher-lint-check-licences.json" + - name: Check licenses + shell: bash + run: ./scripts/enforce_licence.py + generic-nolints: + name: Count generic NOLINTs + runs-on: ubuntu-latest + container: debian:stable + needs: build + steps: + - name: Download built source tree archive + uses: actions/download-artifact@v3 + with: + name: ola-debian-stable-built-source-tree + path: . + - name: SHA256 artifact archive + run: sha256sum ola-debian-stable-built-source-tree.tar.gz + - name: Unarchive artifacts and delete archive + shell: bash + run: | + tar -xvzf ola-debian-stable-built-source-tree.tar.gz . + rm ola-debian-stable-built-source-tree.tar.gz + - name: Display structure of extracted files + if: env.ACTIONS_STEP_DEBUG == 'true' + run: ls -alR + - name: Enable Problem Matcher for GitHub annotations + run: echo "::add-matcher::.github/problem-matcher-lint-generic-nolints.json" + - name: Count the number of generic NOLINTs + shell: bash + run: ./scripts/count_generic_nolints.sh + cpplint: + name: cpplint + runs-on: ubuntu-latest + container: debian:stable + needs: build + steps: + - name: Download built source tree archive + uses: actions/download-artifact@v3 + with: + name: ola-debian-stable-built-source-tree + path: . + - name: SHA256 artifact archive + run: sha256sum ola-debian-stable-built-source-tree.tar.gz + - name: Unarchive artifacts and delete archive + shell: bash + run: | + tar -xvzf ola-debian-stable-built-source-tree.tar.gz . + rm ola-debian-stable-built-source-tree.tar.gz + - name: Display structure of extracted files + if: env.ACTIONS_STEP_DEBUG == 'true' + run: ls -alR + - name: Update package database + run: apt-get update -y + - name: Install build tools + shell: bash + run: | + apt-get -y install pkg-config libtool autoconf \ + automake g++ bison flex make bash-completion dh-autoreconf \ + debhelper devscripts wget python3-pip + - name: Install Python lint tools + run: python3 -m pip install --no-input cpplint flake8 + - name: Install build dependencies + shell: bash + run: | + apt-get -y install libcppunit-dev uuid-dev libncurses5-dev \ + libmicrohttpd-dev protobuf-compiler python3-protobuf \ + libprotobuf-dev libprotoc-dev zlib1g-dev libftdi-dev \ + libusb-1.0-0-dev liblo-dev libavahi-client-dev python3-numpy + - name: Enable Problem Matcher for GitHub annotations + run: echo "::add-matcher::.github/problem-matcher-lint-cpplint.json" + - name: cpplint + run: make cpplint VERBOSE=1 + flake8: + name: flake8 + runs-on: ubuntu-latest + container: debian:stable + needs: build + steps: + - name: Download built source tree archive + uses: actions/download-artifact@v3 + with: + name: ola-debian-stable-built-source-tree + path: . + - name: SHA256 artifact archive + run: sha256sum ola-debian-stable-built-source-tree.tar.gz + - name: Unarchive artifacts and delete archive + shell: bash + run: | + tar -xvzf ola-debian-stable-built-source-tree.tar.gz . + rm ola-debian-stable-built-source-tree.tar.gz + - name: Display structure of extracted files + if: env.ACTIONS_STEP_DEBUG == 'true' + run: ls -alR + - name: Update package database + run: apt-get update -y + - name: Install build tools + shell: bash + run: | + apt-get -y install pkg-config libtool autoconf \ + automake g++ bison flex make bash-completion dh-autoreconf \ + debhelper devscripts wget python3-pip + - name: Install Python lint tools + run: python3 -m pip install --no-input cpplint flake8 + - name: Setup flake8 annotations + uses: rbialon/flake8-annotations@v1 + - name: Install build dependencies + shell: bash + run: | + apt-get -y install libcppunit-dev uuid-dev libncurses5-dev \ + libmicrohttpd-dev protobuf-compiler python3-protobuf \ + libprotobuf-dev libprotoc-dev zlib1g-dev libftdi-dev \ + libusb-1.0-0-dev liblo-dev libavahi-client-dev python3-numpy + - name: flake8 + run: make flake8 VERBOSE=1 diff --git a/AUTHORS b/AUTHORS index 58c01bd6f3..6466a47c38 100644 --- a/AUTHORS +++ b/AUTHORS @@ -21,7 +21,7 @@ Contributors: Masaki Muranaka, various patches Nicolas, for the win32 port of libartnet Nils Van Zuijlen, typos and python lib small update - Perry Naseck, EPoll delay fix + Perry Naseck, EPoll delay fix, various GitHub Actions CI workflows Peter Newman, MilInst Plugin, Scanlime Fadecandy support and numerous fixes Ravindra Nath Kakarla, RDM Test Server (Google Summer of Code 2012) Rowan Maclachlan (hippy) for various changes diff --git a/Makefile.am b/Makefile.am index 130bbb001a..f488cf71a1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -240,13 +240,12 @@ lint: flake8 cpplint .PHONY : flake8 flake8: Makefile.am if FOUND_FLAKE8 - flake8 + flake8 --statistics --count else $(error flake8 not found. Install flake8 and re-run configure.) endif # cpplint linter -CPP_LINT_URL = "https://raw.githubusercontent.com/google/styleguide/gh-pages/cpplint/cpplint.py" CPP_LINT_FILTER = "-legal/copyright,-readability/streams,-runtime/arrays" CPP_LINT_FILES = $(shell find . \( -name "*.h" -or -name "*.cpp" \) -and ! \( \ -wholename "./common/protocol/Ola.pb.*" -or \ @@ -258,16 +257,9 @@ CPP_LINT_FILES = $(shell find . \( -name "*.h" -or -name "*.cpp" \) -and ! \( \ -wholename "./tools/ola_trigger/config.tab.*" -or \ -wholename "./tools/ola_trigger/lex.yy.cpp" \) | xargs) .PHONY : cpplint -if FOUND_CPPLINT cpplint: Makefile.am - cpplint.py --filter=$(CPP_LINT_FILTER) $(CPP_LINT_FILES) +if FOUND_CPPLINT + cpplint --filter=$(CPP_LINT_FILTER) $(CPP_LINT_FILES) else -cpplint: Makefile.am cpplint.py - ./cpplint.py --filter=$(CPP_LINT_FILTER) $(CPP_LINT_FILES) -endif - -if !FOUND_CPPLINT -cpplint.py: - wget -O cpplint.py $(CPP_LINT_URL) - chmod u+x cpplint.py + $(error cpplint not found. Install forked cpplint (e.g. via pip for the latest version) and re-run configure.) endif diff --git a/configure.ac b/configure.ac index 48a9620c59..8315010b98 100644 --- a/configure.ac +++ b/configure.ac @@ -959,7 +959,7 @@ fi # Linters AC_CHECK_PROG([flake8],[flake8],[yes],[no]) AM_CONDITIONAL([FOUND_FLAKE8], [test "x$flake8" = xyes]) -AC_CHECK_PROG([cpplint],[cpplint.py],[yes],[no]) +AC_CHECK_PROG([cpplint],[cpplint],[yes],[no]) AM_CONDITIONAL([FOUND_CPPLINT], [test "x$cpplint" = xyes]) # Output diff --git a/protoc/StrUtil.cpp b/protoc/StrUtil.cpp index d300b71e86..6c6c338ee2 100644 --- a/protoc/StrUtil.cpp +++ b/protoc/StrUtil.cpp @@ -360,39 +360,39 @@ char* FastUInt32ToBufferLeft(uint32_t u, char* buffer) { buffer[0] = ASCII_digits[0]; buffer[1] = ASCII_digits[1]; buffer += 2; - sublt100_000_000: + sublt100_000_000: u -= digits * 100000000; // 100,000,000 - lt100_000_000: + lt100_000_000: digits = u / 1000000; // 1,000,000 ASCII_digits = two_ASCII_digits[digits]; buffer[0] = ASCII_digits[0]; buffer[1] = ASCII_digits[1]; buffer += 2; - sublt1_000_000: + sublt1_000_000: u -= digits * 1000000; // 1,000,000 - lt1_000_000: + lt1_000_000: digits = u / 10000; // 10,000 ASCII_digits = two_ASCII_digits[digits]; buffer[0] = ASCII_digits[0]; buffer[1] = ASCII_digits[1]; buffer += 2; - sublt10_000: + sublt10_000: u -= digits * 10000; // 10,000 - lt10_000: + lt10_000: digits = u / 100; ASCII_digits = two_ASCII_digits[digits]; buffer[0] = ASCII_digits[0]; buffer[1] = ASCII_digits[1]; buffer += 2; - sublt100: + sublt100: u -= digits * 100; - lt100: + lt100: digits = u; ASCII_digits = two_ASCII_digits[digits]; buffer[0] = ASCII_digits[0]; buffer[1] = ASCII_digits[1]; buffer += 2; - done: + done: *buffer = 0; return buffer; } diff --git a/scripts/count_generic_nolints.sh b/scripts/count_generic_nolints.sh new file mode 100755 index 0000000000..32379dbf54 --- /dev/null +++ b/scripts/count_generic_nolints.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# count_generic_nolints.sh +# Copyright (C) 2023 Perry Naseck, Peter Newman + +# This script is based on a Travis CI test by Peter Newman +nolints="$(grep -n --exclude "$(basename $0)" -IR NOLINT * | grep -v "NOLINT(" | sed 's/^\(.*\):\([0-9]\+\):\(.*\)$/error:file:\.\/\1:line \2: Generic NOLINT not permitted/g')" +nolints_count="$(grep --exclude "$(basename $0)" -IR NOLINT * | grep -c -v "NOLINT(")" +if [[ $nolints_count -ne 0 ]]; then + # print the output for info + printf "%s\n" "$nolints" + printf "error: Found $nolints_count generic NOLINTs\n" + exit 1 +else + printf "notice: Found $nolints_count generic NOLINTs\n" +fi diff --git a/scripts/enforce_licence.py b/scripts/enforce_licence.py index b15d157681..8fee722964 100755 --- a/scripts/enforce_licence.py +++ b/scripts/enforce_licence.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -232,11 +232,15 @@ def GetDirectoryLicences(root_dir): subdirs[:] = [d for d in subdirs if not d.startswith('.')] if LICENCE_FILE in files: - f = open(os.path.join(dir_name, LICENCE_FILE)) + licence_path = os.path.join(dir_name, LICENCE_FILE) + f = open(licence_path) lines = f.readlines() f.close() licences[dir_name] = TransformLicence(lines) - print('Found LICENCE for directory %s' % dir_name) + rel_licence_path = os.path.relpath(licence_path) + rel_dir_path = os.path.relpath(dir_name) + print('debug:file:%s: Found LICENCE for directory %s' % + (rel_licence_path, rel_dir_path)) # use this licence for all subdirs licence = licences.get(dir_name) @@ -280,8 +284,10 @@ def CheckLicenceForFile(file_name, licence, lang, diff, fix): return 0 f = open(file_name) + rel_path = os.path.relpath(file_name) # + 1 to include the newline to have a complete line header_size = len(licence) + 1 + file_name_line_count = licence.count('\n') + 2 first_line = None if lang == PYTHON: first_line = f.readline() @@ -289,6 +295,8 @@ def CheckLicenceForFile(file_name, licence, lang, diff, fix): # First line isn't a shebang, ignore it. f.seek(0, os.SEEK_SET) first_line = None + else: + file_name_line_count += 1 # strip the trailing newline off as we don't actually want it header = f.read(header_size).rstrip('\n') file_name_line = f.readline() @@ -296,22 +304,24 @@ def CheckLicenceForFile(file_name, licence, lang, diff, fix): if header == licence: expected_line = TransformLine(os.path.basename(file_name), lang) if lang != JS and file_name_line.rstrip('\n') != expected_line: - print("File %s does not have a filename line after the licence; found " - "\"%s\" expected \"%s\"" % - (file_name, file_name_line.rstrip('\n'), expected_line)) + print("error:file:%s:line %s: File does not have a filename line after " + "the licence; found \"%s\" expected \"%s\"" % + (rel_path, file_name_line_count, file_name_line.rstrip('\n'), + expected_line)) return 1 return 0 if fix: - print('Fixing %s' % file_name) + print('notice:file:%s: Fixing file' % rel_path) if lang == PYTHON and first_line is not None: licence = first_line + licence ReplaceHeader(file_name, licence, lang) return 1 else: - print("File %s does not start with \"%s...\"" % ( - file_name, - licence.split('\n')[(0 if (lang == PYTHON) else 1)])) + print("error:file:%s:lines 1-%s: File does not start with, or not exact " + "match of, \"%s...\"" % + (rel_path, file_name_line_count, + licence.split('\n')[(0 if (lang == PYTHON) else 1)])) if diff: d = difflib.Differ() result = list(d.compare(header.splitlines(1), licence.splitlines(1))) @@ -325,10 +335,11 @@ def main(): errors = 0 for dir_name, licence in licences.items(): errors += CheckLicenceForDir(dir_name, licence, diff=diff, fix=fix) - print('Found %d files with incorrect licences' % errors) - if errors > 0: + if (errors > 0): + print('error: Found %d files with incorrect licences' % errors) sys.exit(1) else: + print('notice: Found %d files with incorrect licences' % errors) sys.exit()