From b0604faa76e479f8cb09108d31c218d6d0c5e3b3 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Thu, 13 Aug 2026 16:02:59 +0000 Subject: [PATCH 1/2] fix: correct third-party license inventory Signed-off-by: Matt Kornfield --- third_party/licenses.jsonl | 14 ++-- .../license/overrides.yaml | 11 +-- .../tests/license/test_license_utils.py | 70 +++++++++++++++++++ 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/third_party/licenses.jsonl b/third_party/licenses.jsonl index 4960d1b3e11..713153478ee 100644 --- a/third_party/licenses.jsonl +++ b/third_party/licenses.jsonl @@ -30,7 +30,7 @@ {"name": "bracex", "license": "MIT", "compatible": true} {"name": "cachetools", "license": "MIT", "compatible": true} {"name": "caio", "license": "APACHE-2.0", "compatible": true} -{"name": "certifi", "license": "LGPL", "compatible": true} +{"name": "certifi", "license": "MPL-2.0", "compatible": true} {"name": "cffi", "license": "MIT", "compatible": true} {"name": "chardet", "license": "LGPL-2.1-OR-LATER", "compatible": true} {"name": "charset-normalizer", "license": "MIT", "compatible": true} @@ -141,7 +141,7 @@ {"name": "google.golang.org/protobuf", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "googleapis-common-protos", "license": "APACHE-2.0", "compatible": true} {"name": "gopkg.in/yaml.v3", "license": "MIT", "compatible": true} -{"name": "greenlet", "license": "MIT", "compatible": true} +{"name": "greenlet", "license": "MIT AND PSF-2.0", "compatible": true} {"name": "griffelib", "license": "ISC", "compatible": true} {"name": "grpcio", "license": "APACHE-2.0", "compatible": true} {"name": "gunicorn", "license": "MIT", "compatible": true} @@ -196,7 +196,7 @@ {"name": "langchain-mcp-adapters", "license": "MIT", "compatible": true} {"name": "langchain-milvus", "license": "MIT", "compatible": true} {"name": "langchain-nvidia-ai-endpoints", "license": "MIT", "compatible": true} -{"name": "langchain-oci", "license": "MIT", "compatible": true} +{"name": "langchain-oci", "license": "UPL-1.0", "compatible": true} {"name": "langchain-openai", "license": "MIT", "compatible": true} {"name": "langchain-protocol", "license": "MIT", "compatible": true} {"name": "langchain-text-splitters", "license": "MIT", "compatible": true} @@ -281,7 +281,7 @@ {"name": "packaging", "license": "APACHE-2.0", "compatible": true} {"name": "pandas", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "pathable", "license": "APACHE-2.0", "compatible": true} -{"name": "pathspec", "license": "LGPL", "compatible": true} +{"name": "pathspec", "license": "MPL-2.0", "compatible": true} {"name": "pillow", "license": "MIT-CMU", "compatible": true} {"name": "pip", "license": "MIT", "compatible": true} {"name": "pkce", "license": "MIT", "compatible": true} @@ -298,7 +298,7 @@ {"name": "propcache", "license": "APACHE-2.0", "compatible": true} {"name": "protobuf", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "psutil", "license": "BSD-3-CLAUSE", "compatible": true} -{"name": "psycopg2-binary", "license": "LGPL", "compatible": true} +{"name": "psycopg2-binary", "license": "LGPL-3.0-OR-LATER WITH OPENVPN-OPENSSL-EXCEPTION", "compatible": true} {"name": "py-key-value-aio", "license": "APACHE-2.0", "compatible": true} {"name": "py-rust-stemmers", "license": "MIT", "compatible": true} {"name": "pyarrow", "license": "APACHE-2.0", "compatible": true} @@ -325,7 +325,7 @@ {"name": "ragas", "license": "APACHE-2.0", "compatible": true} {"name": "realtime", "license": "MIT", "compatible": true} {"name": "referencing", "license": "MIT", "compatible": true} -{"name": "regex", "license": "APACHE-2.0", "compatible": true} +{"name": "regex", "license": "APACHE-2.0 AND CNRI-PYTHON", "compatible": true} {"name": "requests", "license": "APACHE-2.0", "compatible": true} {"name": "requests-oauthlib", "license": "ISC", "compatible": true} {"name": "requests-toolbelt", "license": "APACHE-2.0", "compatible": true} @@ -379,7 +379,7 @@ {"name": "tomli-w", "license": "MIT", "compatible": true} {"name": "tomlkit", "license": "MIT", "compatible": true} {"name": "tornado", "license": "APACHE-2.0", "compatible": true} -{"name": "tqdm", "license": "MIT", "compatible": true} +{"name": "tqdm", "license": "MIT AND MPL-2.0", "compatible": true} {"name": "transformers", "license": "APACHE-2.0", "compatible": true} {"name": "typer", "license": "MIT", "compatible": true} {"name": "types-aioboto3", "license": "MIT", "compatible": true} diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml index 2aaf3b1ea7b..c71e5ece501 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml @@ -103,7 +103,7 @@ overrides: amqp: BSD-3-Clause # AMQP protocol implementation billiard: BSD-3-Clause # Multiprocessing pool (used by Celery) flower: BSD-3-Clause # Celery monitoring tool - psycopg2-binary: LGPL # PostgreSQL adapter + psycopg2-binary: LGPL-3.0-or-later WITH openvpn-openssl-exception # PostgreSQL adapter; reviewed upstream exception expression pyarrow: Apache-2.0 # Apache Arrow Python bindings vine: BSD-3-Clause # Promises/futures library for Celery @@ -169,10 +169,12 @@ overrides: # NLP / AI Evaluation fastembed: Apache-2.0 # Fast text embedding - langchain-oci: MIT # LangChain Oracle Cloud Infrastructure integration + greenlet: MIT AND PSF-2.0 # Reviewed list matches OSV license expression + langchain-oci: UPL-1.0 # Oracle LangChain integration; upstream Oracle project license ragas: Apache-2.0 # RAG evaluation framework range-regex: BSD-3-Clause # Regex for ranges ratelimit: MIT # API rate limiting + regex: Apache-2.0 AND CNRI-Python # Reviewed list matches OSV license expression rouge-score: Apache-2.0 # ROUGE metric for text summarization sacrebleu: Apache-2.0 # Machine translation metrics @@ -228,16 +230,17 @@ overrides: # Licenses not automatically approved aiohappyeyeballs: PSF-2.0 - certifi: LGPL + certifi: MPL-2.0 datetime: ZPL-2.0 email-validator: Unlicense filelock: Unlicense nvidia-cutlass-dsl: NVIDIA-Software-End-User-License-Agreement - pathspec: LGPL + pathspec: MPL-2.0 pillow: MIT-CMU pycountry: LGPL-2.1 python-gitlab: LGPL-3.0 supervisor: BSD-3-Clause + tqdm: MIT AND MPL-2.0 typing-extensions: PSF-2.0 unsloth-zoo: LGPL-3.0-or-later zope-interface: ZPL-2.0 diff --git a/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py b/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py index 78db988fabb..ac50cc06f49 100644 --- a/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py +++ b/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py @@ -256,6 +256,76 @@ def test_format_licenses_fills_missing_osv_package_from_overrides(self, tmp_path '{"name": "cloudpickle", "license": "BSD-3-CLAUSE", "compatible": true}' ) + def test_format_licenses_uses_reviewed_license_overrides(self, tmp_path): + """Reviewed overrides keep the published inventory aligned with OSRB-reviewed data.""" + from nemo_platform_sdk_tools.license.generator import format_licenses + + license_dir = tmp_path / "third_party" + license_dir.mkdir() + osv_json = license_dir / "osv-licenses.json" + packages = [ + ("certifi", "2026.7.22", ["MPL-2.0"]), + ("greenlet", "3.5.4", ["MIT AND PSF-2.0"]), + ("langchain-oci", "0.3.1", ["UPL-1.0"]), + ("pathspec", "1.1.1", ["MPL-2.0"]), + ("psycopg2-binary", "2.9.12", ["non-standard"]), + ("regex", "2026.7.19", ["Apache-2.0 AND CNRI-Python"]), + ("tqdm", "4.70.0", ["MIT AND MPL-2.0"]), + ] + osv_json.write_text( + json.dumps( + { + "results": [ + { + "packages": [ + { + "package": {"name": name, "version": version, "ecosystem": "PyPI"}, + "licenses": licenses, + } + for name, version, licenses in packages + ] + } + ] + } + ), + encoding="utf-8", + ) + (license_dir / "requirements-main.txt").write_text( + "\n".join(f"{name}=={version}" for name, version, _ in packages), + encoding="utf-8", + ) + overrides_file = tmp_path / "overrides.yaml" + overrides_file.write_text( + "\n".join( + [ + "overrides:", + " certifi: MPL-2.0", + " greenlet: MIT AND PSF-2.0", + " langchain-oci: UPL-1.0", + " pathspec: MPL-2.0", + " psycopg2-binary: LGPL-3.0-or-later WITH openvpn-openssl-exception", + " regex: Apache-2.0 AND CNRI-Python", + " tqdm: MIT AND MPL-2.0", + ] + ), + encoding="utf-8", + ) + output_file = license_dir / "licenses.jsonl" + + format_licenses(osv_json, output_file, overrides_file, format_type="jsonl") + + records = [json.loads(line) for line in output_file.read_text(encoding="utf-8").splitlines()] + assert {record["name"]: record["license"] for record in records} == { + "certifi": "MPL-2.0", + "greenlet": "MIT AND PSF-2.0", + "langchain-oci": "UPL-1.0", + "pathspec": "MPL-2.0", + "psycopg2-binary": "LGPL-3.0-OR-LATER WITH OPENVPN-OPENSSL-EXCEPTION", + "regex": "APACHE-2.0 AND CNRI-PYTHON", + "tqdm": "MIT AND MPL-2.0", + } + assert all(record["compatible"] is True for record in records) + def test_format_licenses_deduplicates_osv_and_override_fallback_rows(self, tmp_path): """Reviewed override fallback rows should not duplicate packages already emitted by OSV.""" from nemo_platform_sdk_tools.license.generator import format_licenses From 67a976b69b809c0f64ccd00a975c000538c891fc Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Thu, 13 Aug 2026 16:07:56 +0000 Subject: [PATCH 2/2] test: remove circular license inventory test Signed-off-by: Matt Kornfield --- .../tests/license/test_license_utils.py | 70 ------------------- 1 file changed, 70 deletions(-) diff --git a/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py b/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py index ac50cc06f49..78db988fabb 100644 --- a/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py +++ b/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py @@ -256,76 +256,6 @@ def test_format_licenses_fills_missing_osv_package_from_overrides(self, tmp_path '{"name": "cloudpickle", "license": "BSD-3-CLAUSE", "compatible": true}' ) - def test_format_licenses_uses_reviewed_license_overrides(self, tmp_path): - """Reviewed overrides keep the published inventory aligned with OSRB-reviewed data.""" - from nemo_platform_sdk_tools.license.generator import format_licenses - - license_dir = tmp_path / "third_party" - license_dir.mkdir() - osv_json = license_dir / "osv-licenses.json" - packages = [ - ("certifi", "2026.7.22", ["MPL-2.0"]), - ("greenlet", "3.5.4", ["MIT AND PSF-2.0"]), - ("langchain-oci", "0.3.1", ["UPL-1.0"]), - ("pathspec", "1.1.1", ["MPL-2.0"]), - ("psycopg2-binary", "2.9.12", ["non-standard"]), - ("regex", "2026.7.19", ["Apache-2.0 AND CNRI-Python"]), - ("tqdm", "4.70.0", ["MIT AND MPL-2.0"]), - ] - osv_json.write_text( - json.dumps( - { - "results": [ - { - "packages": [ - { - "package": {"name": name, "version": version, "ecosystem": "PyPI"}, - "licenses": licenses, - } - for name, version, licenses in packages - ] - } - ] - } - ), - encoding="utf-8", - ) - (license_dir / "requirements-main.txt").write_text( - "\n".join(f"{name}=={version}" for name, version, _ in packages), - encoding="utf-8", - ) - overrides_file = tmp_path / "overrides.yaml" - overrides_file.write_text( - "\n".join( - [ - "overrides:", - " certifi: MPL-2.0", - " greenlet: MIT AND PSF-2.0", - " langchain-oci: UPL-1.0", - " pathspec: MPL-2.0", - " psycopg2-binary: LGPL-3.0-or-later WITH openvpn-openssl-exception", - " regex: Apache-2.0 AND CNRI-Python", - " tqdm: MIT AND MPL-2.0", - ] - ), - encoding="utf-8", - ) - output_file = license_dir / "licenses.jsonl" - - format_licenses(osv_json, output_file, overrides_file, format_type="jsonl") - - records = [json.loads(line) for line in output_file.read_text(encoding="utf-8").splitlines()] - assert {record["name"]: record["license"] for record in records} == { - "certifi": "MPL-2.0", - "greenlet": "MIT AND PSF-2.0", - "langchain-oci": "UPL-1.0", - "pathspec": "MPL-2.0", - "psycopg2-binary": "LGPL-3.0-OR-LATER WITH OPENVPN-OPENSSL-EXCEPTION", - "regex": "APACHE-2.0 AND CNRI-PYTHON", - "tqdm": "MIT AND MPL-2.0", - } - assert all(record["compatible"] is True for record in records) - def test_format_licenses_deduplicates_osv_and_override_fallback_rows(self, tmp_path): """Reviewed override fallback rows should not duplicate packages already emitted by OSV.""" from nemo_platform_sdk_tools.license.generator import format_licenses