diff --git a/.github/workflows/check_base_os.yml b/.github/workflows/check_base_os.yml new file mode 100644 index 000000000000..26dd544ac286 --- /dev/null +++ b/.github/workflows/check_base_os.yml @@ -0,0 +1,61 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +name: 'Check Base OS Consistency' + +on: + pull_request: + paths: + - 'projects/**' + +jobs: + check-consistency: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history to compare with main + + - name: Get changed project directories + id: changed-projects + run: | + # Get the list of changed files compared to the target branch + # and filter for unique directories under 'projects/'. + CHANGED_DIRS=$(git diff --name-only ${{ github.base_ref }} ${{ github.head_ref }} | \ + grep '^projects/' | \ + xargs -n 1 dirname | \ + sort -u) + echo "changed_dirs=${CHANGED_DIRS}" >> $GITHUB_OUTPUT + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: pip install PyYAML + + - name: Check each modified project + if: steps.changed-projects.outputs.changed_dirs != '' + run: | + EXIT_CODE=0 + for project_dir in ${{ steps.changed-projects.outputs.changed_dirs }}; + do + echo "--- Checking ${project_dir} ---" + python3 infra/ci/check_base_os.py "${project_dir}" || EXIT_CODE=$? + done + exit $EXIT_CODE diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index 54d1aa8ef1b3..f00ab7c3c8b4 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -685,12 +685,24 @@ def get_gcb_url(build_id, cloud_project='oss-fuzz'): f'{build_id}?project={cloud_project}') -def get_runner_image_name(test_image_suffix): - """Returns the runner image that should be used. Returns the testing image if - |test_image_suffix|.""" +def get_runner_image_name(test_image_suffix, base_image_tag=None): + """Returns the runner image that should be used. + + Returns the testing image if |test_image_suffix|. + """ image = f'gcr.io/{BASE_IMAGES_PROJECT}/base-runner' + + # For trial builds, the version is embedded in the suffix. if test_image_suffix: image += '-' + test_image_suffix + return image + + # For local/manual runs, the version is passed as a tag. + # Only add a tag if it's specified and not 'legacy', as 'legacy' implies + # 'latest', which is the default behavior. + if base_image_tag and base_image_tag != 'legacy': + image += ':' + base_image_tag + return image diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index e95b5065189e..cfc14e6ac884 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -73,6 +73,7 @@ class Config: testing: bool = False test_image_suffix: Optional[str] = None + base_image_tag: Optional[str] = None repo: Optional[str] = DEFAULT_OSS_FUZZ_REPO branch: Optional[str] = None parallel: bool = False @@ -453,8 +454,10 @@ def get_build_steps_for_project(project, f'--architecture {build.architecture} {project.name}\\n' + '*' * 80) # Test fuzz targets. + runner_image_name = build_lib.get_runner_image_name( + config.test_image_suffix, config.base_image_tag) test_step = { - 'name': build_lib.get_runner_image_name(config.test_image_suffix), + 'name': runner_image_name, 'env': env, 'args': [ 'bash', '-c', diff --git a/infra/build/functions/trial_build.py b/infra/build/functions/trial_build.py index a11280882fa9..849effed8515 100644 --- a/infra/build/functions/trial_build.py +++ b/infra/build/functions/trial_build.py @@ -321,6 +321,7 @@ def _do_test_builds(args, test_image_suffix, end_time, version_tag): config = build_project.Config(testing=True, test_image_suffix=test_image_suffix, + base_image_tag=version_tag, repo=args.repo, branch=args.branch, parallel=False, @@ -432,7 +433,7 @@ def _do_build_type_builds(args, config, credentials, build_type, projects): credentials, build_type.type_name, extra_tags=tags, - timeout=PROJECT_BUILD_TIMEOUT)) + timeout=PROJECT_BUILD_TIMEOUT))['id'] time.sleep(1) # Avoid going over 75 requests per second limit. except Exception as error: # pylint: disable=broad-except # Handle flake. diff --git a/infra/build/functions/trial_build_test.py b/infra/build/functions/trial_build_test.py index 9cd20207b1b9..9b74791c6702 100644 --- a/infra/build/functions/trial_build_test.py +++ b/infra/build/functions/trial_build_test.py @@ -73,7 +73,7 @@ def test_build_steps_correct(self, mock_gcb_build_and_push_images, del mock_wait_on_builds self.maxDiff = None # pylint: disable=invalid-name build_id = 1 - mock_run_build.return_value = build_id + mock_run_build.return_value = {'id': build_id} branch_name = 'mybranch' project = 'skcms' args = [ diff --git a/infra/ci/check_base_os.py b/infra/ci/check_base_os.py new file mode 100644 index 000000000000..90f2ccfb2e28 --- /dev/null +++ b/infra/ci/check_base_os.py @@ -0,0 +1,111 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +""" +A CI script to ensure that the base OS version specified in a project's +project.yaml file matches the FROM line in its Dockerfile. +""" + +import os +import sys +import yaml + +# Defines the base OS versions that are currently supported for use in project.yaml. +# For now, only 'legacy' is permitted. This list will be expanded as new +# base images are rolled out. +SUPPORTED_VERSIONS = [ + 'legacy', + # 'ubuntu-20-04', + # 'ubuntu-24-04', +] + +# A map from the base_os_version in project.yaml to the expected Dockerfile +# FROM tag. +BASE_OS_TO_DOCKER_TAG = { + 'legacy': 'latest', + 'ubuntu-20-04': 'ubuntu-20-04', + 'ubuntu-24-04': 'ubuntu-24-04', +} + + +def main(): + """Checks the Dockerfile FROM tag against the project's base_os_version.""" + if len(sys.argv) < 2: + print(f'Usage: {sys.argv[0]} ', file=sys.stderr) + return 1 + + project_path = sys.argv[1] + project_yaml_path = os.path.join(project_path, 'project.yaml') + dockerfile_path = os.path.join(project_path, 'Dockerfile') + + # 1. Get the base_os_version from project.yaml, defaulting to 'legacy'. + base_os_version = 'legacy' + if os.path.exists(project_yaml_path): + with open(project_yaml_path) as f: + config = yaml.safe_load(f) + if config and 'base_os_version' in config: + base_os_version = config['base_os_version'] + + # 2. Validate that the version is currently supported. + if base_os_version not in SUPPORTED_VERSIONS: + print( + f'Error: base_os_version "{base_os_version}" is not yet supported. ' + f'The currently supported versions are: "{", ".join(SUPPORTED_VERSIONS)}"', + file=sys.stderr) + return 1 + + # 3. Get the expected Dockerfile tag from our mapping. + expected_tag = BASE_OS_TO_DOCKER_TAG[base_os_version] + + # 4. Read the Dockerfile and find the tag in the FROM line. + if not os.path.exists(dockerfile_path): + print(f'Error: Dockerfile not found at {dockerfile_path}', file=sys.stderr) + return 1 + + dockerfile_tag = '' + with open(dockerfile_path) as f: + for line in f: + if line.strip().startswith('FROM'): + try: + if ':' not in line: + print( + f'Error: Malformed FROM line in Dockerfile (missing tag): {line.strip()}', + file=sys.stderr) + return 1 + dockerfile_tag = line.split(':')[1].strip() + except IndexError: + print(f'Error: Could not parse tag from Dockerfile FROM line: {line}', + file=sys.stderr) + return 1 + break + + # 5. Compare and report. + if dockerfile_tag != expected_tag: + print( + f'Error: Mismatch found in {project_path}.\n' + f' - project.yaml (base_os_version): "{base_os_version}" (expects Dockerfile tag "{expected_tag}")\n' + f' - Dockerfile FROM tag: "{dockerfile_tag}"\n' + f'Please align the Dockerfile\'s FROM line to use the tag "{expected_tag}".', + file=sys.stderr) + return 1 + + print( + f'Success: {project_path} is consistent (base_os_version: "{base_os_version}", Dockerfile tag: "{dockerfile_tag}").' + ) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/infra/helper.py b/infra/helper.py index 17e0b1c92796..1f03effb1c42 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -40,6 +40,23 @@ BASE_RUNNER_IMAGE = 'gcr.io/oss-fuzz-base/base-runner' + +def _get_base_runner_image(args, debug=False): + """Returns the base runner image to use.""" + image = BASE_RUNNER_IMAGE + if debug: + image += '-debug' + + tag = 'latest' + if hasattr(args, 'base_image_tag') and args.base_image_tag: + tag = args.base_image_tag + elif hasattr(args, 'project') and args.project: + if args.project.base_os_version != 'legacy': + tag = args.project.base_os_version + + return f'{image}:{tag}' + + BASE_IMAGES = { 'generic': [ 'gcr.io/oss-fuzz-base/base-image', @@ -73,6 +90,7 @@ LANGUAGE_REGEX = re.compile(r'[^\s]+') PROJECT_LANGUAGE_REGEX = re.compile(r'\s*language\s*:\s*([^\s]+)') +BASE_OS_VERSION_REGEX = re.compile(r'\s*base_os_version\s*:\s*([^\s]+)') WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') @@ -152,6 +170,23 @@ def language(self): logger.warning('Language not specified in project.yaml. Assuming c++.') return constants.DEFAULT_LANGUAGE + @property + def base_os_version(self): + """Returns the project's base OS version.""" + project_yaml_path = os.path.join(self.build_integration_path, + 'project.yaml') + if not os.path.exists(project_yaml_path): + return 'legacy' + + with open(project_yaml_path) as file_handle: + content = file_handle.read() + for line in content.splitlines(): + match = BASE_OS_VERSION_REGEX.match(line) + if match: + return match.group(1) + + return 'legacy' + @property def coverage_extra_args(self): """Returns project coverage extra args.""" @@ -364,6 +399,7 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals _add_engine_args(check_build_parser, choices=constants.ENGINES) _add_sanitizer_args(check_build_parser, choices=constants.SANITIZERS) _add_environment_args(check_build_parser) + _add_base_image_tag_args(check_build_parser) check_build_parser.add_argument('project', help='name of the project or path (external)') check_build_parser.add_argument('fuzzer_name', @@ -399,6 +435,7 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals _add_engine_args(run_fuzzer_parser) _add_sanitizer_args(run_fuzzer_parser) _add_environment_args(run_fuzzer_parser) + _add_base_image_tag_args(run_fuzzer_parser) _add_external_project_args(run_fuzzer_parser) run_fuzzer_parser.add_argument( '--corpus-dir', help='directory to store corpus for the fuzz target') @@ -465,6 +502,7 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals nargs='*') _add_external_project_args(coverage_parser) _add_architecture_args(coverage_parser) + _add_base_image_tag_args(coverage_parser) introspector_parser = subparsers.add_parser( 'introspector', @@ -524,6 +562,7 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals _add_environment_args(reproduce_parser) _add_external_project_args(reproduce_parser) _add_architecture_args(reproduce_parser) + _add_base_image_tag_args(reproduce_parser) shell_parser = subparsers.add_parser( 'shell', help='Run /bin/bash within the builder container.') @@ -537,6 +576,7 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals _add_sanitizer_args(shell_parser) _add_environment_args(shell_parser) _add_external_project_args(shell_parser) + _add_base_image_tag_args(shell_parser) run_clusterfuzzlite_parser = subparsers.add_parser( 'run_clusterfuzzlite', help='Run ClusterFuzzLite on a project.') @@ -582,12 +622,12 @@ def check_project_exists(project): return False -def _check_fuzzer_exists(project, fuzzer_name, architecture='x86_64'): +def _check_fuzzer_exists(project, fuzzer_name, args, architecture='x86_64'): """Checks if a fuzzer exists.""" platform = 'linux/arm64' if architecture == 'aarch64' else 'linux/amd64' command = ['docker', 'run', '--rm', '--platform', platform] command.extend(['-v', '%s:/out' % project.out]) - command.append(BASE_RUNNER_IMAGE) + command.append(_get_base_runner_image(args)) command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name]) @@ -668,6 +708,12 @@ def _add_environment_args(parser): help="set environment variable e.g. VAR=value") +def _add_base_image_tag_args(parser): + """Adds base image tag arg.""" + parser.add_argument('--base-image-tag', + help='The tag of the base-runner image to use.') + + def build_image_impl(project, cache=True, pull=False, architecture='x86_64'): """Builds image.""" image_name = project.name @@ -1064,11 +1110,13 @@ def _add_oss_fuzz_ci_if_needed(env): def check_build(args): """Checks that fuzzers in the container execute without errors.""" + # Access the property to trigger validation early. + _ = args.project.base_os_version if not check_project_exists(args.project): return False if (args.fuzzer_name and not _check_fuzzer_exists( - args.project, args.fuzzer_name, args.architecture)): + args.project, args.fuzzer_name, args, args.architecture)): return False env = [ @@ -1083,7 +1131,8 @@ def check_build(args): env += args.e run_args = _env_to_docker_args(env) + [ - '-v', f'{args.project.out}:/out', '-t', BASE_RUNNER_IMAGE + '-v', f'{args.project.out}:/out', '-t', + _get_base_runner_image(args) ] if args.fuzzer_name: @@ -1319,7 +1368,7 @@ def coverage(args): # pylint: disable=too-many-branches '-v', '%s:/out' % args.project.out, '-t', - BASE_RUNNER_IMAGE, + _get_base_runner_image(args), ]) run_args.append('coverage') @@ -1447,7 +1496,7 @@ def run_fuzzer(args): if not check_project_exists(args.project): return False - if not _check_fuzzer_exists(args.project, args.fuzzer_name, + if not _check_fuzzer_exists(args.project, args.fuzzer_name, args, args.architecture): return False @@ -1478,7 +1527,7 @@ def run_fuzzer(args): '-v', '%s:/out' % args.project.out, '-t', - BASE_RUNNER_IMAGE, + _get_base_runner_image(args), 'run_fuzzer', args.fuzzer_name, ] + args.fuzzer_args) @@ -1569,7 +1618,8 @@ def fuzzbench_measure(args): def reproduce(args): """Reproduces a specific test case from a specific project.""" return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e, - args.fuzzer_args, args.testcase_path, args.architecture) + args.fuzzer_args, args.testcase_path, args, + args.architecture) def reproduce_impl( # pylint: disable=too-many-arguments @@ -1579,29 +1629,30 @@ def reproduce_impl( # pylint: disable=too-many-arguments env_to_add, fuzzer_args, testcase_path, + args, architecture='x86_64', run_function=docker_run, err_result=False): - """Reproduces a testcase in the container.""" + """Reproduces a specific test case.""" if not check_project_exists(project): return err_result - if not _check_fuzzer_exists(project, fuzzer_name, architecture): + if not _check_fuzzer_exists(project, fuzzer_name, args, architecture): return err_result debugger = '' env = ['HELPER=True', 'ARCHITECTURE=' + architecture] - image_name = 'base-runner' + use_debug_image = bool(valgrind) + image_name = _get_base_runner_image(args, debug=use_debug_image) if valgrind: debugger = 'valgrind --tool=memcheck --track-origins=yes --leak-check=full' if debugger: - image_name = 'base-runner-debug' env += ['DEBUGGER=' + debugger] if env_to_add: - env += env_to_add + env.extend(env_to_add) run_args = _env_to_docker_args(env) + [ '-v', @@ -1609,13 +1660,12 @@ def reproduce_impl( # pylint: disable=too-many-arguments '-v', '%s:/testcase' % _get_absolute_path(testcase_path), '-t', - 'gcr.io/oss-fuzz-base/%s' % image_name, + image_name, 'reproduce', fuzzer_name, '-runs=100', ] + fuzzer_args - - return run_function(run_args, architecture=architecture) + return run_function(run_args, err_result) def _validate_project_name(project_name): @@ -1779,6 +1829,8 @@ def index(args): def shell(args): """Runs a shell within a docker image.""" + # Access the property to trigger validation early. + _ = args.project.base_os_version if not build_image_impl(args.project): return False