diff --git a/ci/build_test_release.groovy b/ci/build_test_release.groovy new file mode 100644 index 0000000000..dc479ecdc5 --- /dev/null +++ b/ci/build_test_release.groovy @@ -0,0 +1,127 @@ +pipeline { + options { + timeout(time: 2, unit: 'HOURS') + } + agent { + label 'win_ovms' + } + environment { + BDBA_CREDS = credentials('BDBA_KEY') + } + stages { + stage ("Build and test windows") { + when { expression { env.PACKAGE_URL == "" } } + steps { + script { + echo "JOB_BASE_NAME: ${env.JOB_BASE_NAME}" + echo "WORKSPACE: ${env.WORKSPACE}" + echo "OVMS_PYTHON_ENABLED: ${env.OVMS_PYTHON_ENABLED}" + def windows = load 'ci/loadWin.groovy' + if (windows != null) { + try { + windows.setup_bazel_remote_cache() + windows.install_dependencies() + windows.clean() + windows.build() + windows.unit_test() + windows.check_tests() + def safeBranchName = env.BRANCH_NAME.replaceAll('/', '_') + def python_presence = "" + if (env.OVMS_PYTHON_ENABLED == "1") { + python_presence = "with_python" + } else { + python_presence = "without_python" + } + bat(returnStatus:true, script: "ECHO F | xcopy /Y /E ${env.WORKSPACE}\\dist\\windows\\ovms.zip \\\\${env.OV_SHARE_05_IP}\\data\\cv_bench_cache\\OVMS_do_not_remove\\ovms-windows-${python_presence}-${safeBranchName}-latest.zip") + } finally { + windows.archive_build_artifacts() + windows.archive_test_artifacts() + } + } else { + error "Cannot load ci/loadWin.groovy file." + } + } + } + } + stage ("Pull files"){ + when { expression { env.PACKAGE_URL != "" } } + steps{ + script { + def windows = load 'ci/loadWin.groovy' + if (windows != null) { + try { + windows.download_package() + } finally { + echo "Pull files finished" + } + } else { + error "Cannot load ci/loadWin.groovy file." + } + } + } + } + stage ("BDBA scans"){ + when { expression { env.BDBA_SCAN == "true" } } + steps { + script { + def windows = load 'ci/loadWin.groovy' + if (windows != null) { + try { + windows.clone_sdl_repo() + windows.clone_bdba_repo() + windows.bdba() + def logFile = "${env.WORKSPACE}\\win_bdba.log" + def lastLine = bat(script: "powershell -Command \"Get-Content -Path '${logFile}' | Select-Object -Last 1\"", returnStdout: true).trim() + if (!lastLine.contains("Found 0 vulnerabilities")) { + unstable(message: lastLine) + } + } finally { + windows.archive_bdba_reports() + } + } else { + error "Cannot load ci/loadWin.groovy file." + } + } + } + } + stage ("Signing files"){ + when { expression { env.SIGN_FILES == "true" } } + steps { + echo "OVMS_PYTHON_ENABLED: ${env.OVMS_PYTHON_ENABLED}" + withCredentials([ + usernamePassword( + credentialsId: 'PRERELEASE_SIGN', + usernameVariable: 'PRERELEASE_USER', + passwordVariable: 'PRERELEASE_PASS'), + usernamePassword( + credentialsId: 'RELEASE_SIGN', + usernameVariable: 'RELEASE_USER', + passwordVariable: 'RELEASE_PASS'), + ]) { + script { + if (env.RELEASE_TYPE == "RELEASE") { + env.SIGNING_USER = env.RELEASE_USER + env.OVMS_PASS = env.RELEASE_PASS + } else if (env.RELEASE_TYPE == "PRE-RELEASE") { + env.SIGNING_USER = env.PRERELEASE_USER + env.OVMS_PASS = env.PRERELEASE_PASS + } else { + error "Unknown RELEASE_TYPE: ${env.RELEASE_TYPE}" + } + def windows = load 'ci/loadWin.groovy' + if (windows != null) { + try { + windows.clone_sdl_repo() + windows.sign() + } finally { + windows.archive_sign_results() + } + } else { + error "Cannot load ci/loadWin.groovy file." + } + } + } + } + } + } +} diff --git a/ci/lib_search.py b/ci/lib_search.py index 202ba24240..d8490f6dde 100644 --- a/ci/lib_search.py +++ b/ci/lib_search.py @@ -149,6 +149,8 @@ def check_dir(start_dir): "lib_custom_nodes_files", "spelling-whitelist.txt", "results.txt", + "windows_bdba.bat", + "windows_sign.bat", ] exclude_directories = ['/dist/', 'release_files/thirdparty-licenses', 'extras/chat_template_examples'] @@ -242,6 +244,8 @@ def check_func(start_dir): "internal_tests", 'cleanup_jenkins.bat', ".bazelversion", + "windows_bdba.bat", + "windows_sign.bat", ] exclude_directories = ['/dist/'] diff --git a/ci/loadWin.groovy b/ci/loadWin.groovy index 905f306a95..29b93e1df0 100644 --- a/ci/loadWin.groovy +++ b/ci/loadWin.groovy @@ -48,7 +48,7 @@ def cleanup_directories() { println "Deleting: " + pathToDelete status = bat(returnStatus: true, script: 'rmdir /s /q ' + pathToDelete) if (status != 0) { - error "Error: Deleting directory ${pathToDelete} failed: ${status}. Check piepeline.log for details." + error "Error: Deleting directory ${pathToDelete} failed: ${status}. Check pipeline.log for details." } else { echo "Deleting directory ${pathToDelete} successful." } @@ -90,7 +90,7 @@ def deleteOldDirectories() { println "Deleting: " + pathToDelete status = bat(returnStatus: true, script: 'rmdir /s /q ' + pathToDelete) if (status != 0) { - error "Error: Deleting directory ${pathToDelete} failed: ${status}. Check piepeline.log for details." + error "Error: Deleting directory ${pathToDelete} failed: ${status}. Check pipeline.log for details." } else { echo "Deleting directory ${pathToDelete} successful." } @@ -110,6 +110,14 @@ def install_dependencies() { def clean() { def output1 = bat(returnStdout: true, script: 'windows_clean_build.bat ' + get_short_bazel_path() + ' ' + env.OVMS_CLEAN_EXPUNGE) + if(fileExists('dist\\windows\\ovms')){ + def status_del = bat(returnStatus: true, script: 'rmdir /s /q ovms') + if (status_del != 0) { + error "Error: Deleting existing ovms directory failed ${status_del}. Check pipeline.log for details." + } else { + echo "Existing ovms directory deleted successfully." + } + } } def build(){ @@ -128,6 +136,121 @@ def build(){ } else { echo "Windows package created successfully." } + def unzipCmd = "tar -xf dist\\windows\\ovms.zip" + def status_unzip = bat(returnStatus: true, script: "${unzipCmd}") + if (status_unzip != 0) { + error "Error: Unzipping package failed: ${status_unzip}." + } else { + echo "Package unzipped successfully." + } +} + +def clone_sdl_repo() +{ + if(!fileExists('sdl_repo')){ + println "Starting code signing" + def statusPull = bat(returnStatus: true, script: 'git clone -b ' + env.SIGN_REPO_BRANCH + ' ' + env.SIGN_REPO + ' sdl_repo') + if (statusPull != 0) { + error "Error: Downloading sdl_repo failed ${statusPull}. Check pipeline.log for details." + } else { + echo "sdl_repo downloaded successfully." + } + }else{ + println "Pulling latest changes in sdl_repo" + dir('sdl_repo') { + def statusPull = bat(returnStatus: true, script: 'git fetch && git reset --hard origin/'+env.SIGN_REPO_BRANCH) + if (statusPull != 0) { + error "Error: Pulling latest changes in sdl_repo failed ${statusPull}. Check pipeline.log for details." + } else { + echo "sdl_repo updated successfully." + } + } + } +} + +def clone_bdba_repo() +{ + if(!fileExists('repo_ci_infra')){ + println "Starting BDBA infrastructure download" + def statusPull = bat(returnStatus: true, script: 'git clone -b ' + env.BDBA_REPO_BRANCH + ' ' + env.BDBA_REPO + ' repo_ci_infra') + if (statusPull != 0) { + error "Error: Downloading BDBA infrastructure failed ${statusPull}. Check pipeline.log for details." + } else { + echo "BDBA infrastructure downloaded successfully." + } + }else{ + println "Pulling latest changes in BDBA infrastructure" + dir('repo_ci_infra') { + def statusPull = bat(returnStatus: true, script: 'git fetch && git reset --hard origin/'+env.BDBA_REPO_BRANCH) + if (statusPull != 0) { + error "Error: Pulling latest changes in BDBA infrastructure failed ${statusPull}. Check pipeline.log for details." + } else { + echo "BDBA infrastructure updated successfully." + } + } + } +} + +def sign(){ + println "SIGNING_USER=${env.SIGNING_USER}" + def status = bat(returnStatus: true, script: 'ci\\windows_sign.bat ' + env.SIGNING_USER + ' dist\\windows ' + env.OVMS_PYTHON_ENABLED) + if (status != 0) { + error "Error: Windows code signing failed ${status}. Check win_sign.log for details." + } else { + echo "Code signing successful." + } +} + +def bdba(){ + println "Starting BDBA scan" + def status = bat(returnStatus: true, script: 'ci\\windows_bdba.bat ' + env.BDBA_CREDS_PSW + ' dist\\windows sdl_repo\\ovms-package') + if (status != 0) { + error "Error: Windows BDBA scan failed ${status}. Check win_bdba.log for details." + } else { + echo "BDBA scan successful." + } +} + +def download_package(){ + println "Downloading package from URL: ${env.PACKAGE_URL}" + if(!fileExists('dist\\windows')){ + def status = bat(returnStatus: true, script: 'mkdir dist\\windows') + if (status != 0) { + error "Error: Creating dist\\windows directory failed ${status}. Check pipeline.log for details." + } else { + echo "Directory dist\\windows created successfully." + } + } + dir('dist\\windows') { + if(fileExists('ovms.zip')){ + def status_del = bat(returnStatus: true, script: 'del /f ovms.zip') + if (status_del != 0) { + error "Error: Deleting existing ovms.zip failed ${status_del}. Check pipeline.log for details." + } else { + echo "Existing ovms.zip deleted successfully." + } + } + if(fileExists('ovms')){ + def status_del = bat(returnStatus: true, script: 'rmdir /s /q ovms') + if (status_del != 0) { + error "Error: Deleting existing ovms directory failed ${status_del}. Check pipeline.log for details." + } else { + echo "Existing ovms directory deleted successfully." + } + } + def status = bat(returnStatus: true, script: 'curl -L -k -o ovms.zip ' + env.PACKAGE_URL) + if (status != 0) { + error "Error: Downloading package failed ${status}. Check pipeline.log for details." + } else { + echo "Package downloaded successfully." + } + def status_unzip = bat(returnStatus: true, script: 'tar -xf ovms.zip') + if (status_unzip != 0) { + error "Error: Unzipping package failed: ${status_unzip}." + } else { + echo "Package unzipped successfully." + } + } } def unit_test(){ @@ -166,7 +289,7 @@ def check_tests(){ status = bat(returnStatus: true, script: 'grep " PASSED " win_full_test.log') if (status != 0) { - error "Error: Windows run test failed ${status}. Expecting PASSED at the end of log. Check piepeline.log for details." + error "Error: Windows run test failed ${status}. Expecting PASSED at the end of log. Check pipeline.log for details." } else { echo "Success: Windows run test finished with success." } @@ -188,6 +311,18 @@ def archive_test_artifacts(){ archiveArtifacts allowEmptyArchive: true, artifacts: "win_test_log.zip" } +def archive_bdba_reports(){ + archiveArtifacts allowEmptyArchive: true, artifacts: "win_bdba.log" + archiveArtifacts allowEmptyArchive: true, artifacts: "ovms_windows_bdba_reports.zip" +} + +def archive_sign_results(){ + def python_suffix = env.OVMS_PYTHON_ENABLED == "0" ? "off" : "on" + archiveArtifacts allowEmptyArchive: true, artifacts: "win_sign.log" + archiveArtifacts allowEmptyArchive: true, artifacts: "dist\\windows\\ovms_windows_python_${python_suffix}.zip" + archiveArtifacts allowEmptyArchive: true, artifacts: "dist\\windows\\ovms_windows_python_${python_suffix}.zip.sha256" +} + def setup_bazel_remote_cache(){ def bazel_remote_cache_url = env.OVMS_BAZEL_REMOTE_CACHE_URL def content = "build --remote_cache=\"${bazel_remote_cache_url}\"" diff --git a/ci/windows_bdba.bat b/ci/windows_bdba.bat new file mode 100644 index 0000000000..37ead33971 --- /dev/null +++ b/ci/windows_bdba.bat @@ -0,0 +1,36 @@ +@echo off +set "BDBA_KEY=%1" +set "OVMS_PATH=..\%2" +set "CONFIG_PATH=..\%3" +cd repo_ci_infra + +python -m venv venv + +call venv\Scripts\activate + +python -m pip install --upgrade pip + +if exist requirements.txt ( + pip install -r requirements.txt +) + +for /f "tokens=2 delims==." %%I in ('wmic os get localdatetime /value') do set datetime=%%I +set datestamp=%datetime:~0,8% +set timestamp=%datetime:~8,4% +set filename=ovms_windows_%datestamp%_%timestamp% +set zipname="%filename%.zip" + +copy %OVMS_PATH%\\ovms.zip %OVMS_PATH%\\%zipname% +if errorlevel 1 ( + echo Failed to copy %OVMS_PATH%\ovms.zip to %OVMS_PATH%\%zipname%. + exit /b 1 +) + +echo "BDBA_KEY=%BDBA_KEY%" +echo "OVMS_PATH=%OVMS_PATH%" + +python binary_scans\ovms_bdba.py --key %BDBA_KEY% --config_dir=%CONFIG_PATH% --type windows --build_dir %OVMS_PATH% --artifacts %zipname% --report_name %filename% 2>&1 | tee ..\win_bdba.log +if errorlevel 1 exit /b %errorlevel% + +tar -a -c -f ..\ovms_windows_bdba_reports.zip ovms_windows* +del "%OVMS_PATH%\%zipname%" diff --git a/ci/windows_sign.bat b/ci/windows_sign.bat new file mode 100644 index 0000000000..e71441891f --- /dev/null +++ b/ci/windows_sign.bat @@ -0,0 +1,28 @@ +@echo off +set "OVMS_USER=%1" +set "OVMS_FILES=..\..\%2" +set "PYTHON=%3" +set PATH=%PATH%;C:\Jenkins\workspace\ovmsc\signfile;C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64 + +cd sdl_repo\windows_signing + +if /I "%PYTHON%"=="1" ( + set "PYTHON_OPT=--python" +) else ( + set "PYTHON_OPT=" +) + +python check_signing.py --user=%OVMS_USER% --path=%OVMS_FILES% %PYTHON_OPT% --auto --verbose --print_all 2>&1 | tee ..\..\win_sign.log +python check_signing.py --zip --path=%OVMS_FILES% %PYTHON_OPT% --auto + +for %%f in (ovms_windows_python_*) do ( + copy "%%f" "%OVMS_FILES%" +) +for /f "tokens=* delims=" %%a in ('type ..\..\win_sign.log ^| tail -n 1') do ( + echo %%a | findstr /C:"[ OK ]" >nul + if not errorlevel 1 ( + exit /b 0 + ) else ( + exit /b 1 + ) +) \ No newline at end of file