konflux: hermetic build

Signed-off-by: Haoyu Sun <[email protected]>

Author: raptorsun

.tekton/lightspeed-stack-pull-request.yaml

@@ -27,9 +27,17 @@ spec:
     value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack:on-pr-{{revision}}
   - name: image-expires-after
     value: 5d
+  # todo: add arm64. refer to https://konflux.pages.redhat.com/docs/users/getting-started/multi-platform-builds.html#arm64-2
   - name: build-platforms
     value:
     - linux/x86_64
+  # todo: change on-push pipeline,too
+  - name: build-source-image
+    value: 'true'
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true", "requirements_files": ["requirements.txt", "requirements.hermetic.txt"]}]'
+  - name: hermetic
+    value: 'true'
   - name: dockerfile
     value: Containerfile
   pipelineSpec:

.tekton/lightspeed-stack-push.yaml

@@ -24,9 +24,16 @@ spec:
     value: '{{revision}}'
   - name: output-image
     value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack:{{revision}}
+  # todo: add arm64. refer to https://konflux.pages.redhat.com/docs/users/getting-started/multi-platform-builds.html#arm64-2
   - name: build-platforms
     value:
     - linux/x86_64
+  - name: build-source-image
+    value: 'true'
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true", "requirements_files": ["requirements.txt", "requirements.hermetic.txt"]}]'
+  - name: hermetic
+    value: 'true'
   - name: dockerfile
     value: Containerfile
   pipelineSpec:

Containerfile

@@ -19,13 +19,24 @@ RUN dnf --disablerepo="*" --enablerepo="ubi-9-appstream-rpms" --enablerepo="ubi-
 # Install uv package manager
 RUN pip3.12 install "uv==0.8.15"
 
+# Install pdm and pdm-backend for hermetic builds
+RUN if [ -f /cachi2/cachi2.env ]; then pip3.12 install "pdm>=2.21.0" "pdm-backend"; fi
+
 # Add explicit files and directories
 # (avoid accidental inclusion of local directories or env files or credentials)
 COPY ${LSC_SOURCE_DIR}/src ./src
-COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ./
+COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ${LSC_SOURCE_DIR}/requirements.txt ./
 
 # Bundle additional dependencies for library mode.
-RUN uv sync --locked --no-dev --group llslibdev
+# Source cachi2 environment for hermetic builds if available, otherwise use normal installation
+# cachi2.env has these env vars:
+# PIP_FIND_LINKS=/cachi2/output/deps/pip
+# PIP_NO_INDEX=true
+RUN if [ -f /cachi2/cachi2.env ]; then \
+        . /cachi2/cachi2.env && uv venv --seed --no-index --find-links ${PIP_FIND_LINKS} && . .venv/bin/activate && pip install --no-index --find-links ${PIP_FIND_LINKS} -r requirements.txt; \
+    else \
+        uv sync --locked --no-dev --group llslibdev; \
+    fi
 
 # Explicitly remove some packages to mitigate some CVEs
 # - GHSA-wj6h-64fc-37mp: python-ecdsa package won't fix it upstream.

pyproject.toml

@@ -87,8 +87,23 @@ Issues = "https://github.com/lightspeed-core/lightspeed-stack/issues"
 name = "pytorch-cpu"
 url = "https://download.pytorch.org/whl/cpu"
 explicit = true
+
+[[tool.uv.index]]
+name = "pypi-default"
+url = "https://pypi.org/simple"
+explicit = true
 [tool.uv.sources]
-torch = [{ index = "pytorch-cpu" }]
+torch = [
+    { index = "pytorch-cpu", group = "llslibdev" },
+    { index = "pypi-default", group = "llslibdev-hermetic" }
+]
+[tool.uv]
+conflicts = [
+    [
+        { group = "llslibdev" },
+        { group = "llslibdev-hermetic" },
+    ],
+]
 
 [dependency-groups]
 dev = [
@@ -159,6 +174,50 @@ llslibdev = [
     "blobfile>=3.0.0",
     "psutil>=7.0.0",
 ]
+llslibdev-hermetic = [
+   # the same as llslibdev, just using default index.
+   "matplotlib>=3.10.0",
+   "pillow>=11.1.0",
+   "pandas>=2.2.3",
+   "scikit-learn>=1.5.2",
+   "psycopg2-binary>=2.9.10",
+   # API eval: inline::meta-reference
+   "tree_sitter>=0.24.0",
+   "pythainlp>=3.0.10",
+   "langdetect>=1.0.9",
+   "emoji>=2.1.0",
+   "nltk>=3.8.1",
+   # API inference: remote::gemini
+   "litellm>=1.75.5.post1",
+   # API inference: inline::sentence-transformers
+   "sentence-transformers>=5.0.0",
+   # API vector_io: inline::faiss
+   "faiss-cpu>=1.11.0",
+   # API scoring: inline::basic
+   "requests>=2.32.4",
+   # API datasetio: inline::localfs
+   "aiosqlite>=0.21.0",
+   # API datasetio: remote::huggingface
+   "datasets>=3.6.0",
+   # API telemetry: inline::meta-reference
+   "opentelemetry-sdk>=1.34.1",
+   "opentelemetry-exporter-otlp>=1.34.1",
+   # API tool_runtime: inline::rag-runtime
+   "transformers>=4.34.0",
+   "numpy==2.2.6",
+   # API tool_runtime: remote::model-context-protocol
+   "mcp>=1.9.4",
+   # API post_training: inline::huggingface
+   "torch==2.7.1; sys_platform == 'linux'",
+   "trl>=0.18.2",
+   "peft>=0.15.2",
+   # Other
+   "autoevals>=0.0.129",
+   "fire>=0.7.0",
+   "opentelemetry-instrumentation>=0.55b0",
+   "blobfile>=3.0.0",
+   "psutil>=7.0.0",
+]
 
 build = [
     "build>=1.2.2.post1",

requirements.hermetic.txt

@@ -0,0 +1,4 @@
+uv==0.8.15
+pdm==2.26.1
+pdm-backend==2.4.6
+pip==24.2