Skip to content

Commit 6eef4cf

Browse files
raptorsunraptorsun
committed
konflux: hermetic build
Signed-off-by: Haoyu Sun <[email protected]>
1 parent fa2121a commit 6eef4cf

File tree

9 files changed

+2955
-4
lines changed

9 files changed

+2955
-4
lines changed

‎.tekton/lightspeed-stack-pull-request.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,17 @@ spec:
2727
value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack:on-pr-{{revision}}
2828
- name: image-expires-after
2929
value: 5d
30+
# todo: add arm64. refer to https://konflux.pages.redhat.com/docs/users/getting-started/multi-platform-builds.html#arm64-2
3031
- name: build-platforms
3132
value:
3233
- linux/x86_64
34+
# todo: change on-push pipeline,too
35+
- name: build-source-image
36+
value: 'true'
37+
- name: prefetch-input
38+
value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true"}]'
39+
- name: hermetic
40+
value: 'true'
3341
- name: dockerfile
3442
value: Containerfile
3543
pipelineSpec:

‎Containerfile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,12 @@ COPY ${LSC_SOURCE_DIR}/src ./src
2525
COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ./
2626

2727
# Bundle additional dependencies for library mode.
28-
RUN uv sync --locked --no-dev --group llslibdev
28+
# Source cachi2 environment for hermetic builds if available, otherwise use normal installation
29+
RUN if [ -f /cachi2/cachi2.env ]; then \
30+
. /cachi2/cachi2.env && uv sync --locked --no-dev --group llslibdev-hermetic; \
31+
else \
32+
uv sync --locked --no-dev --group llslibdev; \
33+
fi
2934

3035
# Explicitly remove some packages to mitigate some CVEs
3136
# - GHSA-wj6h-64fc-37mp: python-ecdsa package won't fix it upstream.

‎pyproject.toml‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,13 @@ Homepage = "https://github.com/lightspeed-core/lightspeed-stack"
8181
Issues = "https://github.com/lightspeed-core/lightspeed-stack/issues"
8282

8383
# PyTorch has multiple wheel variants for different backends - cpu, gpu, etc.
84-
# By default on pypi.org is the gpu variant. Forces uv to use the cpu variant.
84+
# By default on pypi.org is the gpu variant. Forces uv to use the cpu variant for llslibdev group.
8585
[[tool.uv.index]]
8686
name = "pytorch-cpu"
8787
url = "https://download.pytorch.org/whl/cpu"
8888
explicit = true
8989
[tool.uv.sources]
90-
torch = [{ index = "pytorch-cpu" }]
90+
torch = [{ index = "pytorch-cpu", group = "llslibdev" }]
9191

9292
[dependency-groups]
9393
dev = [
@@ -161,6 +161,56 @@ llslibdev = [
161161
"psutil>=7.0.0",
162162
]
163163

164+
llslibdev-hermetic = [
165+
# the same as llslibdev group, with gpu variant of torch package
166+
"matplotlib>=3.10.0",
167+
"pillow>=11.1.0",
168+
"pandas>=2.2.3",
169+
"scikit-learn>=1.5.2",
170+
"psycopg2-binary>=2.9.10",
171+
# API eval: inline::meta-reference
172+
"tree_sitter>=0.24.0",
173+
"pythainlp>=3.0.10",
174+
"langdetect>=1.0.9",
175+
"emoji>=2.1.0",
176+
"nltk>=3.8.1",
177+
# API inference: remote::gemini
178+
"litellm>=1.75.5.post1",
179+
# API inference: inline::sentence-transformers
180+
"sentence-transformers>=5.0.0",
181+
# API vector_io: inline::faiss
182+
"faiss-cpu>=1.11.0",
183+
# API scoring: inline::basic
184+
"requests>=2.32.4",
185+
# API datasetio: inline::localfs
186+
"aiosqlite>=0.21.0",
187+
# API datasetio: remote::huggingface
188+
"datasets>=3.6.0",
189+
# API telemetry: inline::meta-reference
190+
"opentelemetry-sdk>=1.34.1",
191+
"opentelemetry-exporter-otlp>=1.34.1",
192+
# API tool_runtime: inline::rag-runtime
193+
"transformers>=4.34.0",
194+
"numpy==2.2.6",
195+
# API tool_runtime: remote::model-context-protocol
196+
"mcp>=1.9.4",
197+
# API post_training: inline::huggingface
198+
"torch==2.7.1",
199+
"trl>=0.18.2",
200+
"peft>=0.15.2",
201+
# Other
202+
"autoevals>=0.0.129",
203+
"fire>=0.7.0",
204+
"opentelemetry-instrumentation>=0.55b0",
205+
"blobfile>=3.0.0",
206+
"psutil>=7.0.0",
207+
]
208+
209+
tool-hermetic = [
210+
"uv==0.8.15",
211+
"pdm>=2.21.0",
212+
]
213+
164214
build = [
165215
"build>=1.2.2.post1",
166216
"twine>=6.1.0",

‎requirements.hermetic.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
uv==0.8.15
2+
pdm>=2.21.0

‎requirements.txt‎

Lines changed: 2463 additions & 0 deletions
Large diffs are not rendered by default.

‎rpms.in.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
packages: [gcc, jq, patch]
2+
contentOrigin:
3+
repofiles: ["./ubi.repo"]
4+
arches: [x86_64, aarch64]

‎rpms.lock.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
---
2+
lockfileVersion: 1
3+
lockfileVendor: redhat
4+
arches:
5+
- arch: aarch64
6+
packages:
7+
- url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/o/oniguruma-6.9.6-1.el9.5.aarch64.rpm
8+
repoid: ubi-9-appstream-rpms
9+
size: 222582
10+
checksum: sha256:bc2305dad655ddb94f966158112efd6cefa6824d5aa2e80f63881f16cee74598
11+
name: oniguruma
12+
evr: 6.9.6-1.el9.5
13+
sourcerpm: oniguruma-6.9.6-1.el9.5.src.rpm
14+
- url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/j/jq-1.6-17.el9_6.2.aarch64.rpm
15+
repoid: ubi-9-baseos-rpms
16+
size: 185443
17+
checksum: sha256:46c953403e2da3202cb0509f4895afacbbfa95f840b4e228dc4c72c1523b8aae
18+
name: jq
19+
evr: 1.6-17.el9_6.2
20+
sourcerpm: jq-1.6-17.el9_6.2.src.rpm
21+
source: []
22+
module_metadata: []
23+
- arch: x86_64
24+
packages:
25+
- url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/o/oniguruma-6.9.6-1.el9.5.x86_64.rpm
26+
repoid: ubi-9-appstream-rpms
27+
size: 226331
28+
checksum: sha256:6c884cc2216e5b4699ebd8cde27b39e99532520b367f645ed6cc660d081916dc
29+
name: oniguruma
30+
evr: 6.9.6-1.el9.5
31+
sourcerpm: oniguruma-6.9.6-1.el9.5.src.rpm
32+
- url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/j/jq-1.6-17.el9_6.2.x86_64.rpm
33+
repoid: ubi-9-baseos-rpms
34+
size: 191681
35+
checksum: sha256:d3c6d74db82f6c55533f2d9798d2d4e44988d212880b5b2afd855a43fe2b17d9
36+
name: jq
37+
evr: 1.6-17.el9_6.2
38+
sourcerpm: jq-1.6-17.el9_6.2.src.rpm
39+
source: []
40+
module_metadata: []

‎ubi.repo‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
[ubi-9-baseos-rpms]
2+
name = Red Hat Universal Base Image 9 (RPMs) - BaseOS
3+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/os
4+
enabled = 1
5+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
6+
gpgcheck = 1
7+
8+
[ubi-9-baseos-debug-rpms]
9+
name = Red Hat Universal Base Image 9 (Debug RPMs) - BaseOS
10+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/debug
11+
enabled = 0
12+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
13+
gpgcheck = 1
14+
15+
[ubi-9-baseos-source-rpms]
16+
name = Red Hat Universal Base Image 9 (Source RPMs) - BaseOS
17+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/source/SRPMS
18+
enabled = 0
19+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
20+
gpgcheck = 1
21+
22+
[ubi-9-appstream-rpms]
23+
name = Red Hat Universal Base Image 9 (RPMs) - AppStream
24+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/os
25+
enabled = 1
26+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
27+
gpgcheck = 1
28+
29+
[ubi-9-appstream-debug-rpms]
30+
name = Red Hat Universal Base Image 9 (Debug RPMs) - AppStream
31+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/debug
32+
enabled = 0
33+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
34+
gpgcheck = 1
35+
36+
[ubi-9-appstream-source-rpms]
37+
name = Red Hat Universal Base Image 9 (Source RPMs) - AppStream
38+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/source/SRPMS
39+
enabled = 0
40+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
41+
gpgcheck = 1
42+
43+
[ubi-9-codeready-builder-rpms]
44+
name = Red Hat Universal Base Image 9 (RPMs) - CodeReady Builder
45+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/os
46+
enabled = 1
47+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
48+
gpgcheck = 1
49+
50+
[ubi-9-codeready-builder-debug-rpms]
51+
name = Red Hat Universal Base Image 9 (Debug RPMs) - CodeReady Builder
52+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/debug
53+
enabled = 0
54+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
55+
gpgcheck = 1
56+
57+
[ubi-9-codeready-builder-source-rpms]
58+
name = Red Hat Universal Base Image 9 (Source RPMs) - CodeReady Builder
59+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/source/SRPMS
60+
enabled = 0
61+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
62+
gpgcheck = 1

0 commit comments

Comments
 (0)