Skip to content

Commit c29de88

Browse files
leelavgleelavg
authored
Pin Python Packages via requirements file for each component (kadalu#592)
* Pin Python Packages via requirements file for each component * Use makefile to generate requirements of the repo Signed-off-by: Leela Venkaiah G <[email protected]>
1 parent f7b9434 commit c29de88

22 files changed

+358
-90
lines changed

‎.github/workflows/on-demand-pypi-source-push.yml

-43
This file was deleted.

‎.github/workflows/on-pr-merge.yml

+1-2
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,7 @@ jobs:
4242
python-version: '3.x'
4343
- name: Get the CLI build (kubectl-kadalu)
4444
run: |
45-
python -m pip install --upgrade pip
46-
pip install -r requirements.txt
45+
python -m pip install -r requirements/ci_merge-requirements.txt
4746
KADALU_VERSION=devel make cli-build
4847
-
4948
name: Builder Image

‎.github/workflows/on-pr-merge_arm.yml

+1-2
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ jobs:
3838
python-version: '3.x'
3939
- name: Get the CLI build (kubectl-kadalu)
4040
run: |
41-
python -m pip install --upgrade pip
42-
pip install -r requirements.txt
41+
python -m pip install -r requirements/ci_merge-requirements.txt
4342
KADALU_VERSION=devel make cli-build
4443
-
4544
name: Build CSI Image and push

‎.github/workflows/on-pr-submit.yml

+1-3
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,7 @@ jobs:
4343
python-version: '3.x'
4444
- name: Install dependencies pylint
4545
run: |
46-
python -m pip install --upgrade pip
47-
pip install -r requirements.txt
48-
pip install pylint glustercli
46+
python -m pip install -r requirements/ci_submit-requirements.txt -r requirements/ci_merge-requirements.txt
4947
- name: Run pylint
5048
run: make pylint
5149

‎.github/workflows/on-release-tag.yml

+3-6
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,7 @@ jobs:
2424
python-version: '3.x'
2525
- name: Run 'make release'
2626
run: |
27-
python -m pip install --upgrade pip
28-
pip install -r requirements.txt
27+
python -m pip install -r requirements/ci_merge-requirements.txt
2928
KADALU_VERSION=${{ env.kadalu_version }} TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD }} make pypi-upload
3029
3130
publish-artifacts:
@@ -39,8 +38,7 @@ jobs:
3938
python-version: '3.x'
4039
- name: Build kubectl-kadalu
4140
run: |
42-
python -m pip install --upgrade pip
43-
pip install -r requirements.txt
41+
python -m pip install -r requirements/ci_merge-requirements.txt
4442
KADALU_VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') make cli-build
4543
- name: Generate Manifests
4644
run: |
@@ -99,8 +97,7 @@ jobs:
9997
python-version: '3.x'
10098
- name: Get the CLI build (kubectl-kadalu)
10199
run: |
102-
python -m pip install --upgrade pip
103-
pip install -r requirements.txt
100+
python -m pip install -r requirements/ci_merge-requirements.txt
104101
KADALU_VERSION=${{ env.kadalu_version }} make cli-build
105102
-
106103
name: Build Builder Image

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ cli/LICENSE
1010
server/VERSION
1111
server/LICENSE
1212
server/kadalu_quotad/kadalulib.py*
13-
server/kadalu_quotad.egg-info
13+
server/kadalu_quotad.egg-info
14+
*egg-info*

‎Makefile

+13
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ help:
1515
@echo " make prepare-release-manifests - Prepare release manifest files"
1616
@echo " make cli-build - Build CLI binary"
1717
@echo " make helm-chart - Create a tgz archive of Helm chart"
18+
@echo " make gen-requirements - Generate requirements file for kadalu components"
1819

1920
build-grpc:
2021
python3 -m grpc_tools.protoc -I./csi/protos --python_out=csi --grpc_python_out=csi ./csi/protos/csi.proto
@@ -122,6 +123,18 @@ helm-chart:
122123
@echo "Creating tgz archive of helm chart(Version: ${KADALU_VERSION}).."
123124
cd helm; sed -i -e "s/0.0.0-0/${KADALU_VERSION}/" kadalu/Chart.yaml; tar -czf kadalu-helm-chart.tgz kadalu
124125

126+
gen-requirements:
127+
@echo "Generating requirements file for all kadalu components and CI"
128+
@cp -f requirements/setup.py requirements/setup.cfg .
129+
pip-compile --extra=builder -o requirements/builder-requirements.txt --allow-unsafe
130+
pip-compile --extra=operator -o requirements/operator-requirements.txt
131+
pip-compile --extra=csi -o requirements/csi-requirements.txt
132+
pip-compile --extra=server -o requirements/server-requirements.txt
133+
pip-compile --extra=ci_submit -o requirements/ci_submit-requirements.txt
134+
pip-compile --extra=ci_merge -o requirements/ci_merge-requirements.txt --allow-unsafe
135+
@rm setup.py setup.cfg -f
136+
137+
125138
ifeq ($(TWINE_PASSWORD),)
126139
pypi-upload: pypi-build
127140
cd server; twine upload --username kadalu dist/*

‎build.sh

+3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
set -e -o pipefail
44

5+
cp requirements/* .
6+
trap "rm -f *-requirements.txt" EXIT
7+
58
DOCKER_USER="${DOCKER_USER:-kadalu}"
69
KADALU_VERSION="${KADALU_VERSION}"
710

‎csi/Dockerfile

+3-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ FROM kadalu/builder:${builder_version} as builder
44

55
ENV PATH="/kadalu/bin:/opt/bin:/opt/sbin:$PATH"
66

7-
RUN python3 -m pip install googleapis-common-protos pyxattr grpcio
7+
COPY csi-requirements.txt /tmp/
8+
9+
RUN python3 -m pip install -r /tmp/csi-requirements.txt
810

911
FROM ubuntu:20.04 as prod
1012
ENV DEBIAN_FRONTEND=noninteractive

‎extras/Dockerfile.builder

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ ENV DEBIAN_FRONTEND=noninteractive
77
ENV VIRTUAL_ENV=/kadalu
88
ENV PATH="$VIRTUAL_ENV/bin:/opt/sbin:/opt/bin:$PATH"
99
10+
COPY builder-requirements.txt /tmp/
11+
1012
RUN apt-get update -yq && \
1113
apt-get install -y --no-install-recommends python3 curl xfsprogs net-tools telnet wget e2fsprogs \
1214
python3-pip sqlite build-essential g++ python3-dev flex bison openssl libssl-dev libtirpc-dev liburcu-dev \
@@ -17,9 +19,7 @@ RUN apt-get update -yq && \
1719
curl -L https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/`uname -m | sed 's|aarch64|arm64|' | sed 's|x86_64|amd64|' | sed 's|armv7l|arm|'`/kubectl -o /usr/bin/kubectl && \
1820
chmod +x /usr/bin/kubectl && \
1921
python3 -m venv $VIRTUAL_ENV && cd $VIRTUAL_ENV && \
20-
python3 -m pip install --upgrade pip && \
21-
python3 -m pip install --upgrade setuptools && \
22-
pip install prometheus-client jinja2 requests datetime xxhash
22+
python3 -m pip install -r /tmp/builder-requirements.txt
2323
2424
RUN sed -i "s/include-system-site-packages = false/include-system-site-packages = true/g" /kadalu/pyvenv.cfg
2525

‎extras/Dockerfile.python-packages

-19
This file was deleted.

‎operator/Dockerfile

+3-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ FROM kadalu/builder:${builder_version} as builder
44

55
ENV PATH="/kadalu/bin:/opt/bin:/opt/sbin:$PATH"
66

7-
RUN python3 -m pip install kubernetes==11.0.0
7+
COPY operator-requirements.txt /tmp/
8+
9+
RUN python3 -m pip install -r /tmp/operator-requirements.txt
810

911
FROM ubuntu:20.04 as prod
1012

‎requirements.txt

-8
This file was deleted.

‎requirements/builder-requirements.txt

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#
2+
# This file is autogenerated by pip-compile with python 3.6
3+
# To update, run:
4+
#
5+
# pip-compile --allow-unsafe --extra=builder --output-file=requirements/builder-requirements.txt
6+
#
7+
certifi==2021.5.30
8+
# via requests
9+
chardet==4.0.0
10+
# via requests
11+
datetime==4.3
12+
# via Kadalu (setup.py)
13+
idna==2.10
14+
# via requests
15+
jinja2==3.0.1
16+
# via Kadalu (setup.py)
17+
markupsafe==2.0.1
18+
# via jinja2
19+
prometheus-client==0.11.0
20+
# via Kadalu (setup.py)
21+
pytz==2021.1
22+
# via datetime
23+
requests==2.25.1
24+
# via Kadalu (setup.py)
25+
urllib3==1.26.6
26+
# via requests
27+
xxhash==2.0.2
28+
# via Kadalu (setup.py)
29+
zope.interface==5.4.0
30+
# via datetime
31+
32+
# The following packages are considered to be unsafe in a requirements file:
33+
pip==21.1.3
34+
# via Kadalu (setup.py)
35+
setuptools==57.1.0
36+
# via
37+
# Kadalu (setup.py)
38+
# zope.interface

‎requirements/ci_merge-requirements.txt

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
#
2+
# This file is autogenerated by pip-compile with python 3.6
3+
# To update, run:
4+
#
5+
# pip-compile --allow-unsafe --extra=ci_merge --output-file=requirements/ci_merge-requirements.txt
6+
#
7+
attrs==21.2.0
8+
# via pytest
9+
bleach==3.3.0
10+
# via readme-renderer
11+
cachetools==4.2.2
12+
# via google-auth
13+
certifi==2021.5.30
14+
# via
15+
# kubernetes
16+
# requests
17+
cffi==1.14.5
18+
# via cryptography
19+
chardet==4.0.0
20+
# via requests
21+
colorama==0.4.4
22+
# via twine
23+
cryptography==3.4.7
24+
# via secretstorage
25+
docutils==0.17.1
26+
# via readme-renderer
27+
google-auth==1.32.1
28+
# via kubernetes
29+
grpcio==1.38.1
30+
# via Kadalu (setup.py)
31+
idna==2.10
32+
# via requests
33+
importlib-metadata==4.6.1
34+
# via
35+
# keyring
36+
# pluggy
37+
# pytest
38+
# twine
39+
iniconfig==1.1.1
40+
# via pytest
41+
jeepney==0.6.0
42+
# via
43+
# keyring
44+
# secretstorage
45+
jinja2==3.0.1
46+
# via Kadalu (setup.py)
47+
keyring==23.0.1
48+
# via twine
49+
kubernetes==11.0.0
50+
# via Kadalu (setup.py)
51+
markupsafe==2.0.1
52+
# via jinja2
53+
oauthlib==3.1.1
54+
# via requests-oauthlib
55+
packaging==21.0
56+
# via
57+
# bleach
58+
# pytest
59+
pkginfo==1.7.0
60+
# via twine
61+
pluggy==0.13.1
62+
# via pytest
63+
py==1.10.0
64+
# via pytest
65+
pyasn1==0.4.8
66+
# via
67+
# pyasn1-modules
68+
# rsa
69+
pyasn1-modules==0.2.8
70+
# via google-auth
71+
pycparser==2.20
72+
# via cffi
73+
pygments==2.9.0
74+
# via readme-renderer
75+
pyparsing==2.4.7
76+
# via packaging
77+
pytest==6.2.4
78+
# via Kadalu (setup.py)
79+
python-dateutil==2.8.1
80+
# via kubernetes
81+
pyxattr==0.7.2
82+
# via Kadalu (setup.py)
83+
pyyaml==5.4.1
84+
# via kubernetes
85+
readme-renderer==29.0
86+
# via twine
87+
requests==2.25.1
88+
# via
89+
# kubernetes
90+
# requests-oauthlib
91+
# requests-toolbelt
92+
# twine
93+
requests-oauthlib==1.3.0
94+
# via kubernetes
95+
requests-toolbelt==0.9.1
96+
# via twine
97+
rfc3986==1.5.0
98+
# via twine
99+
rsa==4.7.2
100+
# via google-auth
101+
secretstorage==3.3.1
102+
# via keyring
103+
six==1.16.0
104+
# via
105+
# bleach
106+
# google-auth
107+
# grpcio
108+
# kubernetes
109+
# python-dateutil
110+
# readme-renderer
111+
toml==0.10.2
112+
# via pytest
113+
tqdm==4.61.1
114+
# via twine
115+
twine==3.4.1
116+
# via Kadalu (setup.py)
117+
typing-extensions==3.10.0.0
118+
# via importlib-metadata
119+
urllib3==1.26.6
120+
# via
121+
# kubernetes
122+
# requests
123+
webencodings==0.5.1
124+
# via bleach
125+
websocket-client==1.1.0
126+
# via kubernetes
127+
xxhash==2.0.2
128+
# via Kadalu (setup.py)
129+
zipp==3.5.0
130+
# via importlib-metadata
131+
132+
# The following packages are considered to be unsafe in a requirements file:
133+
setuptools==57.1.0
134+
# via
135+
# google-auth
136+
# kubernetes

0 commit comments

Comments
 (0)