-
Notifications
You must be signed in to change notification settings - Fork 16
/
Copy pathsysbox-deploy-k8s.sh
executable file
·1184 lines (970 loc) · 38.3 KB
/
sysbox-deploy-k8s.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#
# Copyright 2019-2023 Nestybox, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Script to install or remove Sysbox (CE) and Sysbox-EE on a Kubernetes node.
# The script assumes it will run inside the sysbox deploy daemonset container,
# and that several host directories are mounted onto the container. The script
# requires full root privileges on the host (e.g., CAP_SYS_ADMIN + write access
# to /proc) in order to install Sysbox on it.
#
# Note: inspired by kata-deploy (github.com/kata-containers/packaging/tree/master/kata-deploy)
#
set -o errexit
set -o pipefail
set -o nounset
# The Sysbox edition to install: Sysbox (CE) or Sysbox-EE.
sysbox_edition=""
# The daemonset Dockerfile places sysbox artifacts here
sysbox_artifacts="/opt/sysbox"
crio_artifacts="/opt/crio-deploy"
# The daemonset spec will set up these mounts.
host_systemd="/mnt/host/lib/systemd/system"
host_sysctl="/mnt/host/lib/sysctl.d"
host_bin="/mnt/host/usr/bin"
host_lib_mod="/mnt/host/usr/lib/modules-load.d"
host_local_bin="/mnt/host/usr/local/bin"
host_etc="/mnt/host/etc"
host_os_release="/mnt/host/os-release"
host_crio_conf_file="${host_etc}/crio/crio.conf"
host_crio_conf_file_backup="${host_crio_conf_file}.orig"
host_run="/mnt/host/run"
host_var_lib="/mnt/host/var/lib"
host_var_lib_sysbox_deploy_k8s="${host_var_lib}/sysbox-deploy-k8s"
#
# Subid default values.
#
# Sysbox supports up 4K sys contaienrs per K8s node, each with 64K subids.
#
# Historical note: prior to Docker's acquisition of Nesytbox, Sysbox-CE was
# limited to 16-pods-per-node via variable subid_alloc_min_range below, whereas
# Sysbox-EE was limited to 4K-pods-per-node. After Docker's acquisition of
# Nestybox (05/22) Sysbox-EE is no longer being offered and therefore Docker has
# decided to lift the Sysbox-CE limit to encourage adoption of Sysbox on K8s
# clusters (the limit will now be 4K-pods-per-node as it was in Sysbox-EE).
#
subid_alloc_min_start=100000
subid_alloc_min_range=268435456
subid_alloc_max_end=4294967295
# We use CRI-O's default user "containers" for the sub-id range (rather than
# user "sysbox").
subid_user="containers"
subid_def_file="${host_etc}/login.defs"
subuid_file="${host_etc}/subuid"
subgid_file="${host_etc}/subgid"
# Shiftfs
shiftfs_min_kernel_ver=5.4
shiftfs_max_kernel_ver=6.2
# Current OS distro and kernel release
os_distro_release=""
os_kernel_release=""
# System platform architecture
sys_arch=""
# Installation flags
do_sysbox_install="true"
do_sysbox_update="false"
do_crio_install="true"
#
# CRI-O Installation Functions
#
function deploy_crio_installer_service() {
echo "Deploying CRI-O installer agent on the host ($k8s_version) ..."
cp ${crio_artifacts}/bin/${k8s_version}/cri-o.${sys_arch}.tar.gz ${host_local_bin}/cri-o.${sys_arch}.tar.gz
cp ${crio_artifacts}/bin/${k8s_version}/crio-patched ${host_local_bin}/crio-patched
cp ${crio_artifacts}/scripts/crio-installer.sh ${host_local_bin}/crio-installer.sh
cp ${crio_artifacts}/scripts/crio-extractor.sh ${host_local_bin}/crio-extractor.sh
cp ${crio_artifacts}/systemd/crio-installer.service ${host_systemd}/crio-installer.service
mkdir -p ${host_var_lib_sysbox_deploy_k8s}
cp ${crio_artifacts}/config/etc_cni_net.d_200-loopback.conf ${host_var_lib_sysbox_deploy_k8s}/etc_cni_net.d_200-loopback.conf
cp ${crio_artifacts}/config/etc_containers_registries.conf.d_000-shortnames.conf ${host_var_lib_sysbox_deploy_k8s}/etc_containers_registries.conf.d_000-shortnames.conf
cp ${crio_artifacts}/config/etc_containers_storage.conf ${host_var_lib_sysbox_deploy_k8s}/etc_containers_storage.conf
cp ${crio_artifacts}/config/etc_containers_registries.conf ${host_var_lib_sysbox_deploy_k8s}/etc_containers_registries.conf
cp ${crio_artifacts}/config/etc_containers_registries.d_default.yaml ${host_var_lib_sysbox_deploy_k8s}/etc_containers_registries.d_default.yaml
cp ${crio_artifacts}/config/etc_containers_policy.json ${host_var_lib_sysbox_deploy_k8s}/etc_containers_policy.json
systemctl daemon-reload
echo "Running CRI-O installer agent on the host (may take several seconds) ..."
systemctl restart crio-installer.service
}
function remove_crio_installer_service() {
echo "Removing CRI-O installer agent from the host ..."
systemctl stop crio-installer.service
systemctl disable crio-installer.service
rm -f ${host_local_bin}/crio-installer.sh
rm -f ${host_local_bin}/crio-extractor.sh
rm -f ${host_systemd}/crio-installer.service
systemctl daemon-reload
}
function deploy_crio_removal_service() {
echo "Deploying CRI-O uninstaller ..."
cp ${crio_artifacts}/scripts/crio-removal.sh ${host_local_bin}/crio-removal.sh
cp ${crio_artifacts}/scripts/crio-extractor.sh ${host_local_bin}/crio-extractor.sh
cp ${crio_artifacts}/systemd/crio-removal.service ${host_systemd}/crio-removal.service
systemctl daemon-reload
systemctl restart crio-removal.service
}
function remove_crio_removal_service() {
echo "Removing the CRI-O uninstaller ..."
systemctl stop crio-removal.service
systemctl disable crio-removal.service
rm -f ${host_local_bin}/crio-removal.sh
rm -f ${host_local_bin}/crio-extractor.sh
rm -f ${host_systemd}/crio-removal.service
systemctl daemon-reload
}
function deploy_kubelet_config_service() {
echo "Deploying Kubelet config agent on the host ..."
mkdir -p ${host_var_lib_sysbox_deploy_k8s}
cp ${crio_artifacts}/scripts/kubelet-config-helper.sh ${host_local_bin}/kubelet-config-helper.sh
cp ${crio_artifacts}/systemd/kubelet-config-helper.service ${host_systemd}/kubelet-config-helper.service
cp ${crio_artifacts}/config/crio-kubelet-options ${host_var_lib_sysbox_deploy_k8s}/crio-kubelet-options
cp /usr/local/bin/crictl ${host_local_bin}/sysbox-deploy-k8s-crictl
echo "Running Kubelet config agent on the host (will restart Kubelet and temporary bring down all pods on this node for ~1 min) ..."
systemctl daemon-reload
systemctl restart kubelet-config-helper.service
}
function remove_kubelet_config_service() {
echo "Stopping the Kubelet config agent on the host ..."
systemctl stop kubelet-config-helper.service
systemctl disable kubelet-config-helper.service
echo "Removing Kubelet config agent from the host ..."
rm -f ${host_local_bin}/kubelet-config-helper.sh
rm -f ${host_systemd}/kubelet-config-helper.service
rm -f ${host_local_bin}/sysbox-deploy-k8s-crictl
systemctl daemon-reload
}
function deploy_kubelet_unconfig_service() {
echo "Deploying Kubelet unconfig agent on the host ..."
cp ${crio_artifacts}/scripts/kubelet-unconfig-helper.sh ${host_local_bin}/kubelet-unconfig-helper.sh
cp ${crio_artifacts}/systemd/kubelet-unconfig-helper.service ${host_systemd}/kubelet-unconfig-helper.service
cp /usr/local/bin/crictl ${host_local_bin}/sysbox-deploy-k8s-crictl
echo "Running Kubelet unconfig agent on the host (will restart Kubelet and temporary bring down all pods on this node for ~1 min) ..."
systemctl daemon-reload
systemctl restart kubelet-unconfig-helper.service
}
function remove_kubelet_unconfig_service() {
echo "Stopping the Kubelet unconfig agent on the host ..."
systemctl stop kubelet-unconfig-helper.service
systemctl disable kubelet-unconfig-helper.service
echo "Removing Kubelet unconfig agent from the host ..."
rm -f ${host_local_bin}/kubelet-unconfig-helper.sh
rm -f ${host_systemd}/kubelet-unconfig-helper.service
rm -f ${host_local_bin}/sysbox-deploy-k8s-crictl
systemctl daemon-reload
}
function config_crio() {
echo "Configuring CRI-O ..."
touch ${host_crio_conf_file}
if [ ! -f ${host_crio_conf_file_backup} ]; then
cp ${host_crio_conf_file} ${host_crio_conf_file_backup}
fi
# Disable selinux for now.
dasel put bool -f ${host_crio_conf_file} -p toml -m "crio.runtime.selinux" false
# Add user "containers" to the /etc/subuid and /etc/subgid files
get_subid_limits
config_subid_range "$subuid_file" "$subid_alloc_min_range" "$subuid_min" "$subuid_max"
config_subid_range "$subgid_file" "$subid_alloc_min_range" "$subgid_min" "$subgid_max"
# Set capabilities to match default caps in containerd/docker
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "CHOWN"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "DAC_OVERRIDE"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "FSETID"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "FOWNER"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "SETUID"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "SETGID"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "SETPCAP"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "SETFCAP"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "NET_BIND_SERVICE"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "KILL"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "AUDIT_WRITE"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "NET_RAW"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "SYS_CHROOT"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.runtime.default_capabilities.[]' "MKNOD"
# Create 'crio.image' table (required for 'pause_image' settings).
dasel put document -f ${host_crio_conf_file} -p toml -m '.crio.image'
# Create 'crio.network' table (required for 'network_dir' settings).
dasel put document -f ${host_crio_conf_file} -p toml -m '.crio.network'
# CRI-O puts a default limit of 1024 processes per pod; this is too small for
# Sysbox pods, since these run sometimes complex software such as Docker,
# K8s, etc. Thus we increase this to 16K processes per pod. Since the max
# limit for Linux is 4M (see /proc/sys/kernel/pid_max), this allows up to
# ~256 Sysbox containers each consuming 16K processes on a given host. It
# also constraints a malicious container executing a fork bomb to 16K
# processes, well below the kernel's max pid limit.
dasel put int -f ${host_crio_conf_file} -p toml -m "crio.runtime.pids_limit" 16384
}
function restart_crio() {
echo "Restarting CRI-O ..."
systemctl restart crio
}
#
# Sysbox Installation Functions
#
function get_artifacts_dir() {
local distro=$os_distro_release
if [[ "$distro" == "ubuntu-22.04" ]] ||
[[ "$distro" == "ubuntu-21.10" ]] ||
[[ "$distro" == "ubuntu-20.04" ]] ||
[[ "$distro" == "ubuntu-18.04" ]] ||
[[ "$distro" =~ "debian" ]]; then
artifacts_dir="${sysbox_artifacts}/bin/generic"
elif [[ "$distro" =~ "flatcar" ]]; then
local release=$(echo $distro | cut -d"-" -f2)
artifacts_dir="${sysbox_artifacts}/bin/flatcar-${release}"
else
die "Sysbox is not supported on this host's distro ($distro)".
fi
echo $artifacts_dir
}
function copy_sysbox_to_host() {
local artifacts_dir=$(get_artifacts_dir)
cp "${artifacts_dir}/sysbox-mgr" "${host_bin}/sysbox-mgr"
cp "${artifacts_dir}/sysbox-fs" "${host_bin}/sysbox-fs"
cp "${artifacts_dir}/sysbox-runc" "${host_bin}/sysbox-runc"
}
function rm_sysbox_from_host() {
rm -f "${host_bin}/sysbox-mgr"
rm -f "${host_bin}/sysbox-fs"
rm -f "${host_bin}/sysbox-runc"
# Remove sysbox from the /etc/subuid and /etc/subgid files
sed -i '/sysbox:/d' "${host_etc}/subuid"
sed -i '/sysbox:/d' "${host_etc}/subgid"
}
function copy_conf_to_host() {
cp "${sysbox_artifacts}/systemd/99-sysbox-sysctl.conf" "${host_sysctl}/99-sysbox-sysctl.conf"
cp "${sysbox_artifacts}/systemd/50-sysbox-mod.conf" "${host_lib_mod}/50-sysbox-mod.conf"
}
function rm_conf_from_host() {
rm -f "${host_sysctl}/99-sysbox-sysctl.conf"
rm -f "${host_lib_mod}/50-sysbox-mod.conf"
}
function copy_systemd_units_to_host() {
cp "${sysbox_artifacts}/systemd/sysbox.service" "${host_systemd}/sysbox.service"
cp "${sysbox_artifacts}/systemd/sysbox-mgr.service" "${host_systemd}/sysbox-mgr.service"
cp "${sysbox_artifacts}/systemd/sysbox-fs.service" "${host_systemd}/sysbox-fs.service"
systemctl daemon-reload
systemctl enable sysbox.service
systemctl enable sysbox-mgr.service
systemctl enable sysbox-fs.service
}
function rm_systemd_units_from_host() {
rm -f "${host_systemd}/sysbox.service"
rm -f "${host_systemd}/sysbox-mgr.service"
rm -f "${host_systemd}/sysbox-fs.service"
systemctl daemon-reload
}
function apply_conf() {
# Note: this requires CAP_SYS_ADMIN on the host
echo "Configuring host sysctls ..."
sysctl -p "${host_sysctl}/99-sysbox-sysctl.conf"
}
function start_sysbox() {
echo "Starting $sysbox_edition ..."
systemctl restart sysbox
systemctl is-active --quiet sysbox
}
function stop_sysbox() {
if systemctl is-active --quiet sysbox; then
echo "Stopping $sysbox_edition ..."
systemctl stop sysbox
fi
}
function install_sysbox() {
# Sysbox could potentially be already installed (during upgrades),
# so stop it first to ensure that copy instructions below can
# succeed.
stop_sysbox
echo "Installing $sysbox_edition on host ..."
copy_sysbox_to_host
copy_conf_to_host
copy_systemd_units_to_host
apply_conf
start_sysbox
}
function remove_sysbox() {
echo "Removing $sysbox_edition from host ..."
stop_sysbox
rm_systemd_units_from_host
rm_conf_from_host
rm_sysbox_from_host
}
function deploy_sysbox_installer_helper() {
echo "Deploying $sysbox_edition installer helper on the host ..."
cp ${sysbox_artifacts}/scripts/sysbox-installer-helper.sh ${host_local_bin}/sysbox-installer-helper.sh
cp ${sysbox_artifacts}/systemd/sysbox-installer-helper.service ${host_systemd}/sysbox-installer-helper.service
systemctl daemon-reload
echo "Running $sysbox_edition installer helper on the host (may take several seconds) ..."
systemctl restart sysbox-installer-helper.service
}
function remove_sysbox_installer_helper() {
echo "Stopping the $sysbox_edition installer helper on the host ..."
systemctl stop sysbox-installer-helper.service
systemctl disable sysbox-installer-helper.service
echo "Removing $sysbox_edition installer helper from the host ..."
rm -f ${host_local_bin}/sysbox-installer-helper.sh
rm -f ${host_systemd}/sysbox-installer-helper.service
systemctl daemon-reload
}
function deploy_sysbox_removal_helper() {
echo "Deploying $sysbox_edition removal helper on the host..."
cp ${sysbox_artifacts}/scripts/sysbox-removal-helper.sh ${host_local_bin}/sysbox-removal-helper.sh
cp ${sysbox_artifacts}/systemd/sysbox-removal-helper.service ${host_systemd}/sysbox-removal-helper.service
systemctl daemon-reload
systemctl restart sysbox-removal-helper.service
}
function remove_sysbox_removal_helper() {
echo "Removing the $sysbox_edition removal helper ..."
systemctl stop sysbox-removal-helper.service
systemctl disable sysbox-removal-helper.service
rm -f ${host_local_bin}/sysbox-removal-helper.sh
rm -f ${host_systemd}/sysbox-removal-helper.service
systemctl daemon-reload
}
function install_sysbox_deps_flatcar() {
# Expected vars layout:
# * artifacts-dir == "/opt/sysbox/bin/flatcar-<release>"
# * distro-release == "flatcar-<release>"
local artifacts_dir=$(get_artifacts_dir)
local distro_release=$(echo ${artifacts_dir} | cut -d"/" -f5)
# Let's try to install from local store first.
if [ -d "$artifacts_dir" ]; then
echo "Copying shiftfs module and sysbox dependencies to host ..."
cp ${artifacts_dir}/shiftfs.ko ${host_lib_mod}/shiftfs.ko
cp ${artifacts_dir}/fusermount ${host_bin}/fusermount
return
fi
# Otherwise fetch the binaries/dependencies from an external location.
echo "Fetching / copying shiftfs module and sysbox dependencies to host ..."
mkdir -p ${artifacts_dir}
pushd ${artifacts_dir}/..
curl -LJOSs https://github.com/nestybox/sysbox-flatcar-preview/releases/download/Sysbox-${distro_release}/${distro_release}.tar.gz
if [ $? -ne 0 ]; then
die "Unable to fetch Sysbox dependencies for ${distro_release} distribution. Exiting ..."
fi
tar -xf ${distro_release}.tar.gz
rm -r ${distro_release}.tar.gz
cp ${artifacts_dir}/shiftfs.ko ${host_lib_mod}/shiftfs.ko
cp ${artifacts_dir}/fusermount ${host_bin}/fusermount
}
function install_sysbox_deps() {
# The installation of sysbox dependencies on the host is done via the
# sysbox-installer-helper agent, which is a systemd service that we drop on
# the host and request systemd to start. This way the agent can install
# packages on the host as needed. One of those dependencies is shiftfs, which
# unlike the other dependencies, needs to be built from source on the host
# machine (with the corresponding kernel headers, etc). The shiftfs sources
# are included in the sysbox-deploy-k8s container image, and here we copy
# them to the host machine (in dir /run/shiftfs_dkms). The
# sysbox-installer-helper agent will build those sources on the host and
# install shiftfs on the host kernel via dkms. For the specific case of
# Flatcar, we carry a pre-built shiftfs binary as we can't easily build it
# on the Flatcar host.
echo "Installing Sysbox dependencies on host ..."
local kversion=$(echo $os_kernel_release | cut -d "." -f1-2)
if semver_lt $kversion 5.4; then
echo "Kernel has version $kversion, which is below the min required for shiftfs ($shiftfs_min_kernel_ver); skipping shiftfs installation."
return
fi
if host_flatcar_distro; then
install_sysbox_deps_flatcar
else
echo "Copying shiftfs sources to host ..."
if semver_ge $kversion 5.4 && semver_lt $kversion 5.8; then
echo "Kernel version $kversion is >= 5.4 and < 5.8"
cp -r "/opt/shiftfs-k5.4" "$host_run/shiftfs-dkms"
elif semver_ge $kversion 5.8 && semver_lt $kversion 5.11; then
echo "Kernel version $kversion is >= 5.8 and < 5.11"
cp -r "/opt/shiftfs-k5.10" "$host_run/shiftfs-dkms"
elif semver_ge $kversion 5.11 && semver_lt $kversion 5.13; then
echo "Kernel version $kversion is >= 5.11 and < 5.13"
cp -r "/opt/shiftfs-k5.11" "$host_run/shiftfs-dkms"
elif semver_ge $kversion 5.13 && semver_lt $kversion 5.15; then
echo "Kernel version $kversion is >= 5.13 and < 5.15"
cp -r "/opt/shiftfs-k5.13" "$host_run/shiftfs-dkms"
elif semver_ge $kversion 5.15 && semver_lt $kversion 5.17; then
echo "Kernel version $kversion is >= 5.15 and < 5.17"
cp -r "/opt/shiftfs-k5.16" "$host_run/shiftfs-dkms"
elif semver_ge $kversion 5.17 && semver_lt $kversion 5.18; then
echo "Kernel version $kversion is 5.17"
cp -r "/opt/shiftfs-k5.17" "$host_run/shiftfs-dkms"
elif semver_ge $kversion 5.18 && semver_lt $kversion 6.1; then
echo "Kernel version $kversion is >= 5.18 and < 6.1"
cp -r "/opt/shiftfs-k5.18" "$host_run/shiftfs-dkms"
elif semver_ge $kversion 6.1 && semver_lt $kversion 6.3; then
echo "Kernel version $kversion is >= 6.1 and < 6.3"
cp -r "/opt/shiftfs-k6.1" "$host_run/shiftfs-dkms"
else
echo "Kernel version $kversion, which is above the max required for shiftfs ($shiftfs_max_kernel_ver); skipping shiftfs installation."
# dont copy shiftfs, but still proceed to install other deps
fi
fi
deploy_sysbox_installer_helper
remove_sysbox_installer_helper
}
function remove_sysbox_deps() {
echo "Removing sysbox dependencies from host ..."
deploy_sysbox_removal_helper
remove_sysbox_removal_helper
rm -rf "$host_run/shiftfs-dkms"
}
function get_subid_limits() {
# Get subid defaults from /etc/login.defs
subuid_min=$subid_alloc_min_start
subuid_max=$subid_alloc_max_end
subgid_min=$subid_alloc_min_start
subgid_max=$subid_alloc_max_end
if [ ! -f $subid_def_file ]; then
return
fi
set +e
res=$(grep "^SUB_UID_MIN" $subid_def_file 2>/dev/null)
if [ $? -eq 0 ]; then
subuid_min=$(echo $res | cut -d " " -f2)
fi
res=$(grep "^SUB_UID_MAX" $subid_def_file 2>/dev/null)
if [ $? -eq 0 ]; then
subuid_max=$(echo $res | cut -d " " -f2)
fi
res=$(grep "^SUB_GID_MIN" $subid_def_file 2>/dev/null)
if [ $? -eq 0 ]; then
subgid_min=$(echo $res | cut -d " " -f2)
fi
res=$(grep "^SUB_GID_MAX" $subid_def_file 2>/dev/null)
if [ $? -eq 0 ]; then
subgid_max=$(echo $res | cut -d " " -f2)
fi
set -e
}
function config_subid_range() {
local subid_file=$1
local subid_size=$2
local subid_min=$3
local subid_max=$4
if [ ! -f $subid_file ] || [ ! -s $subid_file ]; then
echo "$subid_user:$subid_min:$subid_size" >"${subid_file}"
return
fi
readarray -t subid_entries <"${subid_file}"
# if a large enough subid config already exists for user $subid_user, there
# is nothing to do.
for entry in "${subid_entries[@]}"; do
user=$(echo $entry | cut -d ":" -f1)
start=$(echo $entry | cut -d ":" -f2)
size=$(echo $entry | cut -d ":" -f3)
if [[ "$user" == "$subid_user" ]] && [ "$size" -ge "$subid_size" ]; then
return
fi
done
# Sort subid entries by start range
declare -a sorted_subids
if [ ${#subid_entries[@]} -gt 0 ]; then
readarray -t sorted_subids < <(echo "${subid_entries[@]}" | tr " " "\n" | tr ":" " " | sort -n -k 2)
fi
# allocate a range of subid_alloc_range size
hole_start=$subid_min
for entry in "${sorted_subids[@]}"; do
start=$(echo $entry | cut -d " " -f2)
size=$(echo $entry | cut -d " " -f3)
hole_end=$start
if [ $hole_end -ge $hole_start ] && [ $((hole_end - hole_start)) -ge $subid_size ]; then
echo "$subid_user:$hole_start:$subid_size" >>$subid_file
return
fi
hole_start=$((start + size))
done
hole_end=$subid_max
if [ $((hole_end - hole_start)) -lt $subid_size ]; then
echo "failed to allocate $subid_size sub ids in range $subid_min:$subid_max"
return
else
echo "$subid_user:$hole_start:$subid_size" >>$subid_file
return
fi
}
function config_crio_for_sysbox() {
echo "Adding Sysbox to CRI-O config ..."
if [ ! -f ${host_crio_conf_file_backup} ]; then
cp ${host_crio_conf_file} ${host_crio_conf_file_backup}
fi
# overlayfs with metacopy=on improves startup time of CRI-O rootless containers significantly
if ! dasel -n get string -f "${host_crio_conf_file}" -p toml -s 'crio.storage_option' | grep -q "metacopy=on"; then
dasel put string -f "${host_crio_conf_file}" -p toml -m 'crio.storage_driver' "overlay"
dasel put string -f "${host_crio_conf_file}" -p toml -m 'crio.storage_option.[]' "overlay.mountopt=metacopy=on"
fi
# Add Sysbox to CRI-O's runtime list
dasel put object -f "${host_crio_conf_file}" -p toml -t string -t string "crio.runtime.runtimes.sysbox-runc" \
"runtime_path=/usr/bin/sysbox-runc" "runtime_type=oci"
dasel put string -f "${host_crio_conf_file}" -p toml "crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]" \
"io.kubernetes.cri-o.userns-mode"
# In Flatcar's case we must further adjust crio config.
if host_flatcar_distro; then
sed -i 's@/usr/bin/sysbox-runc@/opt/bin/sysbox-runc@' ${host_crio_conf_file}
fi
}
function unconfig_crio_for_sysbox() {
echo "Removing Sysbox from CRI-O config ..."
# Note: dasel does not yet have a proper delete command, so we need the "sed" below.
dasel put document -f "${host_crio_conf_file}" -p toml '.crio.runtime.runtimes.sysbox-runc' ''
sed -i "s/\[crio.runtime.runtimes.sysbox-runc\]//g" "${host_crio_conf_file}"
}
#
# General Helper Functions
#
function die() {
msg="$*"
echo "ERROR: $msg" >&2
exit 1
}
function print_usage() {
echo "Usage: $0 [ce|ee] [install|cleanup]"
}
function get_k8s_version() {
local version=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.kubeletVersion}' | awk -F "." '{print $1 "." $2}')
if [ "$?" -ne 0 ]; then
die "invalid Kubernetes version"
fi
echo "$version"
}
function get_container_runtime() {
local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}')
if [ "$?" -ne 0 ]; then
die "invalid node name"
fi
if echo "$runtime" | grep -qE 'containerd.*-k3s'; then
if systemctl is-active --quiet k3s-agent; then
echo "k3s-agent"
else
echo "k3s"
fi
else
echo "$runtime" | awk -F '[:]' '{print $1}'
fi
}
function get_host_distro() {
local distro_name=$(grep -w "^ID" "$host_os_release" | cut -d "=" -f2)
local version_id=$(grep -w "^VERSION_ID" "$host_os_release" | cut -d "=" -f2 | tr -d '"')
echo "${distro_name}-${version_id}"
}
function get_sys_arch() {
local uname_m=$(uname -m)
if [[ "$uname_m" == "x86_64" ]]; then
sys_arch=amd64
elif [[ "$uname_m" == "aarch64" ]]; then
sys_arch=arm64
elif [[ "$uname_m" == "arm" ]]; then
sys_arch=armhf
elif [[ "$uname_m" == "armel" ]]; then
sys_arch=armel
fi
echo "${sys_arch}"
}
function host_flatcar_distro() {
local distro=$(get_host_distro)
echo $distro | grep -q "flatcar"
}
function get_host_kernel() {
uname -r
}
function is_supported_distro() {
local distro=$os_distro_release
if [[ "$distro" == "ubuntu-22.04" ]] ||
[[ "$distro" == "ubuntu-21.10" ]] ||
[[ "$distro" == "ubuntu-20.04" ]] ||
[[ "$distro" == "ubuntu-18.04" ]] ||
[[ "$distro" =~ "debian" ]] ||
[[ "$distro" =~ "flatcar" ]]; then
return
fi
false
}
function is_supported_kernel() {
local kversion=$(echo $os_kernel_release | cut -d "." -f1-2)
# Ubuntu distro is supported starting with kernel 5.3+.
if [[ "$os_distro_release" =~ "ubuntu" ]]; then
if semver_lt $kversion 5.3; then
echo "Unsupported kernel version $kversion for Ubuntu distribution (< 5.3)."
return 1
fi
return 0
fi
# For all other distros, Sysbox requires 5.5+.
if semver_lt $kversion 5.5; then
echo "Unsupported kernel version $kversion for $os_distro_release distribution (< 5.5)."
return 1
fi
return 0
}
function is_supported_arch() {
if [[ "$sys_arch" == "amd64" ]] || [[ "$sys_arch" == "arm64" ]]; then
return
fi
false
}
function is_supported_k8s_version() {
local ver=$k8s_version
if [[ "$ver" == "v1.26" ]] ||
[[ "$ver" == "v1.27" ]] ||
[[ "$ver" == "v1.28" ]] ||
[[ "$ver" == "v1.29" ]]; then
return
fi
if [[ "$ver" == "v1.19" ]] ||
[[ "$ver" == "v1.20" ]] ||
[[ "$ver" == "v1.21" ]] ||
[[ "$ver" == "v1.22" ]] ||
[[ "$ver" == "v1.23" ]] ||
[[ "$ver" == "v1.24" ]] ||
[[ "$ver" == "v1.25" ]]; then
echo "Unsupported kubernetes version: $ver (EOL release)."
fi
false
}
function is_kernel_upgraded() {
local cur_kernel=$os_kernel_release
if [ ! -f ${host_var_lib_sysbox_deploy_k8s}/os_kernel_release ]; then
false
return
fi
local prev_kernel=$(cat ${host_var_lib_sysbox_deploy_k8s}/os_kernel_release)
if [[ ${cur_kernel} == ${prev_kernel} ]]; then
false
return
fi
true
}
function add_label_to_node() {
local label=$1
echo "Adding K8s label \"$label\" to node ..."
kubectl label node "$NODE_NAME" --overwrite "${label}"
}
function rm_label_from_node() {
local label=$1
echo "Removing K8s label \"$label\" from node ..."
kubectl label node "$NODE_NAME" "${label}-"
}
function add_taint_to_node() {
local taint=$1
echo "Adding K8s taint \"$taint\" to node ..."
kubectl taint nodes "$NODE_NAME" "$taint" --overwrite=true
}
function rm_taint_from_node() {
taint=$1
echo "Removing K8s taint \"$taint\" from node ..."
kubectl taint nodes "$NODE_NAME" "$taint"-
}
function install_precheck() {
if systemctl is-active --quiet crio; then
do_crio_install="false"
fi
if systemctl is-active --quiet sysbox; then
do_sysbox_install="false"
fi
if is_kernel_upgraded; then
do_sysbox_update="true"
fi
}
# Compare two versions in SemVer format.
#
# Examples: (1.0.1, 1.0.1) = 0
# (1.0.1, 1.0.2) = 2
# (1.0.1, 1.0.0) = 1
# (1, 1.0) = 0
# (3.0.4.10, 3.0.4.2) = 1
# (5.0.0-22, 5.0.0-22) = 0
# (5.0.0-22, 5.0.0-21) = 1
# (5.0.0-21, 5.0.0-22) = 2
#
function version_compare() {
if [[ $1 == $2 ]]; then
return 0
fi
local IFS='.|-'
local i ver1=($1) ver2=($2)
# Fill empty fields in ver1 with zeros.
for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do
ver1[i]=0
done
for ((i = 0; i < ${#ver1[@]}; i++)); do
if [[ -z ${ver2[i]} ]]; then
# Fill empty fields in ver2 with zeros.
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]})); then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]})); then
return 2
fi
done
return 0
}
# Compare semantic versions; takes two semantic version numbers of the form
# x.y.z (or x.y), and returns 0 if the first is less than the
# second, and 1 otherwise.
function semver_lt() {
version_compare $1 $2
if [ "$?" -eq "2" ]; then
return 0
else
return 1
fi
}
# Compare semantic versions; takes two semantic version numbers of the form
# x.y.z (or x.y), and returns 0 if the first is greater than or equal to the
# second, and 1 otherwise.
function semver_ge() {
version_compare $1 $2
if [ "$?" -ne "2" ]; then
return 0
else
return 1
fi
}
function do_edition_adjustments() {
local edition_tag=$1
# Set the Sysbox edition name being installed.
if [[ ${edition_tag} == "ce" ]]; then
sysbox_edition="Sysbox"
elif [[ ${edition_tag} == "ee" ]]; then
sysbox_edition="Sysbox-EE"
else
print_usage
die "invalid sysbox edition value: $edition_tag"
fi
}
# Function holds all the adjustments that need to be carried out to meet
# distro-specific requirements. For example, in Flatcar's case these special
# requirements are a consequence of its particular partition scheme (read-only
# /usr). For readability and maintainability purposes, we opted by placing this
# adjustment logic away from the natural location where each file component is
# utilized, so we must keep this point in mind if the files being edited here
# were to be modified prior to the invocation of this routine.
function do_distro_adjustments() {
local distro=$(get_host_distro)
if [[ ! ${distro} =~ "flatcar" ]]; then
return
fi
# Ensure that Flatcar installation proceeds only in Sysbox-EE case.
if [[ ${sysbox_edition} != "Sysbox-EE" ]]; then
die "Flatcar OS distribution is only supported on Sysbox Enterprise-Edition. Exiting ..."
fi
# Adjust global vars.
host_bin="/mnt/host/opt/bin"
host_local_bin="/mnt/host/opt/local/bin"
host_systemd="/mnt/host/etc/systemd/system"
host_sysctl="/mnt/host/opt/lib/sysctl.d"
host_lib_mod="/mnt/host/opt/lib/modules-load.d"
# Ensure that required folders are already present.
mkdir -p ${host_bin} ${host_local_bin} ${host_systemd} ${host_sysctl} ${host_lib_mod}
# Adjust crio helper scripts and services.
sed -i 's@/usr/local/bin/crio@/opt/local/bin/crio@g' ${crio_artifacts}/systemd/crio-installer.service
sed -i '/Type=oneshot/a Environment=PATH=/opt/local/bin:/sbin:/bin:/usr/sbin:/usr/bin' ${crio_artifacts}/systemd/crio-removal.service
sed -i 's@/usr/local/bin/crio@/opt/local/bin/crio@g' ${crio_artifacts}/systemd/crio-removal.service
# Adjust kubelet helper scripts and services.
sed -i '/^ExecStart=/ s@/usr/local/bin@/opt/local/bin@' ${crio_artifacts}/systemd/kubelet-config-helper.service
sed -i '/^ExecStart=/ s@/usr/local/bin@/opt/local/bin@' ${crio_artifacts}/systemd/kubelet-unconfig-helper.service
sed -i '/^crictl_bin/ s@/usr/local/bin@/opt/local/bin@' ${crio_artifacts}/scripts/kubelet-config-helper.sh
sed -i '/^crictl_bin/ s@/usr/local/bin@/opt/local/bin@' ${crio_artifacts}/scripts/kubelet-unconfig-helper.sh
# Adjust sysbox helper scripts and services.
sed -i '/Type=notify/a Environment=PATH=/opt/bin:/sbin:/bin:/usr/sbin:/usr/bin' ${sysbox_artifacts}/systemd/sysbox-mgr.service
sed -i '/^ExecStart=/ s@/usr/bin/sysbox-mgr@/opt/bin/sysbox-mgr@' ${sysbox_artifacts}/systemd/sysbox-mgr.service
sed -i '/^ExecStart=/ s@/usr/bin/sysbox-fs@/opt/bin/sysbox-fs@' ${sysbox_artifacts}/systemd/sysbox-fs.service
sed -i '/Type=notify/a Environment=PATH=/opt/bin:/sbin:/bin:/usr/sbin:/usr/bin' ${sysbox_artifacts}/systemd/sysbox-fs.service
sed -i '/^ExecStart=/ s@/usr/bin@/opt/bin@g' ${sysbox_artifacts}/systemd/sysbox.service
sed -i '/^ExecStart=/ s@/usr/local/bin@/opt/local/bin@g' ${sysbox_artifacts}/systemd/sysbox-installer-helper.service
sed -i '/^ExecStart=/ s@/usr/local/bin@/opt/local/bin@g' ${sysbox_artifacts}/systemd/sysbox-removal-helper.service
# Sysctl adjustments.
sed -i '/^kernel.unprivileged_userns_clone/ s/^#*/# /' ${sysbox_artifacts}/systemd/99-sysbox-sysctl.conf
}
# determines if running on a GKE cluster by checking metadata endpoint
function check_is_gke() {
# GKE nodes will respond with an HTTP 200 for this URL and Metadata-Flavor header.
# Other clouds, URLs, etc. will throw a 404 error. If curl cannot connect the code will be 000.
is_cluster=$(curl -s -o /dev/null \
-w "%{http_code}" \
--connect-timeout 1 \
-H "Metadata-Flavor: Google" \
169.254.169.254/computeMetadata/v1/instance/attributes/cluster-name)
if [ $is_cluster -ne 200 ]; then
false
return
fi
true
}
# Fixes an issue with crio network bridge on GKE not working.
# Also adds correct path to k8s binaries on GKE nodes in /home/kubernetes/bin
function config_crio_for_gke() {
rm -rf ${host_etc}/cni/net.d/100-crio-bridge.conf
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.network.plugin_dirs.[]' "/opt/cni/bin/"
dasel put string -f ${host_crio_conf_file} -p toml -m 'crio.network.plugin_dirs.[]' "/home/kubernetes/bin"
}
#
# Main Function
#
function main() {
euid=$(id -u)
if [[ $euid -ne 0 ]]; then
die "This script must be run as root"
fi
os_distro_release=$(get_host_distro)
if ! is_supported_distro; then
die "Sysbox is not supported on this host's distro ($os_distro_release)".