Setup new node image with swap enabled

config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml

@@ -565,3 +565,79 @@ presubmits:
           env:
             - name: GOPATH
               value: /go
+
+  - name: pull-kubernetes-node-kubelet-swap-ubuntu
+    always_run: false
+    optional: true
+    skip_report: false
+    skip_branches:
+      - release-\d+\.\d+  # per-release image
+    annotations:
+      testgrid-dashboards: sig-node-kubelet
+      testgrid-tab-name: pr-kubelet-gce-e2e-swap-ubuntu
+    labels:
+      preset-service-account: "true"
+      preset-k8s-ssh: "true"
+    spec:
+      containers:
+        - image: gcr.io/k8s-testimages/kubekins-e2e:v20210601-ea6aa4e-master
+          args:
+            - --repo=k8s.io/kubernetes=$(PULL_REFS)
+            - --repo=k8s.io/release
+            - --timeout=240
+            - --root=/go/src
+            - --scenario=kubernetes_e2e
+            - --
+            - --deployment=node
+            - --gcp-project=k8s-jkns-pr-node-e2e
+            - --gcp-zone=us-west1-b
+            - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/swap/image-config-swap.yaml
+            - --node-test-args=--feature-gates=NodeSwapEnabled=true --kubelet-flags="--cgroups-per-qos=true --cgroup-root=/ --fail-swap-on=false"
+            - --node-tests=true
+            - --provider=gce
+            - --test_args=--nodes=8 --focus="\[NodeConformance\]|\[NodeFeature:.+\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]"
+            - --timeout=180m
+          env:
+            - name: GOPATH
+              value: /go
+
+  - name: pull-kubernetes-node-swap-fedora
+    skip_branches:
+    - release-\d+\.\d+  # per-release image
+    annotations:
+      testgrid-dashboards: sig-node-kubelet
+      testgrid-tab-name: pr-kubelet-gce-e2e-swap-fedora
+    always_run: true
+    skip_report: true
+    optional: true
+    max_concurrency: 12
+    labels:
+      preset-service-account: "true"
+      preset-k8s-ssh: "true"
+      preset-pull-kubernetes-e2e: "true"
+      preset-pull-kubernetes-e2e-gce: "true"
+    spec:
+      containers:
+      - image: gcr.io/k8s-testimages/kubekins-e2e:v20210601-ea6aa4e-master
+        args:
+        - --root=/go/src
+        - "--job=$(JOB_NAME)"
+        - "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)"
+        - "--service-account=/etc/service-account/service-account.json"
+        - "--upload=gs://kubernetes-jenkins/pr-logs"
+        - "--timeout=240"
+        - --scenario=kubernetes_e2e
+        - -- # end bootstrap args, scenario args below
+        - --deployment=node
+        - --env=KUBE_SSH_USER=core
+        - --gcp-project=k8s-jkns-pr-node-e2e
+        - --gcp-zone=us-west1-b
+        - '--node-test-args=--feature-gates=NodeSwapEnabled=true --container-runtime=remote --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--fail-swap-on=false --cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service --non-masquerade-cidr=0.0.0.0/0" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"'
+        - --node-tests=true
+        - --provider=gce
+        - --test_args=--nodes=8 --focus="\[NodeConformance\]|\[NodeFeature:.+\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]"
+        - --timeout=180m
+        - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/swap/image-config-swap-fedora.yaml
+        resources:
+          requests:
+            memory: "6Gi"

jobs/e2e_node/swap/crio_swap1g.ign

@@ -0,0 +1,24 @@
+{
+  "ignition": {
+    "version": "3.1.0"
+  },
+  "systemd": {
+    "units": [
+      {
+        "contents": "[Unit]\nDescription=Enable swap on CoreOS\nBefore=swap-enable.service\nConditionFirstBoot=yes\n\n[Service]\nType=oneshot\nExecStart=/bin/sh -c \"sudo rpm-ostree install dbus-tools && sudo systemctl reboot\"\n[Install]\n\nWantedBy=multi-user.target\n",
+        "enabled": true,
+        "name": "dbus-send-install.service"
+      },
+      {
+        "contents": "[Unit]\nDescription=Enable swap on CoreOS\nBefore=crio-install.service\nConditionFirstBoot=no\n\n[Service]\nType=oneshot\nExecStart=/bin/sh -c \"sudo dd if=/dev/zero of=/var/swapfile count=1024 bs=1MiB && sudo chmod 600 /var/swapfile && sudo mkswap /var/swapfile && sudo swapon /var/swapfile && free -h\"\n[Install]\n\nWantedBy=multi-user.target\n",
+        "enabled": true,
+        "name": "swap-enable.service"
+      },
+      {
+        "contents": "[Unit]\nDescription=Download and install crio binaries and configurations.\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStartPre=/usr/sbin/setenforce 1\nExecStartPre=/usr/bin/bash -c '/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -o /usr/local/crio-install.sh  https://raw.githubusercontent.com/cri-o/cri-o/master/scripts/get'\nExecStartPre=/usr/bin/bash /usr/local/crio-install.sh\nExecStartPre=/usr/bin/mkdir -p /var/lib/kubelet\nExecStartPre=/usr/bin/chcon -R -u system_u -r object_r -t var_lib_t /var/lib/kubelet\nExecStartPre=/usr/bin/mount /tmp /tmp -o remount,exec,suid\nExecStartPre=/usr/bin/chcon -u system_u -r object_r -t container_runtime_exec_t /usr/local/bin/crio /usr/local/bin/crio-status /usr/local/bin/runc /usr/local/bin/crun\nExecStartPre=/usr/bin/chcon -u system_u -r object_r -t bin_t /usr/local/bin/conmon /usr/local/bin/crictl /usr/local/bin/pinns\nExecStartPre=/usr/bin/chcon -R -u system_u -r object_r -t bin_t /opt/cni/bin/\nExecStartPre=/usr/bin/rm -f  /etc/cni/net.d/87-podman-bridge.conflist\nExecStartPre=/usr/bin/bash -c 'echo -e \"[crio.runtime]\\n  default_runtime = \\\\\\\"runc\\\\\\\"\\n[crio.runtime.runtimes]\\n  [crio.runtime.runtimes.runc]\\n    runtime_path=\\\\\\\"/usr/local/bin/runc\\\\\\\"\" \u003e /etc/crio/crio.conf.d/20-runc.conf'\nExecStartPre=/usr/bin/bash -c 'echo -e \"[crio.runtime]\\n  [crio.runtime.runtimes]\\n  [crio.runtime.runtimes.test-handler]\\n    runtime_path=\\\\\\\"/usr/local/bin/crun\\\\\\\"\" \u003e /etc/crio/crio.conf.d/10-crun.conf'\nExecStartPre=/usr/bin/chcon -R -u system_u -r object_r -t container_config_t /etc/crio /etc/crio/crio.conf /usr/local/share/oci-umount/oci-umount.d/crio-umount.conf\nExecStartPre=/usr/bin/systemctl enable crio.service\nExecStartPre=/usr/bin/chcon -R -u system_u -r object_r -t systemd_unit_file_t /usr/local/lib/systemd/system/crio.service\nExecStart=/usr/bin/systemctl start crio.service\n\n[Install]\nWantedBy=multi-user.target\n",
+        "enabled": true,
+        "name": "crio-install.service"
+      }
+    ]
+  }
+}

jobs/e2e_node/swap/image-config-swap-fedora.yaml

@@ -0,0 +1,9 @@
+# To copy an image between projects:
+# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
+# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
+images:
+  fedora:
+    image_family: fedora-coreos-stable
+    project: fedora-coreos-cloud 
+    machine: n1-standard-2
+    metadata: "user-data</workspace/test-infra/jobs/e2e_node/swap/crio_swap1g.ign"

jobs/e2e_node/swap/image-config-swap.yaml

@@ -0,0 +1,9 @@
+# To copy an image between projects:
+# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
+# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
+images:
+  ubuntu:
+    image: ubuntu-gke-2004-1-20-v20210401 # docker 19.03.8 / containerd 1.4.3
+    project: ubuntu-os-gke-cloud
+    metadata: "user-data</workspace/test-infra/jobs/e2e_node/swap/ubuntu-swap-1G-allocation.yaml"
+    machine: n1-standard-2

jobs/e2e_node/swap/ubuntu-swap-1G-allocation.yaml

@@ -0,0 +1,16 @@
+#cloud-config
+
+runcmd:
+  - 'echo current swappiness: $(sysctl vm.swappiness)'
+  - 'echo create swap file'
+  - 'fallocate -l 1G /swapfile || true'
+  - 'chmod 600 /swapfile'
+  - 'mkswap /swapfile'
+  - 'swapon /swapfile'
+  - 'echo "/swapfile swap swap defaults 0 0" | tee -a /etc/fstab'
+  - 'echo swap files'
+  - 'swapon --show'
+  - 'free -h'
+  - 'sudo sed -i "s:console=ttyS0:console=ttyS0 cgroup_enable=memory swapaccount=1:g" /etc/default/grub.d/50-cloudimg-settings.cfg' 
+  - 'sudo update-grub'
+  - 'sudo reboot'