From da24b904f83c79f535cc6abcef8a884e2c8872f3 Mon Sep 17 00:00:00 2001 From: Jose Angel Morena Date: Sun, 23 Jul 2023 14:37:44 +0200 Subject: [PATCH] add example of integration with kubevirt Signed-off-by: Jose Angel Morena --- docs/kubevirt.md | 106 ++++++++++++++++++ molecule/kubevirt/converge.yml | 29 +++++ molecule/kubevirt/create.yml | 96 ++++++++++++++++ molecule/kubevirt/destroy.yml | 25 +++++ molecule/kubevirt/molecule.yml | 45 ++++++++ molecule/kubevirt/requirements.yml | 4 + molecule/kubevirt/tasks/create_vm.yml | 59 ++++++++++ .../kubevirt/tasks/create_vm_dictionary.yml | 26 +++++ 8 files changed, 390 insertions(+) create mode 100644 docs/kubevirt.md create mode 100644 molecule/kubevirt/converge.yml create mode 100644 molecule/kubevirt/create.yml create mode 100644 molecule/kubevirt/destroy.yml create mode 100644 molecule/kubevirt/molecule.yml create mode 100644 molecule/kubevirt/requirements.yml create mode 100644 molecule/kubevirt/tasks/create_vm.yml create mode 100644 molecule/kubevirt/tasks/create_vm_dictionary.yml diff --git a/docs/kubevirt.md b/docs/kubevirt.md new file mode 100644 index 0000000000..89cb0c8c35 --- /dev/null +++ b/docs/kubevirt.md @@ -0,0 +1,106 @@ +# Using Kubevirt + +Below you can see a scenario that is using [Kubevirt VMs](https://kubevirt.io/user-guide/) as test hosts. For Ansible to connect with the SSH in the KubeVirt VMs, it will be made accessible through the Service NodePort. +When you run `molecule test --scenario kubevirt` the `create`, `converge` and +`destroy` steps will be run one after another. + +This example is using Ansible playbooks and it does not need any molecule +plugins to run. You can fully control which test requirements you need to be +installed. + +## Prerequisites + +The `create.yml` and `destroy.yml` Ansible playbooks require the Ansible collection `kubernetes.core`. For seamless communication with the Kubernetes API server, the collection uses the following environment variables: + +- `K8S_AUTH_API_KEY`: This is the token from the service account used to authenticate with the Kubernetes cluster. + +- `K8S_AUTH_HOST`: This points to the URL of the Kubernetes cluster's API. + +- `K8S_AUTH_VERIFY_SSL`: If set to `false`, this disables the verification of SSL/TLS certificates, which might pose a security risk. It's mainly used for testing environments, particularly when dealing with self-signed certificates. + +Additionally, for the playbooks to work, the Kubernetes service account needs specific roles and role bindings to operate in a particular namespace. This ensures the playbook has sufficient privileges to execute commands on the Kubernetes resources. These roles include getting, listing, watching, creating, deleting, and editing virtual machines and services. + +```yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: + namespace: +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: + name: +rules: +- apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["get", "list", "watch", "create", "delete", "patch", "edit"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "delete", "patch", "edit"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: + namespace: +subjects: +- kind: ServiceAccount + name: + namespace: +roleRef: + kind: Role + name: + apiGroup: rbac.authorization.k8s.io +``` +You will need to substitute the following placeholders: +- ``: This refers to the name of the Kubernetes Serviceaccount that the molecule test utilizes to create the KubeVirt VM. +- ``: This denotes the name of the Kubernetes namespace where the VMs will be instantiated. +- ``: This is the name of the Kubernetes role which encapsulates the necessary permissions for the molecule test to function. +- ``: This represents the name of the Kubernetes rolebinding that associates the role `` with the serviceaccount ``. + +## Considerations +- This example is using ephemeral VMs, which enhance the speed of VM creation and cleanup. However, it is important to note that any data in the system will not be retained if the VM is rebooted. +- You don't need to worry about setting up SSH keys. The `create.yml` Ansible playbook takes care of configuring a temporary SSH key. + +## Config playbook + +```yaml title="molecule.yml" +{!../molecule/kubevirt/molecule.yml!} +``` + +Please, replace the following parameters: +- ``: This should be replaced with the namespace in Kubernetes where you intend to create the KubeVirt VMs. +- ``: Change this to the fully qualified domain name (FQDN) of the Kubernetes node that Ansible will attempt to SSH into via the Service NodePort. + +```yaml title="requirements.yml" +{!../molecule/kubevirt/requirements.yml!} +``` + +## Create playbook + +```yaml title="create.yml" +{!../molecule/kubevirt/create.yml!} +``` + +```yaml title="tasks/create_vm.yml" +{!../molecule/kubevirt/tasks/create_vm.yml!} +``` + +```yaml title="tasks/create_vm_dictionary.yml" +{!../molecule/kubevirt/tasks/create_vm_dictionary.yml!} +``` + +## Converge playbook + +```yaml title="converge.yml" +{!../molecule/kubevirt/converge.yml!} +``` + +## Destroy playbook + +```yaml title="destroy.yml" +{!../molecule/kubevirt/destroy.yml!} +``` \ No newline at end of file diff --git a/molecule/kubevirt/converge.yml b/molecule/kubevirt/converge.yml new file mode 100644 index 0000000000..5000a2be97 --- /dev/null +++ b/molecule/kubevirt/converge.yml @@ -0,0 +1,29 @@ +- name: Fail if molecule group is missing + hosts: localhost + tasks: + - name: Print some info + ansible.builtin.debug: + msg: "{{ groups }}" + + - name: Assert group existence + ansible.builtin.assert: + that: "'molecule' in groups" + fail_msg: | + molecule group was not found inside inventory groups: {{ groups }} + +- name: Converge + hosts: molecule + # We disable gather facts because it would fail due to our container not + # having python installed. This will not prevent use from running 'raw' + # commands. Most molecule users are expected to use containers that already + # have python installed in order to avoid notable delays installing it. + gather_facts: false + tasks: + - name: Check uname + ansible.builtin.raw: uname -a + register: result + changed_when: false + + - name: Print some info + ansible.builtin.assert: + that: result.stdout | regex_search("^Linux") diff --git a/molecule/kubevirt/create.yml b/molecule/kubevirt/create.yml new file mode 100644 index 0000000000..f7a6efea6d --- /dev/null +++ b/molecule/kubevirt/create.yml @@ -0,0 +1,96 @@ +- name: Create + hosts: localhost + connection: local + gather_facts: false + vars: + temporary_ssh_key_size: 2048 # Variable for the size of the SSH key + tasks: + - name: Set default SSH key path # Sets the path of the SSH key + set_fact: + tempoary_ssh_key_path: "{{ molecule_ephemeral_directory }}/identity_file" + + - name: Generate SSH key pair # Generates a new SSH key pair + community.crypto.openssh_keypair: + path: "{{ tempoary_ssh_key_path }}" + size: "{{ temporary_ssh_key_size }}" + register: temporary_ssh_keypair # Stores the output of this task in a variable + + - name: Set SSH public key # Sets the SSH public key from the key pair + set_fact: + temporary_ssh_public_key: "{{ temporary_ssh_keypair.public_key }}" + + - name: Create VM in KubeVirt # Calls another file to create the VM in KubeVirt + include_tasks: tasks/create_vm.yml + loop: "{{ molecule_yml.platforms }}" # Loops over all platforms defined in molecule_yml + loop_control: + loop_var: vm # Sets the variable for the current item in the loop + + - name: Create Nodeport service if ssh_type is set to NodePort # Conditional block, executes if vm.ssh_service.type is NodePort + block: + - name: Create ssh NodePort Kubernetes Services # Creates a new NodePort service in Kubernetes + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Service + metadata: + name: "{{ vm.name }}" + namespace: "{{ vm.namespace }}" + spec: + ports: + - port: 22 + protocol: TCP + targetPort: 22 + selector: + kubevirt.io/domain: "{{ vm.name }}" + type: NodePort + loop: "{{ molecule_yml.platforms }}" # Loops over all platforms defined in molecule_yml + loop_control: + loop_var: vm # Sets the variable for the current item in the loop + + - name: Retrieve Service Info # Retrieves information about the service + kubernetes.core.k8s_info: + api_version: v1 + kind: Service + name: "{{ vm.name }}" + namespace: "{{ vm.namespace }}" + loop: "{{ molecule_yml.platforms }}" # Loops over all platforms defined in molecule_yml + loop_control: + loop_var: vm # Sets the variable for the current item in the loop + register: node_port_services # Stores the output of this task in a variable + when: "vm.ssh_service.type == 'NodePort'" # The block is executed when this condition is met + + - name: Create VM dictionary # Calls another file to create a dictionary with information about the VM + include_tasks: tasks/create_vm_dictionary.yml + loop: "{{ molecule_yml.platforms }}" # Loops over all platforms defined in molecule_yml + loop_control: + loop_var: vm # Sets the variable for the current item in the loop + + - name: Create ansible inventory from dictionary # Creates an Ansible inventory file from the dictionary + vars: + molecule_inventory: + all: + children: + molecule: + hosts: "{{ molecule_systems }}" + ansible.builtin.copy: + content: "{{ molecule_inventory | to_nice_yaml }}" + dest: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + mode: 0600 # Sets the permissions of the file to -rw------- + + - name: Refresh inventory # Refreshes the inventory + ansible.builtin.meta: refresh_inventory + + - name: Assert molecule group exists # Checks if the 'molecule' group exists in the inventory + ansible.builtin.assert: + that: "'molecule' in groups" + fail_msg: "Molecule group was not found in inventory groups: {{ groups }}" + run_once: true # Ensures this task is only run once, not on every host in 'hosts' + +- name: Validate that inventory was refreshed # New playbook to validate the inventory + hosts: molecule # Runs on hosts in the 'molecule' group + gather_facts: false # Disables fact gathering + tasks: + - name: Wait for the host to be reachable # Waits for the host to become reachable + ansible.builtin.wait_for_connection: + timeout: 120 # Waits for up to 120 seconds \ No newline at end of file diff --git a/molecule/kubevirt/destroy.yml b/molecule/kubevirt/destroy.yml new file mode 100644 index 0000000000..d95afc05bd --- /dev/null +++ b/molecule/kubevirt/destroy.yml @@ -0,0 +1,25 @@ +--- +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: Delete VM Instance in KubeVirt + kubernetes.core.k8s: + state: absent + kind: VirtualMachine + name: "{{ vm.name }}" + namespace: "{{ vm.namespace }}" + loop: "{{ molecule_yml.platforms }}" + loop_control: + loop_var: vm + + - name: Delete VM Instance in KubeVirt + kubernetes.core.k8s: + state: absent + kind: Service + name: "{{ vm.name }}" + namespace: "{{ vm.namespace }}" + loop: "{{ molecule_yml.platforms }}" + loop_control: + loop_var: vm \ No newline at end of file diff --git a/molecule/kubevirt/molecule.yml b/molecule/kubevirt/molecule.yml new file mode 100644 index 0000000000..950bcf8679 --- /dev/null +++ b/molecule/kubevirt/molecule.yml @@ -0,0 +1,45 @@ +--- +dependency: + name: galaxy + options: + requirements-file: requirements.yml + role-file: requirements.yml +platforms: + - name: rhel9 + image: registry.redhat.io/rhel9/rhel-guest-image + namespace: + ssh_service: + type: NodePort + nodeport_host: + ansible_user: cloud-user + memory: 1Gi + - name: rhel8 + image: registry.redhat.io/rhel8/rhel-guest-image + namespace: + ssh_service: + type: NodePort + nodeport_host: + ansible_user: cloud-user + memory: 1Gi +provisioner: + name: ansible + config_options: + defaults: + interpreter_python: auto_silent + callback_whitelist: profile_tasks, timer, yaml + ssh_connection: + pipelining: false + log: true +verifier: + name: ansible +scenario: + test_sequence: + - dependency + - destroy + - syntax + - create + - converge + - idempotence + - side_effect + - verify + - destroy \ No newline at end of file diff --git a/molecule/kubevirt/requirements.yml b/molecule/kubevirt/requirements.yml new file mode 100644 index 0000000000..b6eac3a1b2 --- /dev/null +++ b/molecule/kubevirt/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: kubernetes.core + - name: community.crypto diff --git a/molecule/kubevirt/tasks/create_vm.yml b/molecule/kubevirt/tasks/create_vm.yml new file mode 100644 index 0000000000..0d6fe17c46 --- /dev/null +++ b/molecule/kubevirt/tasks/create_vm.yml @@ -0,0 +1,59 @@ +--- +- name: Create VM in KubeVirt + kubernetes.core.k8s: # Uses the k8s module from the kubernetes.core Ansible collection + state: present # Ensures the VM exists. If it doesn't, it will be created. + definition: + apiVersion: kubevirt.io/v1 # KubeVirt's API version + kind: VirtualMachine # The type of Kubernetes resource to create + metadata: + labels: + kubevirt.io/domain: "{{ vm.name }}" # Labels for the VM + name: "{{ vm.name }}" # Name of the VM + namespace: "{{ vm.namespace }}" # Namespace where the VM will be created + spec: + running: true # Starts the VM after creation + template: + metadata: + labels: + kubevirt.io/domain: "{{ vm.name }}" # Labels for the VM's template + spec: + domain: + devices: + disks: + - disk: + bus: virtio # Type of disk bus + name: containerdisk # Name of the container disk + - disk: + bus: virtio # Type of disk bus + name: cloudinitdisk # Name of the cloud-init disk + - name: emptydisk # Name of the empty disk + disk: + bus: virtio # Type of disk bus + resources: + requests: + memory: "{{ vm.memory | default('1Gi') }}" # Amount of memory requested for the VM + volumes: + - name: emptydisk + emptyDisk: + capacity: "{{ vm.capacity | default('2Gi') }}" # Capacity of the empty ephemeral disk + - containerDisk: + image: "{{ vm.image }}" # The image used for the container disk + name: containerdisk + - cloudInitNoCloud: # Cloud-init configuration + userData: | # User-data script + #cloud-config + preserve_hostname: true + hostname: "{{ vm.name }}" # Sets the hostname + fqdn: "{{ vm.name }}" # Fully Qualified Domain Name + prefer_fqdn_over_hostname: true + users: + - default + - name: {{ vm.ansible_user }} + lock_passwd: true # Locks the password + ssh_authorized_keys: + - "{{ temporary_ssh_public_key }}" # SSH public key + runcmd: + - [ sh, -c, "hostnamectl set-hostname {{ vm.name }}" ] # Sets the hostname + - [ sudo, yum, install, -y, qemu-guest-agent ] # Installs qemu-guest-agent + - [ sudo, systemctl, start, qemu-guest-agent ] # Starts qemu-guest-agent + name: cloudinitdisk diff --git a/molecule/kubevirt/tasks/create_vm_dictionary.yml b/molecule/kubevirt/tasks/create_vm_dictionary.yml new file mode 100644 index 0000000000..a5d218fc34 --- /dev/null +++ b/molecule/kubevirt/tasks/create_vm_dictionary.yml @@ -0,0 +1,26 @@ +--- +- name: Create VM dictionary + vars: + # This variable block is setting the `ssh_service_address` variable. + # It first checks if the service type of the SSH service is 'NodePort'. + # If it is, it retrieves the 'nodePort' from the services results. + ssh_service_address: >- + {%- set svc_type = vm.ssh_service.type | default(None) -%} + {%- if svc_type == 'NodePort' -%} + {{(node_port_services.results | selectattr('vm.name','==',vm.name) | first)['resources'][0]['spec']['ports'][0]['nodePort'] }} + {%- endif -%} + set_fact: + # Here, the task is updating the `molecule_systems` dictionary with new VM information. + # If `molecule_systems` doesn't exist, it is created as an empty dictionary. + # Then it is combined with a new dictionary for the current VM, containing ansible connection details. + molecule_systems: >- + {{ + molecule_systems | default({}) | combine({ + vm.name: { + 'ansible_user': 'cloud-user', + 'ansible_host': vm.ssh_service.nodeport_host, + 'ansible_ssh_port': ssh_service_address, + 'ansible_ssh_private_key_file': tempoary_ssh_key_path + } + }) + }}