From c7139f93d4ad9af0e31eb733771803d5b4597407 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Wed, 23 Jun 2021 11:48:26 +0200 Subject: [PATCH] Add aws image generation based of the raw image Uses the new cos-deploy script to install the image from recovery and the user-data facility to partition the disk Signed-off-by: Itxaka --- .github/build.yaml.gomplate | 4 +- .github/workflows/build-master.yaml | 8 +- .github/workflows/build-nightly.yaml | 8 +- .github/workflows/build-pr.yaml | 8 +- make/Makefile.iso | 6 +- packer/aws/readme.md | 29 +++++++ packer/aws/setup-disk.yaml | 13 +++ packer/images.json | 100 ----------------------- packer/images.json.pkr.hcl | 115 +++++++++++++++++++++++++++ packer/variables.pkr.hcl | 87 ++++++++++++++++++++ 10 files changed, 263 insertions(+), 115 deletions(-) create mode 100644 packer/aws/readme.md create mode 100644 packer/aws/setup-disk.yaml delete mode 100644 packer/images.json create mode 100644 packer/images.json.pkr.hcl create mode 100644 packer/variables.pkr.hcl diff --git a/.github/build.yaml.gomplate b/.github/build.yaml.gomplate index 8d7ab1fadbe..c87eca0fad9 100644 --- a/.github/build.yaml.gomplate +++ b/.github/build.yaml.gomplate @@ -220,7 +220,7 @@ jobs: brew install qemu - name: Build QEMU Image 🔧 run: | - PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu" make packer + PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-{{{$subset}}}-{{{ $flavor }}}.qcow @@ -249,7 +249,7 @@ jobs: # brew install hashicorp/tap/packer - name: Build VBox Image 🔧 run: | - PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso" make packer + PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-{{{$subset}}}-{{{ $flavor }}}.ova diff --git a/.github/workflows/build-master.yaml b/.github/workflows/build-master.yaml index 8368910c855..dd5fbad74e1 100644 --- a/.github/workflows/build-master.yaml +++ b/.github/workflows/build-master.yaml @@ -195,7 +195,7 @@ jobs: brew install qemu - name: Build QEMU Image 🔧 run: | - PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu" make packer + PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-squashfs-opensuse.qcow @@ -224,7 +224,7 @@ jobs: # brew install hashicorp/tap/packer - name: Build VBox Image 🔧 run: | - PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso" make packer + PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-squashfs-opensuse.ova @@ -348,7 +348,7 @@ jobs: brew install qemu - name: Build QEMU Image 🔧 run: | - PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu" make packer + PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-nonsquashfs-opensuse.qcow @@ -377,7 +377,7 @@ jobs: # brew install hashicorp/tap/packer - name: Build VBox Image 🔧 run: | - PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso" make packer + PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-nonsquashfs-opensuse.ova diff --git a/.github/workflows/build-nightly.yaml b/.github/workflows/build-nightly.yaml index 9aadcc3490c..6d5f0d36530 100644 --- a/.github/workflows/build-nightly.yaml +++ b/.github/workflows/build-nightly.yaml @@ -174,7 +174,7 @@ jobs: brew install qemu - name: Build QEMU Image 🔧 run: | - PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu" make packer + PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-squashfs-opensuse.qcow @@ -203,7 +203,7 @@ jobs: # brew install hashicorp/tap/packer - name: Build VBox Image 🔧 run: | - PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso" make packer + PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-squashfs-opensuse.ova @@ -327,7 +327,7 @@ jobs: brew install qemu - name: Build QEMU Image 🔧 run: | - PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu" make packer + PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-nonsquashfs-opensuse.qcow @@ -356,7 +356,7 @@ jobs: # brew install hashicorp/tap/packer - name: Build VBox Image 🔧 run: | - PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso" make packer + PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-nonsquashfs-opensuse.ova diff --git a/.github/workflows/build-pr.yaml b/.github/workflows/build-pr.yaml index 66de3e91f6c..208eaf2efbb 100644 --- a/.github/workflows/build-pr.yaml +++ b/.github/workflows/build-pr.yaml @@ -180,7 +180,7 @@ jobs: brew install qemu - name: Build QEMU Image 🔧 run: | - PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu" make packer + PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-squashfs-opensuse.qcow @@ -209,7 +209,7 @@ jobs: # brew install hashicorp/tap/packer - name: Build VBox Image 🔧 run: | - PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso" make packer + PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-squashfs-opensuse.ova @@ -333,7 +333,7 @@ jobs: brew install qemu - name: Build QEMU Image 🔧 run: | - PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu" make packer + PACKER_ARGS="-var='accelerator=hvf' -var='feature=vagrant' -only qemu.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-nonsquashfs-opensuse.qcow @@ -362,7 +362,7 @@ jobs: # brew install hashicorp/tap/packer - name: Build VBox Image 🔧 run: | - PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso" make packer + PACKER_ARGS="-var='feature=vagrant' -only virtualbox-iso.cos" make packer - uses: actions/upload-artifact@v2 with: name: cOS-nonsquashfs-opensuse.ova diff --git a/make/Makefile.iso b/make/Makefile.iso index 6e7d3334636..308a79620f9 100644 --- a/make/Makefile.iso +++ b/make/Makefile.iso @@ -125,4 +125,8 @@ ifeq ("$(ISO)","") @echo "Please run 'make iso' or 'make local-iso' first" @exit 1 endif - cd $(ROOT_DIR)/packer && $(PACKER) build -var "iso=$(ISO)" $(PACKER_ARGS) images.json + cd $(ROOT_DIR)/packer && $(PACKER) build -var "iso=$(ISO)" $(PACKER_ARGS) . + + +packer-aws: + cd $(ROOT_DIR)/packer && $(PACKER) build -only amazon-ebs.cos . diff --git a/packer/aws/readme.md b/packer/aws/readme.md new file mode 100644 index 00000000000..cac8e7d51df --- /dev/null +++ b/packer/aws/readme.md @@ -0,0 +1,29 @@ +# steps + + - build cos (sudo make build) + - create repo (sudo make create-repo) + - create raw image (sudo make raw_disk) + - upload raw image to s3 (aws s3 cp IMAGE s3://cos-images/) + - import image as snapshot (aws ec2 import-snapshot --description "Cos raw import" --disk-container "file://containers.json") + +containers.json: +```json +{ + "Description": "Example image originally in raw format", + "Format": "raw", + "UserBucket": { + "S3Bucket": "cos-images", + "S3Key": "IMAGE_NAME" + } +} +``` + + - create ami from snapshot in aws console + - launch packer with aws image creation (make packer-aws) + + +# image creation + + - Set proper disk partitions (aws/setup-disk.yaml) + - cos-deploy: + `cos-deploy --docker-image quay.io/costoolkit/releases-opensuse:cos-system-0.5.3-3` \ No newline at end of file diff --git a/packer/aws/setup-disk.yaml b/packer/aws/setup-disk.yaml new file mode 100644 index 00000000000..950e2c1d7ef --- /dev/null +++ b/packer/aws/setup-disk.yaml @@ -0,0 +1,13 @@ +name: "Default deployment" +stages: + rootfs.after: + - name: "Repart image" + layout: + device: + label: COS_RECOVERY + add_partitions: + - fsLabel: COS_STATE + size: 9192 + pLabel: state + - fsLabel: COS_PERSISTENT + pLabel: persistent \ No newline at end of file diff --git a/packer/images.json b/packer/images.json deleted file mode 100644 index 0ee0c05ba36..00000000000 --- a/packer/images.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "builders": [ - { - "boot_wait": "{{user `sleep`}}", - "disk_size": "{{user `disk_size`}}", - "guest_additions_mode": "disable", - "guest_os_type": "cOS", - "headless": true, - "iso_url": "{{user `iso`}}", - "iso_checksum": "none", - "shutdown_command": "shutdown -hP now", - "cpus": "{{ user `cpus` }}", - "memory": "{{ user `memory` }}", - "ssh_password": "{{user `root_password`}}", - "ssh_username": "{{user `root_username`}}", - "format": "ova", - "ssh_timeout": "5m", - "ssh_handshake_attempts": "20", - "type": "virtualbox-iso", - "vm_name": "cOS" - }, - { - "boot_wait": "{{user `sleep`}}", - "disk_interface": "ide", - "disk_size": "{{user `disk_size`}}", - "format": "qcow2", - "accelerator": "{{user `accelerator`}}", - "headless": true, - "iso_url": "{{user `iso`}}", - "cpus": "{{ user `cpus` }}", - "iso_checksum": "none", - "qemuargs": [ - [ - "-m", - "{{ user `memory` }}M" - ] - ], - "shutdown_command": "shutdown -hP now", - "ssh_password": "{{user `root_password`}}", - "ssh_timeout": "5m", - "ssh_handshake_attempts": "20", - "ssh_username": "{{user `root_username`}}", - "type": "qemu", - "vm_name": "cOS" - } - ], - "description": "cOS", - "post-processors": [ - { - "only": [ - "virtualbox-iso", - "qemu" - ], - "output": "cOS_{{user `build`}}_{{user `arch`}}_{{user `flavor`}}.box", - "type": "vagrant" - }, - { - "only": [ - "virtualbox-iso", - "qemu" - ], - "output": "cOS_{{user `build`}}_{{user `arch`}}_{{user `flavor`}}.tar.gz", - "type": "compress" - } - ], - "provisioners": [ - { - "destination": "/90_custom.yaml", - "source": "config.yaml", - "type": "file" - }, - { - "destination": "/vagrant.yaml", - "source": "vagrant.yaml", - "type": "file" - }, - { - "inline": [ - "INTERACTIVE=false cos-installer --config /90_custom.yaml /dev/sda", - "if [ -n \"{{user `feature`}}\" ]; then mount /dev/disk/by-label/COS_OEM /oem; cos-feature enable {{user `feature`}}; fi" - ], - "pause_after": "30s", - "type": "shell" - } - ], - "variables": { - "arch": "amd64", - "build": "dev", - "disk_size": "50000", - "cpus": "3", - "memory": "8192", - "flavor": "leap", - "root_password": "cos", - "root_username": "root", - "iso": "", - "sleep": "30s", - "accelerator": "kvm", - "feature": "" - } - } diff --git a/packer/images.json.pkr.hcl b/packer/images.json.pkr.hcl new file mode 100644 index 00000000000..974a81e3351 --- /dev/null +++ b/packer/images.json.pkr.hcl @@ -0,0 +1,115 @@ +source "amazon-ebs" "cos" { + access_key = var.aws_access_key + ami_name = "cos-${var.cos_version}-${formatdate("DDMMYYYY", timestamp())}" + ami_description = "cos-${var.cos_version}-${formatdate("DDMMYYYY", timestamp())}" + instance_type = "t3.small" + region = var.aws_region + secret_key = var.aws_secret_key + ssh_password = "cos" + ssh_username = "root" + source_ami_filter { + filters = { + name = "*cos*recovery*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["self"] + } + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 15 + } + user_data_file = "aws/setup-disk.yaml" + tags = { + Name = "cOS" + Version = var.cos_version + Base_AMI_ID = "{{ .SourceAMI }}" + Base_AMI_Name = "{{ .SourceAMIName }}" + } +} + +source "qemu" "cos" { + accelerator = "${var.accelerator}" + boot_wait = "${var.sleep}" + cpus = "${var.cpus}" + disk_interface = "ide" + disk_size = "${var.disk_size}" + format = "qcow2" + headless = true + iso_checksum = "none" + iso_url = "${var.iso}" + qemuargs = [["-m", "${var.memory}M"]] + shutdown_command = "shutdown -hP now" + ssh_handshake_attempts = "20" + ssh_password = "${var.root_password}" + ssh_timeout = "5m" + ssh_username = "${var.root_username}" + vm_name = "cOS" +} + +source "virtualbox-iso" "cos" { + boot_wait = "${var.sleep}" + cpus = "${var.cpus}" + disk_size = "${var.disk_size}" + format = "ova" + guest_additions_mode = "disable" + guest_os_type = "cOS" + headless = true + iso_checksum = "none" + iso_url = "${var.iso}" + memory = "${var.memory}" + shutdown_command = "shutdown -hP now" + ssh_handshake_attempts = "20" + ssh_password = "${var.root_password}" + ssh_timeout = "5m" + ssh_username = "${var.root_username}" + vm_name = "cOS" +} + +# a build block invokes sources and runs provisioning steps on them. The +# documentation for build blocks can be found here: +# https://www.packer.io/docs/templates/hcl_templates/blocks/build +build { + description = "cOS" + + sources = ["source.amazon-ebs.cos", "source.qemu.cos", "source.virtualbox-iso.cos"] + + provisioner "file" { + except = ["amazon-ebs.cos"] + destination = "/90_custom.yaml" + source = "config.yaml" + } + + provisioner "file" { + except = ["amazon-ebs.cos"] + destination = "/vagrant.yaml" + source = "vagrant.yaml" + } + + provisioner "shell" { + except = ["amazon-ebs.cos"] + inline = ["INTERACTIVE=false cos-installer --config /90_custom.yaml /dev/sda", + "if [ -n \"${var.feature}\" ]; then mount /dev/disk/by-label/COS_OEM /oem; cos-feature enable ${var.feature}; fi" + ] + pause_after = "30s" + } + + provisioner "shell" { + only = ["amazon-ebs.cos"] + inline = [ + "${var.aws_cos_install_args}", + "sync" + ] + pause_after = "30s" + } + + post-processor "vagrant" { + only = ["virtualbox-iso.cos", "qemu.cos"] + output = "cOS_${var.build}_${var.arch}_${var.flavor}.box" + } + post-processor "compress" { + only = ["virtualbox-iso.cos", "qemu.cos"] + output = "cOS_${var.build}_${var.arch}_${var.flavor}.tar.gz" + } +} diff --git a/packer/variables.pkr.hcl b/packer/variables.pkr.hcl new file mode 100644 index 00000000000..e65b75bc8a6 --- /dev/null +++ b/packer/variables.pkr.hcl @@ -0,0 +1,87 @@ +variable "accelerator" { + type = string + default = "kvm" +} + +variable "arch" { + type = string + default = "amd64" +} + +variable "aws_access_key" { + type = string + default = env("AWS_ACCESS_KEY_ID") + sensitive = true +} + +variable "aws_secret_key" { + type = string + default = env("AWS_SECRET_ACCESS_KEY") + sensitive = true +} + +variable "build" { + type = string + default = "dev" +} + +variable "cos_version" { + type = string + default = "0.5.5" +} + +variable "aws_cos_install_args" { + type = string + default = "cos-deploy --docker-image quay.io/costoolkit/releases-opensuse:cos-system-0.5.5" +} + +variable "cpus" { + type = string + default = "3" +} + +variable "disk_size" { + type = string + default = "50000" +} + +variable "feature" { + type = string + default = "" +} + +variable "flavor" { + type = string + default = "leap" +} + +variable "iso" { + type = string + default = "" +} + +variable "memory" { + type = string + default = "8192" +} + +variable "aws_region" { + type = string + default = env("AWS_DEFAULT_REGION") + sensitive = true +} + +variable "root_password" { + type = string + default = "cos" +} + +variable "root_username" { + type = string + default = "root" +} + +variable "sleep" { + type = string + default = "30s" +} \ No newline at end of file