diff --git a/01_install_requirements.sh b/01_install_requirements.sh index be510eb64..a39e7da72 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -45,7 +45,8 @@ fi sudo yum -y update # make sure additional requirments are installed -sudo yum install -y bind-utils ansible python-netaddr python-virtualbmc libvirt libvirt-devel libvirt-daemon-kvm qemu-kvm virt-install jq python-ironicclient python-ironic-inspector-client python-openstackclient +sudo yum install -y bind-utils ansible python-netaddr python-virtualbmc libvirt libvirt-devel libvirt-daemon-kvm qemu-kvm virt-install jq \ + python-ironicclient python-ironic-inspector-client python-openstackclient podman if [ ! -f $HOME/.ssh/id_rsa.pub ]; then ssh-keygen -f ~/.ssh/id_rsa -P "" diff --git a/02_configure_host.sh b/02_configure_host.sh index f08fdb887..d44d9292d 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -36,7 +36,9 @@ EOF fi # Allow ipmi to the virtual bmc processes that we just started -sudo iptables -I INPUT -i baremetal -p udp -m udp --dport 6230:6235 -j ACCEPT +if ! sudo iptables -C INPUT -i baremetal -p udp -m udp --dport 6230:6235 -j ACCEPT ; then + sudo iptables -I INPUT -i baremetal -p udp -m udp --dport 6230:6235 -j ACCEPT +fi # Need to route traffic from the provisioning host. if [ "$EXT_IF" ]; then diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh new file mode 100755 index 000000000..7df1daf0b --- /dev/null +++ b/04_setup_ironic.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -ex + +source common.sh + +# Get the various images +source get_images.sh + +# ironic dnsmasq and ipxe config +cp ironic/dnsmasq.conf $IRONIC_DATA_DIR/ +cp ironic/dualboot.ipxe ironic/inspector.ipxe $IRONIC_DATA_DIR/html/ + +# Either pull or build the ironic images +# To build the IRONIC image set +# IRONIC_IMAGE=https://github.com/metalkube/metalkube-ironic +for IMAGE_VAR in IRONIC_IMAGE IRONIC_INSPECTOR_IMAGE ; do + IMAGE=${!IMAGE_VAR} + # Is it a git repo? + if [[ "$IMAGE" =~ "://" ]] ; then + REPOPATH=~/${IMAGE##*/} + # Clone to ~ if not there already + [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH + cd $REPOPATH + export $IMAGE_VAR=localhost/${IMAGE##*/}:latest + sudo podman build -t ${!IMAGE_VAR} . + cd - + else + sudo podman pull "$IMAGE" + fi +done + +# Adding an IP address in the libvirt definition for this network results in +# dnsmasq being run, we don't want that as we have our own dnsmasq, so set +# the IP address here +if [ ! -e /etc/sysconfig/network-scripts/ifcfg-brovc ] ; then + echo -e "DEVICE=brovc\nONBOOT=yes\nNM_CONTROLLED=no\nTYPE=Ethernet\nBOOTPROTO=static\nIPADDR=172.22.0.1\nNETMASK=255.255.255.0" | sudo dd of=/etc/sysconfig/network-scripts/ifcfg-brovc + sudo ifdown brovc || true + sudo ifup brovc +fi + +# Workaround so that the dracut network module does dhcp on eth0 & eth1 +RHCOS_IMAGE_FILENAME_RAW="${RHCOS_IMAGE_FILENAME_OPENSTACK}.raw" +if [ ! -e "$IRONIC_DATA_DIR/html/images/$RHCOS_IMAGE_FILENAME_DUALDHCP" ] ; then + pushd $IRONIC_DATA_DIR/html/images + qemu-img convert "$RHCOS_IMAGE_FILENAME_OPENSTACK" "${RHCOS_IMAGE_FILENAME_RAW}" + LOOPBACK=$(sudo losetup --show -f "${RHCOS_IMAGE_FILENAME_RAW}" | cut -f 3 -d /) + mkdir -p /tmp/mnt + sudo kpartx -a /dev/$LOOPBACK + sudo mount /dev/mapper/${LOOPBACK}p1 /tmp/mnt + sudo sed -i -e 's/ip=eth0:dhcp/ip=eth0:dhcp ip=eth1:dhcp/g' /tmp/mnt/grub2/grub.cfg + sudo umount /tmp/mnt + sudo kpartx -d /dev/${LOOPBACK} + sudo losetup -d /dev/${LOOPBACK} + qemu-img convert -O qcow2 -c "$RHCOS_IMAGE_FILENAME_RAW" "$RHCOS_IMAGE_FILENAME_DUALDHCP" + rm "$RHCOS_IMAGE_FILENAME_RAW" + popd +fi + + +for name in ironic ironic-inspector ; do + sudo podman ps | grep -w "$name$" && sudo podman kill $name + sudo podman ps --all | grep -w "$name$" && sudo podman rm $name +done + +# Start Ironic and inspector +sudo podman run -d --net host --privileged --name ironic \ + -v $IRONIC_DATA_DIR/dnsmasq.conf:/etc/dnsmasq.conf \ + -v $IRONIC_DATA_DIR/html/images:/var/www/html/images \ + -v $IRONIC_DATA_DIR/html/dualboot.ipxe:/var/www/html/dualboot.ipxe \ + -v $IRONIC_DATA_DIR/html/inspector.ipxe:/var/www/html/inspector.ipxe ${IRONIC_IMAGE} +sudo podman run -d --net host --privileged --name ironic-inspector "${IRONIC_INSPECTOR_IMAGE}" diff --git a/04_build_ocp_installer.sh b/05_build_ocp_installer.sh similarity index 100% rename from 04_build_ocp_installer.sh rename to 05_build_ocp_installer.sh diff --git a/05_deploy_bootstrap_vm.sh b/06_deploy_bootstrap_vm.sh similarity index 70% rename from 05_deploy_bootstrap_vm.sh rename to 06_deploy_bootstrap_vm.sh index c1505ce51..6e7e04fd7 100755 --- a/05_deploy_bootstrap_vm.sh +++ b/06_deploy_bootstrap_vm.sh @@ -4,7 +4,6 @@ set -e source ocp_install_env.sh source common.sh -source get_images.sh source utils.sh # FIXME this is configuring for the libvirt backend which is dev-only ref @@ -98,46 +97,6 @@ sudo systemctl reload NetworkManager # Wait for ssh to start while ! ssh -o "StrictHostKeyChecking=no" core@$IP id ; do sleep 5 ; done -# ironic dnsmasq and ipxe config -cat ironic/dnsmasq.conf | ssh -o "StrictHostKeyChecking=no" core@$IP sudo dd of=dnsmasq.conf -cat ironic/dualboot.ipxe | ssh -o "StrictHostKeyChecking=no" core@$IP sudo dd of=dualboot.ipxe -cat ironic/inspector.ipxe | ssh -o "StrictHostKeyChecking=no" core@$IP sudo dd of=inspector.ipxe - -# Workaround so that the dracut network module does dhcp on eth0 & eth1 -if [ ! -e images/redhat-coreos-maipo-47.284-openstack_dualdhcp.qcow2 ] ; then - qemu-img convert images/redhat-coreos-maipo-47.284-openstack.qcow2 images/redhat-coreos-maipo-47.284-openstack.raw - LOOPBACK=$(sudo losetup --show -f images/redhat-coreos-maipo-47.284-openstack.raw | cut -f 3 -d /) - mkdir -p /tmp/mnt - sudo kpartx -a /dev/$LOOPBACK - sudo mount /dev/mapper/${LOOPBACK}p1 /tmp/mnt - sudo sed -i -e 's/ip=eth0:dhcp/ip=eth0:dhcp ip=eth1:dhcp/g' /tmp/mnt/grub2/grub.cfg - sudo umount /tmp/mnt - sudo kpartx -d /dev/${LOOPBACK} - sudo losetup -d /dev/${LOOPBACK} - qemu-img convert -O qcow2 -c images/redhat-coreos-maipo-47.284-openstack.raw images/redhat-coreos-maipo-47.284-openstack_dualdhcp.qcow2 - rm images/redhat-coreos-maipo-47.284-openstack.raw -fi - -# Copy images the bootstrap node -tar -cf - images | ssh -o "StrictHostKeyChecking=no" "core@$IP" tar -xf - - -# Retrieve and start the ironic container -IRONIC_IMAGE=${IRONIC_IMAGE:-"quay.io/metalkube/metalkube-ironic"} -echo -e "RHCOS_IMAGE_FILENAME_OPENSTACK=${RHCOS_IMAGE_FILENAME_OPENSTACK}\nIRONIC_IMAGE=${IRONIC_IMAGE}" \ - | ssh -o "StrictHostKeyChecking=no" core@$IP sudo dd of=/etc/ironicservice - -# Now that we have the Environment and the image, we can pull the image and start the ironic service -ssh -o "StrictHostKeyChecking=no" core@$IP sudo podman pull "$IRONIC_IMAGE" -ssh -o "StrictHostKeyChecking=no" core@$IP sudo systemctl start ironic.service - -# Retrieve and start the inspector container -IRONIC_INSPECTOR_IMAGE=${IRONIC_INSPECTOR_IMAGE:-"quay.io/metalkube/metalkube-ironic-inspector"} -ssh -o "StrictHostKeyChecking=no" "core@$IP" sudo podman pull "${IRONIC_INSPECTOR_IMAGE}" - -ssh -o "StrictHostKeyChecking=no" core@$IP sudo podman run \ - -d --net host --privileged --name ironic-inspector \ - "${IRONIC_INSPECTOR_IMAGE}" - # Create a master_nodes.json file jq '.nodes[0:3] | {nodes: .}' "${NODES_FILE}" | tee "${MASTER_NODES_FILE}" diff --git a/06_deploy_masters.sh b/07_deploy_masters.sh similarity index 75% rename from 06_deploy_masters.sh rename to 07_deploy_masters.sh index d84d1023c..3443eaec0 100755 --- a/06_deploy_masters.sh +++ b/07_deploy_masters.sh @@ -7,7 +7,7 @@ source common.sh # Note This logic will likely run in a container (on the bootstrap VM) # for the final solution, but for now we'll prototype the workflow here export OS_TOKEN=fake-token -export OS_URL=http://api.ostest.test.metalkube.org:6385/ +export OS_URL=http://localhost:6385/ wait_for_json ironic \ "${OS_URL}/v1/nodes" \ @@ -29,7 +29,7 @@ cp ocp/master.ign configdrive/openstack/latest/user_data for node in $(jq -r .nodes[].name $MASTER_NODES_FILE); do # FIXME(shardy) we should parameterize the image - openstack baremetal node set $node --instance-info image_source=http://172.22.0.1/images/redhat-coreos-maipo-47.284-openstack_dualdhcp.qcow2 --instance-info image_checksum=$(md5sum images/redhat-coreos-maipo-47.284-openstack_dualdhcp.qcow2 | awk '{print $1}') --instance-info root_gb=25 --property root_device="{\"name\": \"$ROOT_DISK\"}" + openstack baremetal node set $node --instance-info "image_source=http://172.22.0.1/images/$RHCOS_IMAGE_FILENAME_DUALDHCP" --instance-info image_checksum=$(md5sum "$IRONIC_DATA_DIR/html/images/$RHCOS_IMAGE_FILENAME_DUALDHCP" | awk '{print $1}') --instance-info root_gb=25 --property root_device="{\"name\": \"$ROOT_DISK\"}" openstack baremetal node manage $node --wait openstack baremetal node provide $node --wait done diff --git a/Makefile b/Makefile index 8bceb5e25..b00cb23e4 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -.PHONY: default requirements configure repo_sync build ocp_run clean ocp_cleanup libvirt_cleanup -default: requirements configure repo_sync build ocp_run +.PHONY: default requirements configure repo_sync ironic build ocp_run clean ocp_cleanup host_cleanup +default: requirements configure repo_sync ironic build ocp_run requirements: ./01_install_requirements.sh @@ -10,18 +10,21 @@ configure: repo_sync: ./03_ocp_repo_sync.sh +ironic: + ./04_setup_ironic.sh + build: - ./04_build_ocp_installer.sh + ./05_build_ocp_installer.sh ocp_run: - ./05_deploy_bootstrap_vm.sh - ./06_deploy_masters.sh + ./06_deploy_bootstrap_vm.sh + ./07_deploy_masters.sh -clean: ocp_cleanup libvirt_cleanup +clean: ocp_cleanup host_cleanup ocp_cleanup: ./ocp_cleanup.sh -libvirt_cleanup: - ./libvirt_cleanup.sh +host_cleanup: + ./host_cleanup.sh diff --git a/README.md b/README.md index 800b94015..1b1dd8335 100644 --- a/README.md +++ b/README.md @@ -49,12 +49,16 @@ server with: $ go run "${GOPATH}/src/github.com/metalkube/facet/main.go" server ``` -- `./04_build_ocp_installer.sh` +- `./04_setup_ironic.sh` + +This will setup Ironic on the host server and download the resources it requires + +- `./05_build_ocp_installer.sh` These will pull and build the openshift-install and some other things from source. -- `./05_deploy_bootstrap_vm.sh` +- `./06_deploy_bootstrap_vm.sh` This will run the openshift-install to generate ignition configs and boot the bootstrap VM, including a bootstrap ironic all in one container. @@ -70,7 +74,7 @@ Then you can interact with the k8s API on the bootstrap VM e.g You can also see the status of the bootkube.sh script which is running via `journalctl -b -f -u bootkube.service`. -- `./06_deploy_masters.sh` +- `./07_deploy_masters.sh` This will deploy the master nodes via ironic, using the Ignition config generated in the previous step. @@ -84,17 +88,11 @@ openstack baremetal node list ... ``` -To ssh to the master nodes, you can route trafic through the bootstrap node -``` -sudo ip route add 172.22.0.0/24 via $(getent hosts api.ostest.test.metalkube.org | grep 192 | awk '{ print $1 }') -ssh core@etcd-.ostest.test.metalkube.org -``` - ## Cleanup - To clean up the ocp deployment run `./ocp_cleanup.sh` -- To clean up the dummy baremetal VMs and associated libvirt resources run `./libvirt_cleanup.sh` +- To clean up the dummy baremetal VMs and associated libvirt resources run `./host_cleanup.sh` e.g. to clean and re-install ocp run: diff --git a/common.sh b/common.sh index ba85f17a5..ccb8f1de5 100644 --- a/common.sh +++ b/common.sh @@ -57,6 +57,12 @@ export RHCOS_IMAGE_NAME="redhat-coreos-maipo-${RHCOS_IMAGE_VERSION}" # config drive support, or perhaps a completely new image? export RHCOS_IMAGE_FILENAME="${RHCOS_IMAGE_NAME}-qemu.qcow2" export RHCOS_IMAGE_FILENAME_OPENSTACK="${RHCOS_IMAGE_NAME}-openstack.qcow2" +export RHCOS_IMAGE_FILENAME_DUALDHCP="${RHCOS_IMAGE_NAME}-dualdhcp.qcow2" + +# Ironic vars +export IRONIC_IMAGE=${IRONIC_IMAGE:-"quay.io/metalkube/metalkube-ironic"} +export IRONIC_INSPECTOR_IMAGE=${IRONIC_INSPECTOR_IMAGE:-"quay.io/metalkube/metalkube-ironic-inspector"} +export IRONIC_DATA_DIR="$WORKING_DIR/ironic" # Log output automatically LOGDIR="$(dirname $0)/logs" diff --git a/config_example.sh b/config_example.sh index ab627a9f0..834672635 100644 --- a/config_example.sh +++ b/config_example.sh @@ -3,3 +3,7 @@ # Get a valid pull secret (json string) from # You can get this secret from https://cloud.openshift.com/clusters/install#pull-secret export PULL_SECRET='' + +# Uncomment to build a copy of ironic or inspector locally +#export IRONIC_INSPECTOR_IMAGE=https://github.com/metalkube/metalkube-ironic-inspector +#export IRONIC_IMAGE=https://github.com/metalkube/metalkube-ironic diff --git a/get_images.sh b/get_images.sh index 62b09c99f..6266cf9ee 100755 --- a/get_images.sh +++ b/get_images.sh @@ -7,9 +7,15 @@ if [ ! -f "$RHCOS_IMAGE_FILENAME" ]; then curl --insecure --compressed -L -o "${RHCOS_IMAGE_FILENAME}" "${RHCOS_IMAGE_URL}/${RHCOS_IMAGE_VERSION}/${RHCOS_IMAGE_FILENAME}".gz fi -mkdir -p images -pushd images +mkdir -p "$IRONIC_DATA_DIR/html/images" +# Move images from the old directory to the new one if we have already downloaded them +# TODO: delete this in a week or so +if [ -d images ] ; then + find images -type f -exec mv {} "$IRONIC_DATA_DIR/html/images/" \; + rmdir images +fi +pushd "$IRONIC_DATA_DIR/html/images" if [ ! -f "${RHCOS_IMAGE_FILENAME_OPENSTACK}" ]; then curl --insecure --compressed -L -o "${RHCOS_IMAGE_FILENAME_OPENSTACK}" "${RHCOS_IMAGE_URL}/${RHCOS_IMAGE_VERSION}/${RHCOS_IMAGE_FILENAME_OPENSTACK}".gz fi diff --git a/libvirt_cleanup.sh b/host_cleanup.sh similarity index 73% rename from libvirt_cleanup.sh rename to host_cleanup.sh index be6584b72..c11223496 100755 --- a/libvirt_cleanup.sh +++ b/host_cleanup.sh @@ -3,6 +3,12 @@ set -xe source common.sh +# Kill and remove the running ironic +for name in ironic ironic-inspector ; do + sudo podman ps | grep -w "$name$" && sudo podman kill $name + sudo podman ps --all | grep -w "$name$" && sudo podman rm $name +done + ANSIBLE_FORCE_COLOR=true ansible-playbook \ -e "working_dir=$WORKING_DIR" \ -e "local_working_dir=$HOME/.quickstart" \ diff --git a/ignition_patches/bootstrap/03_ironic.json b/ignition_patches/bootstrap/03_ironic.json deleted file mode 100644 index cc00a2029..000000000 --- a/ignition_patches/bootstrap/03_ironic.json +++ /dev/null @@ -1 +0,0 @@ -[{"op": "add", "path": "/storage/files/-", "value": {"filesystem": "root", "path": "/etc/sysconfig/network-scripts/ifcfg-eth1", "user": {"name": "root"}, "contents": {"source": "data:text/plain;charset=utf-8;base64,REVWSUNFPWV0aDEKT05CT09UPXllcwpUWVBFPUV0aGVybmV0CkJPT1RQUk9UTz1zdGF0aWMKSVBBRERSPTE3Mi4yMi4wLjEKTkVUTUFTSz0yNTUuMjU1LjI1NS4wCg==", "verification": {}}, "mode": 420}}, {"op": "add", "path": "/systemd/units/-", "value": {"contents": "[Unit]\nDescription=Run Ironic baremetal provisioning\nWants=network-online.target\nAfter=network-online.target\nConditionPathExistsGlob=|/home/core/images/redhat-coreos-maipo-*.qcow2\n\n[Service]\nEnvironmentFile=/etc/ironicservice\nExecStartPre=-/usr/bin/podman pull ${IRONIC_IMAGE}\nExecStartPre=-/usr/bin/podman rm ironic\nExecStart=/usr/bin/podman run --name ironic --net host --privileged -v /home/core/dnsmasq.conf:/etc/dnsmasq.conf -v /home/core/dualboot.ipxe:/var/www/html/dualboot.ipxe -v /home/core/inspector.ipxe:/var/www/html/inspector.ipxe -v /home/core/images:/var/www/html/images ${IRONIC_IMAGE}\nExecReload=-/usr/bin/podman stop ironic\nExecReload=-/usr/bin/podman rm ironic\nRestart=on-failure\nRestartSec=30\n", "enabled": false, "name": "ironic.service"}}] diff --git a/ironic/dnsmasq.conf b/ironic/dnsmasq.conf index 4ecf9094a..b9993d917 100644 --- a/ironic/dnsmasq.conf +++ b/ironic/dnsmasq.conf @@ -1,4 +1,6 @@ -interface=eth1 +interface=brovc +except-interface=lo +bind-dynamic dhcp-range=172.22.0.10,172.22.0.100 enable-tftp tftp-root=/tftpboot