diff --git a/.config/dictionary.txt b/.config/dictionary.txt index 1e36a18d..f6a3caca 100644 --- a/.config/dictionary.txt +++ b/.config/dictionary.txt @@ -21,6 +21,7 @@ endgroup gunicorn libera libonig +lightspeed microdnf mknod modifyitems diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 975508c7..8a674900 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -42,7 +42,10 @@ jobs: devspaces devel ee-amd64:tox -e ee:runner=devtools-multiarch-builder - ee-arm64:tox -e ee:runner=ubuntu-24.04-arm64-2core + ee-arm64:tox -e ee:runner=ubuntu-24.04-arm + selenium-amd64:tox -e selenium:runner=devtools-multiarch-builder + selenium-arm64:tox -e selenium:runner=ubuntu-24.04-arm + # https://docs.github.com/en/actions/reference/runners/github-hosted-runners secrets: inherit # needed for logging to the ghcr.io registry codeql: @@ -82,9 +85,17 @@ jobs: with: category: "/language:${{matrix.language}}" - publish-ee: + publish-image: # environment: release # approval runs-on: ubuntu-24.04 + name: publish-${{ matrix.image }} + strategy: + fail-fast: false + matrix: + image: + - ee + - selenium + needs: - tox if: github.ref == 'refs/heads/main' || (github.event_name == 'release' && github.event.action == 'published') @@ -97,7 +108,7 @@ jobs: - name: pull-merge-push for the the two arch images under a single manifest env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - run: ./tools/ee.sh --publish "${{ github.event.release.tag_name || github.sha }}" "${{ (github.event_name == 'release' && github.event.action == 'published') || '--dry' }}" + run: ./tools/${{ matrix.image }}.sh --publish "${{ github.event.release.tag_name || github.sha }}" "${{ (github.event_name == 'release' && github.event.action == 'published') || '--dry' }}" publish-devspaces: runs-on: ubuntu-24.04 @@ -189,7 +200,7 @@ jobs: if: github.event_name == 'release' && github.event.action == 'published' needs: - publish-wheel - - publish-ee + - publish-image - publish-devspaces runs-on: ubuntu-24.04 diff --git a/cspell.config.yaml b/cspell.config.yaml index 10a83f14..d4213f6f 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -20,6 +20,8 @@ ignorePaths: - mkdocs.yml - pyproject.toml - tox.ini + - selenium/Containerfile + - selenium/init.go languageSettings: - languageId: python diff --git a/pyproject.toml b/pyproject.toml index 1ab29078..8d628631 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -556,6 +556,16 @@ editable = true extras = ["server"] skip_install = false +[tool.tox.env.selenium] +commands = [["python", "-m", "build", "--version"], ["./tools/selenium.sh"]] +commands_post = [["./tools/cleanup.sh"]] +commands_pre = [] +dependency_groups = ["ee", "dev"] +description = "Build the selenium-adt container image" +editable = true +extras = ["server"] +skip_install = false + [tool.tox.env.lint] commands = [["pre-commit", "run", "--all-files", "--show-diff-on-failure"]] commands_post = [] @@ -603,7 +613,8 @@ allowlist_externals = [ "sh", "./tools/cleanup.sh", "./tools/devspaces.sh", - "./tools/ee.sh" + "./tools/ee.sh", + "./tools/selenium.sh" ] commands = [ ["sh", "-c", "ansible --version | head -n 1"], diff --git a/selenium/Containerfile b/selenium/Containerfile new file mode 100644 index 00000000..f05f9064 --- /dev/null +++ b/selenium/Containerfile @@ -0,0 +1,166 @@ +FROM registry.fedoraproject.org/fedora:43 AS selenium-atd +LABEL maintainer="ansible-devtools@redhat.com" + +COPY init.go . + +COPY settings.json /home/selenium/.local/share/code-server/User/settings.json + +# Firefox releases +# https://download-installer.cdn.mozilla.net/pub/firefox/releases/ +ARG FIREFOX_URL="https://download-installer.cdn.mozilla.net/pub/firefox/releases/140.7.0esr/linux-x86_64/en-US/firefox-140.7.0esr.tar.xz" +# Gecko driver releases +# https://github.com/mozilla/geckodriver/releases +ARG GECKODRIVER_VERSION="v0.36.0" +# Chrome versions +# https://www.ubuntuupdates.org/package/google_chrome/stable/main/base/google-chrome-stable +ARG CHROME_VERSION="144.0.7559.59-1" + +ARG SELENIUM_MAJOR_VERSION=4 + +ARG SELENIUM_MINOR_VERSION=39 + +ARG SELENIUM_PATCH_VERSION=0 + +ENV SELENIUM_HOME=/home/selenium + +ENV SELENIUM_PORT=4444 \ + SELENIUM_SESSION_TIMEOUT=1800 \ + VNC_PORT=5999 \ + API_PORT=8000 \ + DISPLAY=:99 \ + DBUS_SESSION_BUS_ADDRESS=/dev/null \ + HOME=${SELENIUM_HOME} \ + VNC_GEOMETRY="1600x900" \ + SELENIUM_VERSION=${SELENIUM_MAJOR_VERSION}.${SELENIUM_MINOR_VERSION}.${SELENIUM_PATCH_VERSION} \ + SELENIUM_PATH=${SELENIUM_HOME}/selenium-server/selenium-server-standalone.jar \ + SELENIUM_HTTP_JDK_CLIENT_PATH=${SELENIUM_HOME}/selenium-server/selenium-http-jdk-client.jar \ + PATH=${SELENIUM_HOME}/firefox:/opt/google/chrome:${PATH} + +EXPOSE ${SELENIUM_PORT} + +EXPOSE ${VNC_PORT} + +EXPOSE ${API_PORT} + +WORKDIR ${SELENIUM_HOME} + +RUN PACKAGES="\ +alsa-lib \ +at-spi2-atk \ +at-spi2-core \ +atk \ +avahi-libs \ +bzip2 \ +cairo \ +cairo-gobject \ +cups-libs \ +dbus-glib \ +dbus-libs \ +expat \ +fluxbox \ +fontconfig \ +freetype \ +fribidi \ +gdk-pixbuf2 \ +graphite2 \ +gtk3 \ +go \ +harfbuzz \ +imlib2 \ +java-latest-openjdk-headless \ +jq \ +libcloudproviders \ +libdatrie \ +libdrm \ +libepoxy \ +liberation-fonts \ +liberation-fonts-common \ +liberation-mono-fonts \ +liberation-sans-fonts \ +liberation-serif-fonts \ +libfontenc \ +libglvnd \ +libglvnd-glx \ +libICE \ +libjpeg-turbo \ +libpng \ +libSM \ +libthai \ +libwayland-client \ +libwayland-cursor \ +libwayland-egl \ +libwayland-server \ +libwebp \ +libX11 \ +libX11-common \ +libX11-xcb \ +libXau \ +libxcb \ +libXcomposite \ +libXcursor \ +libXdamage \ +libXdmcp \ +libXext \ +libXfixes \ +libXfont2 \ +libXft \ +libXi \ +libXinerama \ +libxkbcommon \ +libxkbfile \ +libXpm \ +libXrandr \ +libXrender \ +libXtst \ +libxshmfence \ +libXt \ +mesa-libgbm \ +nspr \ +nss \ +nss-softokn \ +nss-softokn-freebl \ +nss-util \ +nss-tools \ +nss-sysinit \ +pango \ +pixman \ +tar \ +tigervnc-server-minimal \ +tzdata-java \ +unzip \ +vulkan-loader \ +wget \ +xdg-utils \ +xkbcomp \ +xkeyboard-config" && \ +microdnf -q -y install ${PACKAGES} >/dev/null +# ^ https://github.com/rpm-software-management/dnf5/issues/570 + +RUN mkdir -p .cache/dconf .mozilla/plugins .vnc/ .fluxbox/ && \ +echo "session.screen0.toolbar.autoHide: true" > .fluxbox/init && \ +touch .Xauthority .vnc/config && \ +mkdir -p ${SELENIUM_HOME}/selenium-server && \ +curl -L https://github.com/SeleniumHQ/selenium/releases/download/selenium-${SELENIUM_MAJOR_VERSION}.${SELENIUM_MINOR_VERSION}.0/selenium-server-${SELENIUM_VERSION}.jar \ +-o ${SELENIUM_PATH} && \ +curl -L https://repo1.maven.org/maven2/org/seleniumhq/selenium/selenium-http-jdk-client/${SELENIUM_VERSION}/selenium-http-jdk-client-${SELENIUM_VERSION}.jar \ +-o ${SELENIUM_HTTP_JDK_CLIENT_PATH} && \ +curl -L https://download-installer.cdn.mozilla.net/pub/firefox/releases/140.7.0esr/linux-x86_64/en-US/firefox-140.7.0esr.tar.xz|tar --xz -x && \ +curl -LO https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz && \ +tar -C /usr/bin/ -xvf geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz && \ +rm -f geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz && \ +curl -fsSL https://code-server.dev/install.sh | sh && \ +code-server --install-extension ms-python.vscode-python-envs --install-extension redhat.vscode-yaml --install-extension redhat.vscode-redhat-account && \ +mkdir -p workspace && touch workspace/playbook.yaml && \ +modutil -fips true -dbdir /etc/pki/nssdb -force && \ +chown -R 0:0 /etc/pki/nssdb && \ +chmod 644 /etc/pki/nssdb/* && \ +chown -R 1001:0 ${SELENIUM_HOME} && \ +chmod -R g=u ${SELENIUM_HOME} + +USER 1001 + +# install packages needed for go file +RUN go vet /init.go + +# run init.go to start all process in order +CMD ["sh", "-c", "go run /init.go" ] diff --git a/selenium/README.md b/selenium/README.md new file mode 100644 index 00000000..ceef09db --- /dev/null +++ b/selenium/README.md @@ -0,0 +1,32 @@ +# selenium-atd container + +This container has selenium and vscode server in addition to 'adt' tools and +is used for testing vscode-ansible extension. + +```bash +docker build -t {name} -f Dockerfile . +``` + +or + +```bash +podman build -t {name} -f Dockerfile . +``` + +To run it: + +```bash +docker run -it --shm-size=2g -p 4444:4444 -p 5999:5999 {name} +``` + +Alternatively, a pre built image can be pulled from quay: + +```bash +docker run -it --shm-size=2g -p 4444:4444 -p 5999:5999 ghcr.io/ansible/selenium-adt +``` + +When the container starts, it will automatically start all the needed services. + +You can connect to the container using a vnc client on port 5999 and see what +happens in the container the vs-code server runs on port 8080 and selenium runs +on port 4444 diff --git a/selenium/init.go b/selenium/init.go new file mode 100644 index 00000000..1ec18d38 --- /dev/null +++ b/selenium/init.go @@ -0,0 +1,163 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "os/signal" + "syscall" + "time" +) + +func startXvnc() *exec.Cmd { + xvnc := exec.Command( + "Xvnc", + os.Getenv("DISPLAY"), + "-alwaysshared", + "-depth", + "16", + "-geometry", + os.Getenv("VNC_GEOMETRY"), + "-securitytypes", + "none", + "-auth", + fmt.Sprintf("%s/.Xauthority", os.Getenv("HOME")), + "-fp", + "catalogue:/etc/X11/fontpath.d", + "-pn", + "-rfbport", + os.Getenv("VNC_PORT")) + fmt.Println("Starting Xvnc") + xvnc.Start() + return xvnc +} + +func waitForPort() { + n := 1 + address := net.JoinHostPort("localhost", os.Getenv("VNC_PORT")) + for n < 50 { + conn, _ := net.Dial("tcp", address) + if conn != nil { + conn.Close() + break + } + n++ + time.Sleep(10 * time.Millisecond) + } +} + +func startFluxbox() *exec.Cmd { + fluxbox := exec.Command("fluxbox") + fmt.Println("Starting fluxbox") + fluxbox.Start() + return fluxbox +} + +func printSeleniumCombinedOutput(seleniumStdout io.ReadCloser) { + scanner := bufio.NewScanner(seleniumStdout) + for scanner.Scan() { + line := scanner.Text() + fmt.Println(line) + } +} + +func startSelenium() *exec.Cmd { + fmt.Println("Starting selenium standalone") + selenium := exec.Command( + "java", + "-Dwebdriver.http.factory=jdk-http-client", + "-jar", + os.Getenv("SELENIUM_PATH"), + "--ext", + os.Getenv("SELENIUM_HTTP_JDK_CLIENT_PATH"), + "standalone", + "--port", + os.Getenv("SELENIUM_PORT"), + "--session-timeout", + os.Getenv("SELENIUM_SESSION_TIMEOUT"), + ) + seleniumStdout, _ := selenium.StdoutPipe() + selenium.Stderr = selenium.Stdout + go printSeleniumCombinedOutput(seleniumStdout) + selenium.Start() + return selenium +} + +func startCodeServer() *exec.Cmd { + vscode := exec.Command( + "code-server", + "./workspace", + "--auth", + "none", + ) + fmt.Println("Starting vscode-server") + vscode.Start() + return vscode +} + +func startProcesses() (*exec.Cmd, *exec.Cmd, *exec.Cmd, *exec.Cmd) { + xvnc := startXvnc() + waitForPort() + fluxbox := startFluxbox() + selenium := startSelenium() + vscode := startCodeServer() + return xvnc, fluxbox, selenium, vscode +} + +func stopProcesses(xvnc *exec.Cmd, fluxbox *exec.Cmd, selenium *exec.Cmd, vscode *exec.Cmd) { + fmt.Println("Stopping selenium") + selenium.Process.Kill() + selenium.Wait() + fmt.Println("Stopping fluxbox") + fluxbox.Process.Kill() + fluxbox.Wait() + fmt.Println("Stopping Xvnc") + xvnc.Process.Kill() + xvnc.Wait() + fmt.Println("Stopping code-server") + vscode.Process.Kill() + vscode.Wait() +} + +func main() { + // start procs, then allow graceful shutdown with HTTP API or signal handler + xvnc, fluxbox, selenium, vscode := startProcesses() + + // shutdown handler based on: + // https://medium.com/@int128/shutdown-http-server-by-endpoint-in-go-2a0e2d7f9b8c + // using signal.NotifyContext instead of context.WithCancel + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + m := http.NewServeMux() + address := net.JoinHostPort("localhost", os.Getenv("API_PORT")) + s := http.Server{Addr: address, Handler: m} + m.HandleFunc("/shutdown", func(w http.ResponseWriter, r *http.Request) { + fmt.Println("Received shutdown request via HTTP") + w.Write([]byte("OK")) + stop() + }) + go func() { + fmt.Printf("HTTP server listening on '%s'\n", address) + if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed { + fmt.Println(err) + os.Exit(1) + } + }() + + select { + case <-ctx.Done(): + // Shutdown the HTTP server if context is cancelled + // Context can either be cancelled by '/shutdown' handler or by SIGINT/SIGTERM + s.Shutdown(ctx) + } + + stopProcesses(xvnc, fluxbox, selenium, vscode) + fmt.Println("Bye bye") + os.Exit(0) +} diff --git a/selenium/settings.json b/selenium/settings.json new file mode 100644 index 00000000..89c113b0 --- /dev/null +++ b/selenium/settings.json @@ -0,0 +1,6 @@ +{ + "ansible.lightspeed.enabled": true, + "ansible.lightspeed.suggestions.enabled": true, + "ansible.lightspeed.URL": "https://stage.ai.ansible.redhat.com", + "security.workspace.trust.enabled": false +} diff --git a/tools/selenium.sh b/tools/selenium.sh new file mode 100755 index 00000000..56686524 --- /dev/null +++ b/tools/selenium.sh @@ -0,0 +1,71 @@ +#!/bin/bash -e +# cspell: ignore exuo,outdir,aarch64,iname,buildx +set -exuo pipefail + + +ADT_CONTAINER_ENGINE=${ADT_CONTAINER_ENGINE:-docker} + +# Identify the architecture in format used by the container engines because +# `arch` command returns either arm64 or aarch64 depending on the system +ARCH=$(arch) +if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "arm64" ]; then + ARCH="arm64" +elif [ "$ARCH" == "x86_64" ]; then + ARCH="amd64" +else + echo "Unsupported architecture: $ARCH" + exit 1 +fi + + +# keep the localhost/ prefix on image name all the time or we will face various +# problems related to docker/podman differences when this is missing. +IMAGE_NAME=localhost/selenium-adt:test + +# BUILD_CMD="podman build --squash-all" +BUILD_CMD="${ADT_CONTAINER_ENGINE} buildx build --progress=plain" + +# Publish should run on CI only on main branch, with or without release tag +if [ "--publish" == "${1:-}" ]; then + if [ -z "${2:-}" ]; then + echo "Please also pass the tag to be published for the merged image. Source image will use the sha tag." + exit 1 + fi + + set +x # Disable echo for lines with GITHUB_TOKEN + if [ -n "${GITHUB_TOKEN:-}" ] && [ -n "${GITHUB_ACTOR:-}" ]; then + echo "${GITHUB_TOKEN:-}" | ${ADT_CONTAINER_ENGINE} login ghcr.io -u "${GITHUB_ACTOR:-}" --password-stdin + fi + set -x + if [ -z "${GITHUB_SHA:-}" ]; then + echo "Unable to find GITHUB_SHA variable." + exit 1 + fi + ${ADT_CONTAINER_ENGINE} pull -q "ghcr.io/ansible/selenium-adt-tmp:${GITHUB_SHA:-}-arm64" + ${ADT_CONTAINER_ENGINE} pull -q "ghcr.io/ansible/selenium-adt-tmp:${GITHUB_SHA:-}-amd64" + + for TAG in ghcr.io/ansible/selenium-adt:${2:-} ghcr.io/ansible/selenium-adt:latest; do + ${ADT_CONTAINER_ENGINE} manifest create "$TAG" --amend "ghcr.io/ansible/selenium-adt-tmp:${GITHUB_SHA:-}-amd64" --amend "ghcr.io/ansible/selenium-adt-tmp:${GITHUB_SHA:-}-arm64" + ${ADT_CONTAINER_ENGINE} manifest annotate --arch arm64 "$TAG" "ghcr.io/ansible/selenium-adt-tmp:${GITHUB_SHA:-}-arm64" + + # We push only when there is a release, and that is when $2 is not the same as GITHUB_SHA + if [ "--dry" != "${3:-}" ]; then + ${ADT_CONTAINER_ENGINE} manifest push "$TAG" + fi + done + exit 0 +fi + +$BUILD_CMD -f selenium/Containerfile selenium/ --tag "${IMAGE_NAME}" + +if [[ -n "${GITHUB_SHA:-}" && "${GITHUB_EVENT_NAME:-}" != "pull_request" ]]; then + FQ_IMAGE_NAME="ghcr.io/ansible/selenium-adt-tmp:${GITHUB_SHA}-$ARCH" + $ADT_CONTAINER_ENGINE tag $IMAGE_NAME "${FQ_IMAGE_NAME}" + # https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry + set +x # Disable echo for lines with GITHUB_TOKEN + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + fi + set -x + $ADT_CONTAINER_ENGINE push "${FQ_IMAGE_NAME}" +fi