WIP: CI for guides

.github/workflows/site-guides.yml

@@ -0,0 +1,78 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Site Guides
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'site/content/guides/**'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'site/content/guides/**'
+  workflow_dispatch:
+    # Allow manual execution
+
+jobs:
+  test-site-guides:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+      - name: Set up JDK 21
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
+        with:
+          python-version: '3.14'
+      - name: Setup test environment
+        uses: ./.github/actions/setup-test-env
+
+      - name: Test site testing
+        working-directory: site/it
+        run: |
+          echo "::group::Setup"
+          python3 -m pip install --upgrade pip
+          python3 -m venv .venv
+          . .venv/bin/activate
+          UV_VERSION="$(cat pyproject.toml | grep -A1 tool.uv | grep -v tool.uv | grep required-version | sed 's/required-version *= *"\([^"]*\)".*/\1/')"
+          pip install "uv$UV_VERSION"
+          uv sync --all-packages
+          echo "::endgroup::"
+          echo "::group::pytest"
+          pytest
+          echo "::endgroup::"
+
+      - name: Free disk space
+        uses: ./.github/actions/free-disk-space
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+      - name: Guide testing
+        working-directory: site
+        run: it/markdown-testing.py
+      - name: Archive test results
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        if: always()
+        with:
+          name: upload-guides-test-artifacts
+          path: |
+            site/it/build/tests/**

build.gradle.kts

@@ -69,6 +69,7 @@ tasks.named<RatTask>("rat").configure {
 
   // Files copied from Docsy (ASLv2 licensed) don't have header
   excludes.add("site/layouts/docs/baseof.html")
+  excludes.add("site/layouts/guides/baseof.html")
   excludes.add("site/layouts/shortcodes/redoc-polaris.html")
   excludes.add("site/layouts/community/list.html")
   excludes.add("site/layouts/partials/navbar.html")
@@ -113,6 +114,12 @@ tasks.named<RatTask>("rat").configure {
   excludes.add("site/resources/_gen/**")
   excludes.add("node_modules/**")
 
+  // Guides testing
+  excludes.add("site/content/guides/ceph/**/*.keyring")
+  excludes.add("site/it/.idea")
+  excludes.add("/venv")
+  excludes.add("/getting-started") // it's a symlink
+
   // Python
   excludes.add("**/.venv/**")
   excludes.add("**/polaris-venv/**")

getting-started

@@ -0,0 +1 @@
+site/content/guides/

site/content/guides/.it-setup/index.md

@@ -0,0 +1,68 @@
+---
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+
+# Guides CI setup, this is NOT a guide
+
+**This page is not rendered on the website.**
+
+The directory name starting with a `.` ensures that this page is executed by the guides integration tests
+before any other guide's code blocks.
+
+## Eager code block execution
+
+This code block is eagerly executed here to further skip the repeated and expensive Polaris image builds
+present in the guides' code blocks.
+
+```shell
+./gradlew \
+   :polaris-server:assemble \
+   :polaris-server:quarkusAppPartsBuild --rerun \
+   :polaris-admin:assemble \
+   :polaris-admin:quarkusAppPartsBuild --rerun \
+   -Dquarkus.container-image.build=true
+```
+
+## Ignored code blocks
+
+These code blocks are not executed in guide tests because the Docker images have already been generated.
+
+The `skip_all` directive is used to skip this and all following executions of the same code block,
+irrespective whether `skip_all` is present or not.
+
+> **Note:** The `skip_all` directive is only supported by the `shell` code block.
+
+> **Note:** The code block must be _exactly_ the same, including all whitespaces.
+
+```shell skip_all
+./gradlew \
+   :polaris-server:assemble \
+   :polaris-server:quarkusAppPartsBuild --rerun \
+   -Dquarkus.container-image.build=true
+```
+
+```shell skip_all
+./gradlew \
+   :polaris-server:assemble \
+   :polaris-server:quarkusAppPartsBuild --rerun \
+   :polaris-admin:assemble \
+   :polaris-admin:quarkusAppPartsBuild --rerun \
+   -Dquarkus.container-image.build=true
+```

getting-started/quickstart/README.md → site/content/guides/README.md

@@ -6,9 +6,9 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
-
+ 
    http://www.apache.org/licenses/LICENSE-2.0
-
+ 
   Unless required by applicable law or agreed to in writing,
   software distributed under the License is distributed on an
   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,6 +17,6 @@
   under the License.
 -->
 
-# Apache Polaris Quickstart
+# Getting Started with Apache Polaris
 
-Please see the [quickstart guide](../../site/content/in-dev/unreleased/getting-started/quick-start.md) for more information.
+See the [Guides section on the web site](https://polaris.apache.org/guides) for more information.

site/content/guides/_index.md

@@ -0,0 +1,159 @@
+---
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+linkTitle: "Guides"
+title: "Apache Polaris Guides"
+weight: 200
+cascade:
+    type: docs
+menus:
+    main:
+        parent: Guides
+        weight: 1
+        identifier: overview
+        name: Overview
+---
+
+You can quickly get started with Polaris by playing with the docker-compose examples provided
+by the guides shown on the left of this page.
+Each guide has detailed instructions.
+
+## Prerequisites
+
+- [Docker](https://docs.docker.com/get-docker/)
+- [Docker Compose](https://docs.docker.com/compose/install/)
+- [jq](https://stedolan.github.io/jq/download/) (for some examples)
+
+## Getting Started Examples
+
+- [Spark](spark): An example that uses an in-memory metastore, automatically bootstrapped, with
+  Apache Spark and a Jupyter notebook.
+
+- [Telemetry](telemetry): An example that includes Prometheus and Jaeger to collect metrics and
+  traces from Apache Polaris. This example automatically creates a `polaris_demo` catalog.
+
+- [Keycloak](keycloak): An example that uses Keycloak as an external identity provider (IDP) for
+  authentication.
+
+## Authoring Guides
+
+Writing new or updating Guides, especially those that use Docker Compose, should follow these
+guidelines. This ensures that the examples work as expected and that the guides pass the tests
+run in CI and locally.
+
+### Running guides CI tests locally
+
+Requirements: Python 3.14 installed locally. Other Python 3 versions should work as well, but
+have not been validated.
+
+To run tests for all guides in `site/content/guides`, run
+```shell
+cd site/it
+./markdown-testing.py
+```
+
+To run tests for one or more guides in `site/content/guides`, use the following pattern:
+```shell
+cd site/it
+./markdown-testing.py content/guides/rustfs/index.md content/guides/telemetry/index.md
+```
+
+More information about guides-testing can be found in the `site/it/README.md` file in the
+[source repository](https://github.com/apache/polaris).
+
+### Constraints
+
+1. `docker compose` invocations must be on a single line in a `shell` code block.
+2. Spark SQL shell invocations must be the only statement `shell` code block in a guide.
+
+### Tips for Docker usage
+
+- Do not use any of the `--interactive` or `--tty` options
+
+### Tips for Docker Compose files
+
+- All "daemon" services should have a healthcheck and depend on it using
+  `condition: service_healthy`.
+- Always put port-mappings in quotes, for example:
+  ```yaml
+  ports:
+    - "9874:9874"
+  ```
+- "Final setup services," for example the service that sets up Polaris, are a bit
+  tricky to get right in the sense that those do not let `docker compose up --detach --wait`
+  fail.
+  The former docker-compose command would yield an error if any service exits.
+  To prevent this, setup-services should have a final "endless" command to keep them
+  running and have a health check to inform about its success.
+
+  "Intermediate setup services," for example the service to create a bucket, do **not**
+  and **should not** have a final `sleep` command, because that would delay the overall
+  startup. The Polaris service must depend on the intermediate service to setup a bucket
+  with the `condition` `service_completed_successfully`.
+
+  Working example:
+  ```yaml
+  polaris-setup:
+    image: alpine/curl:8.17.0
+    depends_on:
+      polaris:
+        condition: service_healthy
+    (...)
+    entrypoint: "/bin/sh"
+    command:
+      - "-c"
+      - >-
+        do_stuff &&
+        touch /tmp/polaris-setup-done &&
+        tail -f /dev/null
+    healthcheck:
+      test: ["CMD", "test", "-f", "/tmp/polaris-setup-done"]
+  ```
+- Take care to propagate _all_ failures of a setup service's commands.
+  This can be achieved by chaining the commands with `&&` and using `set -e` at the
+  beginning of executed scripts.
+- For `curl`, use the `--fail-with-body` or `--fail` options or add specialized logic
+  to validate the HTTP response code.
+- Service dependencies (`depends_on`) must be declared with the correct `condition`,
+  meaning to use the "Long syntax" as described in the
+  [Docker Compose documentation](https://docs.docker.com/compose/compose-file/compose-file-v3/#depends_on).
+- Dependencies on "setup" services MUST be declared using a syntax like this:
+  ```yaml
+  polaris:
+    image: apache/polaris:latest
+    depends_on:
+      setup_bucket:
+        condition: service_completed_successfully
+  ```  
+- Dependencies on "daemon" services with a "proper" health check should be declared using
+  a syntax like this:
+  ```yaml
+  polaris:
+    image: apache/polaris:latest
+    depends_on:
+      minio:
+        condition: service_healthy
+  ```
+
+### Tips for inspecting Docker compose logs
+
+When inspecting the logs of Docker Compose, watch the logged timestamps.
+The order of the log entries emitted by Docker Compose is not necessarily in the
+natural order of execution between different services.
+Timestamps of each log entry are emitted to the console and the test log files.

getting-started/assets/cloud_providers/deploy-aws.sh → site/content/guides/assets/cloud_providers/deploy-aws.sh

@@ -94,4 +94,4 @@ export STORAGE_LOCATION="s3://$S3_BUCKET_NAME/quickstart_catalog/"
        -Dquarkus.container-image.build=true \
        --no-build-cache
 
-docker compose -p polaris -f getting-started/jdbc/docker-compose-bootstrap-db.yml -f getting-started/jdbc/docker-compose.yml up -d
+docker compose -p polaris -f site/content/guides/jdbc/docker-compose-bootstrap-db.yml -f site/content/guides/jdbc/docker-compose.yml up -d