From 5554ca58e2d48bc92ebcd3bf88d3b47ae6816843 Mon Sep 17 00:00:00 2001 From: grafanabot Date: Mon, 17 Nov 2025 07:23:21 +0000 Subject: [PATCH] Update `make docs` procedure --- docs/docs.mk | 60 ++--- docs/make-docs | 712 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 605 insertions(+), 167 deletions(-) diff --git a/docs/docs.mk b/docs/docs.mk index 9b0718ac9569..d23c8fe589e6 100644 --- a/docs/docs.mk +++ b/docs/docs.mk @@ -31,14 +31,6 @@ ifeq ($(PROJECTS),) $(error "PROJECTS variable must be defined in variables.mk") endif -# First project is considered the primary one used for doc-validator. -PRIMARY_PROJECT := $(subst /,-,$(firstword $(subst :, ,$(firstword $(PROJECTS))))) - -# Name for the container. -ifeq ($(origin DOCS_CONTAINER), undefined) -export DOCS_CONTAINER := $(PRIMARY_PROJECT)-docs -endif - # Host port to publish container port to. ifeq ($(origin DOCS_HOST_PORT), undefined) export DOCS_HOST_PORT := 3002 @@ -49,12 +41,7 @@ ifeq ($(origin DOCS_IMAGE), undefined) export DOCS_IMAGE := grafana/docs-base:latest endif -# Container image used for doc-validator linting. -ifeq ($(origin DOC_VALIDATOR_IMAGE), undefined) -export DOC_VALIDATOR_IMAGE := grafana/doc-validator:latest -endif - -# Container image used for vale linting. +# Container image used for Vale linting. ifeq ($(origin VALE_IMAGE), undefined) export VALE_IMAGE := grafana/vale:latest endif @@ -70,48 +57,59 @@ ifeq ($(origin HUGO_REFLINKSERRORLEVEL), undefined) export HUGO_REFLINKSERRORLEVEL := WARNING endif +# Whether to pull the latest container image before running the container. +ifeq ($(origin PULL), undefined) +export PULL := true +endif + .PHONY: docs-rm docs-rm: ## Remove the docs container. $(PODMAN) rm -f $(DOCS_CONTAINER) .PHONY: docs-pull docs-pull: ## Pull documentation base image. - $(PODMAN) pull $(DOCS_IMAGE) + $(PODMAN) pull -q $(DOCS_IMAGE) make-docs: ## Fetch the latest make-docs script. make-docs: - if [[ ! -f "$(PWD)/make-docs" ]]; then + if [[ ! -f "$(CURDIR)/make-docs" ]]; then echo 'WARN: No make-docs script found in the working directory. Run `make update` to download it.' >&2 exit 1 fi .PHONY: docs -docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. See also `docs-no-pull`. +docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. To not pull the image, set `PULL=false`. +ifeq ($(PULL), true) docs: docs-pull make-docs - $(PWD)/make-docs $(PROJECTS) - -.PHONY: docs-no-pull -docs-no-pull: ## Serve documentation locally without pulling the `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. -docs-no-pull: make-docs - $(PWD)/make-docs $(PROJECTS) +else +docs: make-docs +endif + $(CURDIR)/make-docs $(PROJECTS) .PHONY: docs-debug docs-debug: ## Run Hugo web server with debugging enabled. TODO: support all SERVER_FLAGS defined in website Makefile. docs-debug: make-docs - WEBSITE_EXEC='hugo server --bind 0.0.0.0 --port 3002 --debug' $(PWD)/make-docs $(PROJECTS) - -.PHONY: doc-validator -doc-validator: ## Run doc-validator on the entire docs folder. -doc-validator: make-docs - DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) $(PWD)/make-docs $(PROJECTS) + WEBSITE_EXEC='hugo server --bind 0.0.0.0 --port 3002 --logLevel debug' $(CURDIR)/make-docs $(PROJECTS) .PHONY: vale -vale: ## Run vale on the entire docs folder. +vale: ## Run vale on the entire docs folder which includes pulling the latest `VALE_IMAGE` (default: `grafana/vale:latest`) container image. To not pull the image, set `PULL=false`. vale: make-docs - DOCS_IMAGE=$(VALE_IMAGE) $(PWD)/make-docs $(PROJECTS) +ifeq ($(PULL), true) + $(PODMAN) pull -q $(VALE_IMAGE) +endif + DOCS_IMAGE=$(VALE_IMAGE) $(CURDIR)/make-docs $(PROJECTS) .PHONY: update update: ## Fetch the latest version of this Makefile and the `make-docs` script from Writers' Toolkit. curl -s -LO https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/docs.mk curl -s -LO https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs chmod +x make-docs + +# ls static/templates/ | sed 's/-template\.md//' | xargs +TOPIC_TYPES := concept multiple-tasks reference section task tutorial visualization +.PHONY: $(patsubst %,topic/%,$(TOPIC_TYPES)) +topic/%: ## Create a topic from the Writers' Toolkit template. Specify the topic type as the target, for example, `make topic/task TOPIC_PATH=sources/my-new-topic.md`. +$(patsubst %,topic/%,$(TOPIC_TYPES)): + $(if $(TOPIC_PATH),,$(error "You must set the TOPIC_PATH variable to the path where the $(@F) topic will be created. For example: make $(@) TOPIC_PATH=sources/my-new-topic.md")) + mkdir -p $(dir $(TOPIC_PATH)) + curl -s -o $(TOPIC_PATH) https://raw.githubusercontent.com/grafana/writers-toolkit/refs/heads/main/docs/static/templates/$(@F)-template.md diff --git a/docs/make-docs b/docs/make-docs index 8d55ffaa9575..7c8efd45137b 100755 --- a/docs/make-docs +++ b/docs/make-docs @@ -1,94 +1,314 @@ #!/bin/sh +# shellcheck disable=SC2034 +# # The source of this file is https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs. # # `make-docs` procedure changelog # # Updates should conform to the guidelines in https://keepachangelog.com/en/1.1.0/. # [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes. # Changes are relevant to this script and the support docs.mk GNU Make interface. +# +# ## 10.1.0 (2025-11-11) +# +# ### Fixed +# +# - Extend readiness probes to prevent confusing output. +# +# Before, the probes could fail too soon and long an error message which contradicted the service starting up. +# +# ## 10.0.0 (2025-10-13) +# +# ### Changed +# +# - Hugo no longer supports the `--debug` option, use `--logLevel debug` instead. +# +# Thank you to @karlskewes for their contribution! +# +# ## 9.0.0 (2025-04-05) +# +# ### Removed +# +# - doc-validator target and associated scripts. +# +# Most useful rules have been migrated to Vale and the others are often false positives. +# +# ## 8.5.2 (2025-02-28) +# +# ### Fixed +# +# - topic/ targets are no longer no-ops as a result of 8.5.1. +# +# ## 8.5.1 (2025-02-18) +# +# ### Fixed +# +# - PHONY declaration for topic/ targets. +# +# ## 8.5.0 (2025-02-13) +# +# ### Added +# +# - make topic/ TOPIC_PATH= target to create a new topic from the Writers' Toolkit templates. +# +# ## 8.4.0 (2025-01-27) +# +# ### Fixed +# +# - Correct mount for the /docs/grafana-cloud/send-data/fleet-management/ project. +# +# ## 8.3.0 (2024-12-27) +# +# ### Added +# +# - Debug output of the final command when DEBUG=true. +# +# Useful to inspect if the script is correctly constructing the final command. +# +# ## 8.2.0 (2024-12-22) +# +# ### Removed +# +# - Special cases for Oracle and Datadog plugins now that they exist in the plugins monorepo. +# +# ## 8.1.0 (2024-08-22) +# +# ### Added +# +# - Additional website mounts for projects that use the website repository. +# +# Mounts are required for `make docs` to work in the website repository or with the website project. +# The Makefile is also mounted for convenient development of the procedure in that repository. +# +# ## 8.0.1 (2024-07-01) +# +# ### Fixed +# +# - Update log suppression to catch new format of website /docs/ homepage REF_NOT_FOUND warnings. +# +# These warnings are related to missing some pages during the build that are required for the /docs/ homepage. +# They were previously suppressed but the log format changed and without this change they reappear in the latest builds. +# +# ## 8.0.0 (2024-05-28) +# +# ### Changed +# +# - Add environment variable `OUTPUT_FORMAT` to control the output of commands. +# +# The default value is `human` and means the output format is human readable. +# The value `json` is also supported and outputs JSON. +# +# Note that the `json` format isn't supported by `make docs`, only `make doc-validator` and `make vale`. +# +# ## 7.0.0 (2024-05-03) +# +# ### Changed +# +# - Pull images for all recipes that use containers by default. +# +# Use the `PULL=false` variable to disable this behavior. +# +# ### Removed +# +# - The `docs-no-pull` target as it's redundant with the new `PULL=false` variable. +# +# ## 6.1.0 (2024-04-22) +# +# ### Changed +# +# - Mount volumes with SELinux labels. +# +# https://docs.docker.com/storage/bind-mounts/#configure-the-selinux-label +# +# ### Added +# +# - Pseudo project for including only website resources and no website content. +# +# Facilitates testing shortcodes and layout changes with a small documentation set instead of Grafana Cloud or the entire website. +# +# ## 6.0.1 (2024-02-28) +# +# ### Added +# +# - Suppress new errors relating to absent content introduced in https://github.com/grafana/website/pull/17561. +# +# ## 6.0.0 (2024-02-16) +# +# ### Changed +# +# - Require `jq` for human readable `make doc-validator` output. +# +# ## 5.4.0 (2024-02-12) +# +# ### Changed +# +# - Set `WEBSITE_MOUNTS=true` when a user includes the `website` project. +# +# Ensures consistent behavior across repositories. +# To disable website mounts, add `export WEBSITE_MOUNTS := false` to your `variables.mk` or `variables.mk.local` file. +# - Use website mounts and container volumes also when a user includes the `grafana-cloud` project. +# +# ## 5.3.0 (2024-02-08) +# +# ### Changed +# +# - Updated support for plugins monorepo now that multiple projects have been moved into it. +# - Use `printf` instead of `echo` for better portability of output. +# +# https://www.in-ulm.de/~mascheck/various/echo+printf/ +# +# ## 5.2.0 (2024-01-18) +# +# ### Changed +# +# - Updated `make vale` to use latest Vale style and configuration. +# - Updated `make vale` to use platform appropriate image. +# +# ## 5.1.2 (2023-11-08) +# +# ### Added +# +# - Hide manual_mount warning messages from non-debug output. +# Set the DEBUG environment variable to see all hidden messages. +# +# ## 5.1.1 (2023-10-30) +# +# ### Added +# +# - Support for Datadog and Oracle data source plugins repositories. +# +# ## 5.1.0 (2023-10-20) +# +# ### Added +# +# - Support for the plugins monorepo. +# +# ## 5.0.0 (2023-10-18) +# +# ### Added +# +# - Improved support for website repository. +# +# Mount more content and provide some feedback to users that the build can take time. +# +# - Ability to enter the `grafana/docs-base` container with a shell using the `ENTER` environment variable. +# +# ### Fixed +# +# - Correct key combination for interrupting the process. +# +# Keyboards use capital letters so this more accurately reflects the exact key combination users are expected to press. +# +# ### Removed +# +# - Imperfect implementation of container name. +# +# Facilitates running `make vale` and `make docs` at once. +# Container names are convenient for recognition in `docker ps` but the current implementation has more downsides than upsides. +# +# - Forced platform specification now that multiple architecture images exist. +# +# Significantly speeds up build times on larger repositories. +# +# ## 4.2.2 (2023-10-05) -# ## 4.1.1 (2023-07-20) +# - Added support for Jira data source and MongoDB data source plugins repositories. +# +# ## 4.2.1 (2023-09-13) +# ## Fixed +# +# - Improved consistency of the webserver request loop by polling the Hugo port rather than the proxy port. +# +# ## 4.2.0 (2023-09-01) +# +# ### Added +# +# - Retry the initial webserver request up to ten times to allow for the process to start. +# If it is still failing after ten seconds, an error message is logged. +# +# ## 4.1.1 (2023-07-20) +# # ### Fixed - +# # - Replaced use of `realpath` with POSIX compatible alternative to determine default value for REPOS_PATH. - +# # ## 4.1.0 (2023-06-16) - +# # ### Added - +# # - Mounts of `layouts` and `config` directories for the `website` project. # Ensures that local changes to mounts or shortcodes are reflected in the development server. - +# # ### Fixed - +# # - Version inference for versioned docs pages. # Pages in versioned projects now have the `versioned: true` front matter set to ensure that "version" in $.Page.Scratch is set on builds. - +# # ## 4.0.0 (2023-06-06) - +# # ### Removed - +# # - `doc-validator/%` target. # The behavior of the target was not as described. # Instead, to limit `doc-validator` to only specific files, refer to https://grafana.com/docs/writers-toolkit/writing-guide/tooling-and-workflows/validate-technical-documentation/#run-on-specific-files. - +# # ## 3.0.0 (2023-05-18) - +# # ### Fixed - +# # - Compatibility with the updated Make targets in the `website` repository. # `docs` now runs this script itself, `server-docs` builds the site with the `docs` Hugo environment. - +# # ## 2.0.0 (2023-05-18) - +# # ### Added - +# # - Support for the grafana-cloud/frontend-observability/faro-web-sdk project. # - Use of `doc-validator` v2.0.x which includes breaking changes to command line options. - +# # ### Fixed - +# # - Source grafana-cloud project from website repository. - +# # ### Added - +# # - Support for running the Vale linter with `make vale`. - +# # ## 1.2.1 (2023-05-05) - +# # ### Fixed - +# # - Use `latest` tag of `grafana/vale` image by default instead of hardcoded older version. # - Fix mounting multiple projects broken by the changes in 1.0.1 - +# # ## 1.2.0 (2023-05-05) - +# # ### Added - +# # - Support for running the Vale linter with `make vale`. - +# # ### Fixed - +# # ## 1.1.0 (2023-05-05) - +# # ### Added - +# # - Rewrite error output so it can be followed by text editors. - +# # ### Fixed - +# # - Fix `docs-debug` container process port. - +# # ## 1.0.1 (2023-05-04) - +# # ### Fixed - +# # - Ensure complete section hierarchy so that all projects have a visible menu. - +# # ## 1.0.0 (2023-05-04) - +# # ### Added - +# # - Build multiple projects simultaneously if all projects are checked out locally. # - Run [`doc-validator`](https://github.com/grafana/technical-documentation/tree/main/tools/cmd/doc-validator) over projects. # - Redirect project root to mounted version. @@ -118,7 +338,6 @@ set -ef -readonly DOCS_CONTAINER="${DOCS_CONTAINER:-make-docs}" readonly DOCS_HOST_PORT="${DOCS_HOST_PORT:-3002}" readonly DOCS_IMAGE="${DOCS_IMAGE:-grafana/docs-base:latest}" @@ -128,11 +347,28 @@ readonly DOC_VALIDATOR_SKIP_CHECKS="${DOC_VALIDATOR_SKIP_CHECKS:-^image-}" readonly HUGO_REFLINKSERRORLEVEL="${HUGO_REFLINKSERRORLEVEL:-WARNING}" readonly VALE_MINALERTLEVEL="${VALE_MINALERTLEVEL:-error}" readonly WEBSITE_EXEC="${WEBSITE_EXEC:-make server-docs}" -# If set, the docs-base image will run a prebuild script that sets up Hugo mounts. -readonly WEBSITE_MOUNTS="${WEBSITE_MOUNTS:-}" + +readonly OUTPUT_FORMAT="${OUTPUT_FORMAT:-human}" PODMAN="$(if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)" +if ! command -v curl >/dev/null 2>&1; then + if ! command -v wget >/dev/null 2>&1; then + # shellcheck disable=SC2016 + errr 'either `curl` or `wget` must be installed for this script to work.' + + exit 1 + fi +fi + +if ! command -v "${PODMAN}" >/dev/null 2>&1; then + # shellcheck disable=SC2016 + errr 'either `podman` or `docker` must be installed for this script to work.' + + exit 1 +fi + + about() { cat <&2 - echo "NOTE: you must have a checkout of the project '${_repo}' at '${REPOS_PATH##:*}/${_repo}'." >&2 - echo "NOTE: if you have cloned the repository into a directory with a different name, consider changing it to ${_repo}." >&2 + errr "could not find project '${_repo}' in any of the paths in REPOS_PATH '${REPOS_PATH}'." + note "you must have a checkout of the project '${_repo}' at '${REPOS_PATH##:*}/${_repo}'." + note "if you have cloned the repository into a directory with a different name, consider changing it to ${_repo}." + unset _repo exit 1 } @@ -359,7 +671,7 @@ proj_canonical() { $1 POSIX_HERESTRING - if [ "${_project}" = 'website' ]; then + if [ "${_project}" = website ]; then echo '/docs' unset _project _version @@ -393,7 +705,7 @@ POSIX_HERESTRING case "${_project}" in # Workaround for arbitrary mounts where the version field is expected to be the local directory - # and the repo field is expected to be the container directory. + # and the repo field is expected to be the container directory. arbitrary) echo "${_project}^${_version}^${_repo}^" # TODO ;; @@ -406,6 +718,11 @@ POSIX_HERESTRING proj_to_url_src_dst_ver "$(new_proj helm-charts/mimir-distributed "${_version}")" proj_to_url_src_dst_ver "$(new_proj enterprise-metrics "${_version}")" ;; + resources) + _repo="$(repo_path website)" + echo "arbitrary^${_repo}/config^/hugo/config" "arbitrary^${_repo}/layouts^/hugo/layouts" "arbitrary^${_repo}/scripts^/hugo/scripts" + unset _repo + ;; traces) proj_to_url_src_dst_ver "$(new_proj tempo "${_version}")" proj_to_url_src_dst_ver "$(new_proj enterprise-traces "${_version}")" @@ -419,6 +736,94 @@ POSIX_HERESTRING unset _project _version _repo _path } +await_build() { + url="$1" + req="$(if command -v curl >/dev/null 2>&1; then echo 'curl -s -o /dev/null'; else echo 'wget -q'; fi)" + + # Initial delay to allow container to start before beginning healthchecks + sleep 3 + + # Fast retries for initial startup (10 attempts, 1 second apart) + i=1 + max=10 + while [ "${i}" -ne "${max}" ] + do + sleep 1 + debg "Retrying request to web server assuming the process is still starting up." + i=$((i + 1)) + + if ${req} "${url}"; then + printf '\r\nView documentation locally:\r\n' + for x in ${url_src_dst_vers}; do + IFS='^' read -r url _ _ <&2 + fi +} + +errr() { + printf 'ERRR: %s\r\n' "$1" >&2 +} + +note() { + printf 'NOTE: %s\r\n' "$1" >&2 +} + url_src_dst_vers="$(url_src_dst_vers "$@")" volumes="" @@ -428,11 +833,23 @@ for arg in "$@"; do IFS=: read -r _project _ _repo _ <&2 - echo "Is '${_src}' the correct source directory?" >&2 + errr "Index file '${_src}/_index.md' does not exist." + note "Is '${_src}' the correct source directory?" exit 1 fi fi - echo "DEBG: Mounting '${_src}' at container path '${_dst}'" >&2 + debg "Mounting '${_src}' at container path '${_dst}'" + if [ -z "${volumes}" ]; then - volumes="--volume=${_src}:${_dst}" + volumes="--volume=${_src}:${_dst}:z" else - volumes="${volumes} --volume=${_src}:${_dst}" + volumes="${volumes} --volume=${_src}:${_dst}:z" fi if [ -n "${_ver}" ] && [ "${_ver}" != 'UNVERSIONED' ]; then @@ -472,44 +890,51 @@ ${DOCS_IMAGE} POSIX_HERESTRING case "${image}" in - 'grafana/doc-validator') - proj="$(new_proj "$1")" - echo - "${PODMAN}" run \ - --init \ - --interactive \ - --name "${DOCS_CONTAINER}" \ - --platform linux/amd64 \ - --rm \ - --tty \ - ${volumes} \ - "${DOCS_IMAGE}" \ - "--include=${DOC_VALIDATOR_INCLUDE}" \ - "--skip-checks=${DOC_VALIDATOR_SKIP_CHECKS}" \ - /hugo/content/docs \ - "$(proj_canonical "${proj}")" | sed "s#$(proj_dst "${proj}")#sources#" - ;; 'grafana/vale') proj="$(new_proj "$1")" - echo - "${PODMAN}" run \ - --init \ - --interactive \ - --name "${DOCS_CONTAINER}" \ - --platform linux/amd64 \ - --rm \ - --tty \ - ${volumes} \ - "${DOCS_IMAGE}" \ - "--minAlertLevel=${VALE_MINALERTLEVEL}" \ - --config=/etc/vale/.vale.ini \ - --output=line \ - /hugo/content/docs | sed "s#$(proj_dst "${proj}")#sources#" + printf '\r\n' + IFS='' read -r cmd <"${tempfile}" #!/usr/bin/env bash + +tc() { + set \${*,,} + echo \${*^} +} + for redirect in ${redirects}; do IFS='^' read -r path ver <<<"\${redirect}" echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\nversioned: true\\n---\\n" > "\${path/\${ver}/_index.md}" @@ -518,8 +943,12 @@ done for x in "${url_src_dst_vers}"; do IFS='^' read -r _ _ dst _ <<<"\${x}" + title="\${dst%/*}" + title="\$(tc \${title##*/})" while [[ -n "\${dst}" ]]; do - touch "\${dst}/_index.md" + if [[ ! -f "\${dst}/_index.md" ]]; then + echo -e "---title: \${title}\\n---\\n\\n# \${title}\\n\\n{{< section >}}" > "\${dst}/_index.md" + fi dst="\${dst%/*}" done done @@ -531,36 +960,47 @@ fi ${WEBSITE_EXEC} EOF chmod +x "${tempfile}" - volumes="${volumes} --volume=${tempfile}:/entrypoint" + volumes="${volumes} --volume=${tempfile}:/entrypoint:z" readonly volumes - echo - echo "Documentation will be served at the following URLs:" - for x in ${url_src_dst_vers}; do - IFS='^' read -r url _ _ <&1\ + | sed -u \ + -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \ + -e '/^hugo server/ d' \ + -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \ + -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \ + -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \ + -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \ + -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \ + -e '/website-proxy/ d' \ + -e '/rm -rf dist*/ d' \ + -e '/Press Ctrl+C to stop/ d' \ + -e '/make/ d' \ + -e '/WARNING: The manual_mount source directory/ d' \ + -e '/"docs\/_index.md" not found/d' + fi ;; esac