diff --git a/docs/docs.mk b/docs/docs.mk index c4ba766fb4f0..a08830d907c1 100644 --- a/docs/docs.mk +++ b/docs/docs.mk @@ -34,11 +34,6 @@ endif # First project is considered the primary one used for doc-validator. PRIMARY_PROJECT := $(subst /,-,$(firstword $(subst :, ,$(firstword $(PROJECTS))))) -# Name for the container. -ifeq ($(origin DOCS_CONTAINER), undefined) -export DOCS_CONTAINER := $(PRIMARY_PROJECT)-docs -endif - # Host port to publish container port to. ifeq ($(origin DOCS_HOST_PORT), undefined) export DOCS_HOST_PORT := 3002 diff --git a/docs/make-docs b/docs/make-docs index 94683de344c8..43efdb5faad3 100755 --- a/docs/make-docs +++ b/docs/make-docs @@ -5,107 +5,191 @@ # Updates should conform to the guidelines in https://keepachangelog.com/en/1.1.0/. # [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes. # Changes are relevant to this script and the support docs.mk GNU Make interface. - +# +# ## 6.0.1 (2024-02-28) +# +# ### Added +# +# - Suppress new errors relating to absent content introduced in https://github.com/grafana/website/pull/17561. +# +# ## 6.0.0 (2024-02-16) +# +# ### Changed +# +# - Require `jq` for human readable `make doc-validator` output. +# +# ## 5.4.0 (2024-02-12) +# +# ### Changed +# +# - Set `WEBSITE_MOUNTS=true` when a user includes the `website` project. +# +# Ensures consistent behavior across repositories. +# To disable website mounts, add `export WEBSITE_MOUNTS := false` to your `variables.mk` or `variables.mk.local` file. +# - Use website mounts and container volumes also when a user includes the `grafana-cloud` project. +# +# ## 5.3.0 (2024-02-08) +# +# ### Changed +# +# - Updated support for plugins monorepo now that multiple projects have been moved into it. +# - Use `printf` instead of `echo` for better portability of output. +# +# https://www.in-ulm.de/~mascheck/various/echo+printf/ +# +# ## 5.2.0 (2024-01-18) +# +# ### Changed +# +# - Updated `make vale` to use latest Vale style and configuration. +# - Updated `make vale` to use platform appropriate image. +# +# ## 5.1.2 (2023-11-08) +# +# ### Added +# +# - Hide manual_mount warning messages from non-debug output. +# Set the DEBUG environment variable to see all hidden messages. +# +# ## 5.1.1 (2023-10-30) +# +# ### Added +# +# - Support for Datadog and Oracle data source plugins repositories. +# +# ## 5.1.0 (2023-10-20) +# +# ### Added +# +# - Support for the plugins monorepo. +# +# ## 5.0.0 (2023-10-18) +# +# ### Added +# +# - Improved support for website repository. +# +# Mount more content and provide some feedback to users that the build can take time. +# +# - Ability to enter the `grafana/docs-base` container with a shell using the `ENTER` environment variable. +# +# ### Fixed +# +# - Correct key combination for interrupting the process. +# +# Keyboards use capital letters so this more accurately reflects the exact key combination users are expected to press. +# +# ### Removed +# +# - Imperfect implementation of container name. +# +# Facilitates running `make vale` and `make docs` at once. +# Container names are convenient for recognition in `docker ps` but the current implementation has more downsides than upsides. +# +# - Forced platform specification now that multiple architecture images exist. +# +# Significantly speeds up build times on larger repositories. +# # ## 4.2.2 (2023-10-05) # - Added support for Jira data source and MongoDB data source plugins repositories. - +# # ## 4.2.1 (2023-09-13) # ## Fixed - +# # - Improved consistency of the webserver request loop by polling the Hugo port rather than the proxy port. - +# # ## 4.2.0 (2023-09-01) - +# # ### Added - +# # - Retry the initial webserver request up to ten times to allow for the process to start. # If it is still failing after ten seconds, an error message is logged. - +# # ## 4.1.1 (2023-07-20) - +# # ### Fixed - +# # - Replaced use of `realpath` with POSIX compatible alternative to determine default value for REPOS_PATH. - +# # ## 4.1.0 (2023-06-16) - +# # ### Added - +# # - Mounts of `layouts` and `config` directories for the `website` project. # Ensures that local changes to mounts or shortcodes are reflected in the development server. - +# # ### Fixed - +# # - Version inference for versioned docs pages. # Pages in versioned projects now have the `versioned: true` front matter set to ensure that "version" in $.Page.Scratch is set on builds. - +# # ## 4.0.0 (2023-06-06) - +# # ### Removed - +# # - `doc-validator/%` target. # The behavior of the target was not as described. # Instead, to limit `doc-validator` to only specific files, refer to https://grafana.com/docs/writers-toolkit/writing-guide/tooling-and-workflows/validate-technical-documentation/#run-on-specific-files. - +# # ## 3.0.0 (2023-05-18) - +# # ### Fixed - +# # - Compatibility with the updated Make targets in the `website` repository. # `docs` now runs this script itself, `server-docs` builds the site with the `docs` Hugo environment. - +# # ## 2.0.0 (2023-05-18) - +# # ### Added - +# # - Support for the grafana-cloud/frontend-observability/faro-web-sdk project. # - Use of `doc-validator` v2.0.x which includes breaking changes to command line options. - +# # ### Fixed - +# # - Source grafana-cloud project from website repository. - +# # ### Added - +# # - Support for running the Vale linter with `make vale`. - +# # ## 1.2.1 (2023-05-05) - +# # ### Fixed - +# # - Use `latest` tag of `grafana/vale` image by default instead of hardcoded older version. # - Fix mounting multiple projects broken by the changes in 1.0.1 - +# # ## 1.2.0 (2023-05-05) - +# # ### Added - +# # - Support for running the Vale linter with `make vale`. - +# # ### Fixed - +# # ## 1.1.0 (2023-05-05) - +# # ### Added - +# # - Rewrite error output so it can be followed by text editors. - +# # ### Fixed - +# # - Fix `docs-debug` container process port. - +# # ## 1.0.1 (2023-05-04) - +# # ### Fixed - +# # - Ensure complete section hierarchy so that all projects have a visible menu. - +# # ## 1.0.0 (2023-05-04) - +# # ### Added - +# # - Build multiple projects simultaneously if all projects are checked out locally. # - Run [`doc-validator`](https://github.com/grafana/technical-documentation/tree/main/tools/cmd/doc-validator) over projects. # - Redirect project root to mounted version. @@ -135,7 +219,6 @@ set -ef -readonly DOCS_CONTAINER="${DOCS_CONTAINER:-make-docs}" readonly DOCS_HOST_PORT="${DOCS_HOST_PORT:-3002}" readonly DOCS_IMAGE="${DOCS_IMAGE:-grafana/docs-base:latest}" @@ -145,8 +228,6 @@ readonly DOC_VALIDATOR_SKIP_CHECKS="${DOC_VALIDATOR_SKIP_CHECKS:-^image-}" readonly HUGO_REFLINKSERRORLEVEL="${HUGO_REFLINKSERRORLEVEL:-WARNING}" readonly VALE_MINALERTLEVEL="${VALE_MINALERTLEVEL:-error}" readonly WEBSITE_EXEC="${WEBSITE_EXEC:-make server-docs}" -# If set, the docs-base image will run a prebuild script that sets up Hugo mounts. -readonly WEBSITE_MOUNTS="${WEBSITE_MOUNTS:-}" PODMAN="$(if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)" @@ -217,9 +298,8 @@ SOURCES_grafana_cloud_frontend_observability_faro_web_sdk='faro-web-sdk' SOURCES_helm_charts_mimir_distributed='mimir' SOURCES_helm_charts_tempo_distributed='tempo' SOURCES_opentelemetry='opentelemetry-docs' -SOURCES_plugins_grafana_jira_datasource='jira-datasource' -SOURCES_plugins_grafana_mongodb_datasource='mongodb-datasource' -SOURCES_plugins_grafana_splunk_datasource='splunk-datasource' +SOURCES_plugins_grafana_datadog_datasource='datadog-datasource' +SOURCES_plugins_grafana_oracle_datasource='oracle-datasource' VERSIONS_as_code='UNVERSIONED' VERSIONS_grafana_cloud='UNVERSIONED' @@ -229,6 +309,8 @@ VERSIONS_grafana_cloud_k6='UNVERSIONED' VERSIONS_grafana_cloud_data_configuration_integrations='UNVERSIONED' VERSIONS_grafana_cloud_frontend_observability_faro_web_sdk='UNVERSIONED' VERSIONS_opentelemetry='UNVERSIONED' +VERSIONS_plugins_grafana_datadog_datasource='latest' +VERSIONS_plugins_grafana_oracle_datasource='latest' VERSIONS_technical_documentation='UNVERSIONED' VERSIONS_website='UNVERSIONED' VERSIONS_writers_toolkit='UNVERSIONED' @@ -237,8 +319,10 @@ PATHS_grafana_cloud='content/docs/grafana-cloud' PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed' PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed' PATHS_mimir='docs/sources/mimir' +PATHS_plugins_grafana_datadog_datasource='docs/sources' +PATHS_plugins_grafana_oracle_datasource='docs/sources' PATHS_tempo='docs/sources/tempo' -PATHS_website='content/docs' +PATHS_website='content' # identifier STR # Replace characters that are not valid in an identifier with underscores. @@ -253,6 +337,77 @@ aget() { eval echo '$'"$(identifier "$1")_$(identifier "$2")" } +# src returns the project source repository name for a project. +src() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget SOURCES "${_project}")" ]; then + echo plugins-private + else + aget SOURCES "${_project}" + fi + ;; + *) + if [ -z "$(aget SOURCES "${_project}")" ]; then + echo "${_project}" + else + aget SOURCES "${_project}" + fi + ;; + esac + + unset _project +} + +# path returns the relative path within the repository that contain the docs for a project. +path() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget PATHS "${_project}")" ]; then + echo "${_project}/docs/sources" + else + aget PATHS "${_project}" + fi + ;; + *) + if [ -z "$(aget PATHS "${_project}")" ]; then + echo "docs/sources" + else + aget PATHS "${_project}" + fi + esac + + unset _project +} + +# version returns the version for a project. Unversioned projects return the special value 'UNVERSIONED'. +version() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget VERSIONS "${_project}")" ]; then + echo "UNVERSIONED" + else + aget VERSIONS "${_project}" + fi + ;; + *) + if [ -z "$(aget VERSIONS "${_project}")" ]; then + echo latest + else + aget VERSIONS "${_project}" + fi + esac + + unset _project +} + + # new_proj populates a new project structure. new_proj() { _project="$1" @@ -263,31 +418,19 @@ new_proj() { # If version is not set, use the script mapping of project to default versions if it exists. # Fallback to 'latest'. if [ -z "${_version}" ]; then - if [ -z "$(aget VERSIONS "${_project}")" ]; then - _version=latest - else - _version="$(aget VERSIONS "${_project}")" - fi + _version="$(version "${_project}")" fi # If repo is not set, use the script mapping of project to repo name if it exists. # Fallback to using the project name. if [ -z "${_repo}" ]; then - if [ -z "$(aget SOURCES "${_project}")" ]; then - _repo="${_project}" - else - _repo="$(aget SOURCES "${_project}")" - fi + _repo="$(src "${_project}")" fi # If path is not set, use the script mapping of project to docs sources path if it exists. # Fallback to using 'docs/sources'. if [ -z "${_path}" ]; then - if [ -z "$(aget PATHS "${_project}")" ]; then - _path="docs/sources" - else - _path="$(aget PATHS "${_project}")" - fi + _path="$(path "${_project}")" fi echo "${_project}:${_version}:${_repo}:${_path}" @@ -301,7 +444,7 @@ proj_url() { $1 POSIX_HERESTRING - if [ "${_project}" = 'website' ]; then + if [ "${_project}" = website ]; then echo "http://localhost:${DOCS_HOST_PORT}/docs/" unset _project _version @@ -335,8 +478,8 @@ proj_dst() { $1 POSIX_HERESTRING - if [ "${_project}" = 'website' ]; then - echo '/hugo/content/docs' + if [ "${_project}" = website ]; then + echo '/hugo/content' unset _project _version return @@ -394,7 +537,7 @@ proj_canonical() { $1 POSIX_HERESTRING - if [ "${_project}" = 'website' ]; then + if [ "${_project}" = website ]; then echo '/docs' unset _project _version @@ -463,12 +606,11 @@ await_build() { while [ "${i}" -ne "${max}" ] do sleep 1 - debg "Retrying request to webserver assuming the process is still starting up." + debg "Retrying request to web server assuming the process is still starting up." i=$((i + 1)) if ${req} "${url}"; then - echo - echo "View documentation locally:" + printf '\r\nView documentation locally:\r\n' for x in ${url_src_dst_vers}; do IFS='^' read -r url _ _ <&2 + printf 'DEBG: %s\r\n' "$1" >&2 fi } errr() { - echo "ERRR: $1" >&2 + printf 'ERRR: %s\r\n' "$1" >&2 } note() { - echo "NOTE: $1" >&2 + printf 'NOTE: %s\r\n' "$1" >&2 } url_src_dst_vers="$(url_src_dst_vers "$@")" @@ -518,11 +659,20 @@ for arg in "$@"; do IFS=: read -r _project _ _repo _ </dev/null 2>&1; then + errr '`jq` must be installed for the `doc-validator` target to work.' + note 'To install `jq`, refer to https://jqlang.github.io/jq/download/,' + + exit 1 + fi + proj="$(new_proj "$1")" - echo + printf '\r\n' "${PODMAN}" run \ --init \ --interactive \ - --name "${DOCS_CONTAINER}" \ --platform linux/amd64 \ --rm \ --tty \ @@ -577,30 +733,37 @@ case "${image}" in "${DOCS_IMAGE}" \ "--include=${DOC_VALIDATOR_INCLUDE}" \ "--skip-checks=${DOC_VALIDATOR_SKIP_CHECKS}" \ - /hugo/content/docs \ - "$(proj_canonical "${proj}")" | sed "s#$(proj_dst "${proj}")#sources#" + "/hugo/content$(proj_canonical "${proj}")" \ + "$(proj_canonical "${proj}")" \ + | sed "s#$(proj_dst "${proj}")#sources#" \ + | jq -r '"ERROR: \(.location.path):\(.location.range.start.line // 1):\(.location.range.start.column // 1): \(.message)" + if .suggestions[0].text then "\nSuggestion: \(.suggestions[0].text)" else "" end' ;; 'grafana/vale') proj="$(new_proj "$1")" - echo + printf '\r\n' "${PODMAN}" run \ --init \ --interactive \ - --name "${DOCS_CONTAINER}" \ - --platform linux/amd64 \ --rm \ + --workdir /etc/vale \ --tty \ ${volumes} \ "${DOCS_IMAGE}" \ "--minAlertLevel=${VALE_MINALERTLEVEL}" \ - --config=/etc/vale/.vale.ini \ - --output=line \ + '--glob=*.md' \ + --output=/etc/vale/rdjsonl.tmpl \ /hugo/content/docs | sed "s#$(proj_dst "${proj}")#sources#" ;; *) tempfile="$(mktemp -t make-docs.XXX)" cat <"${tempfile}" #!/usr/bin/env bash + +tc() { + set \${*,,} + echo \${*^} +} + for redirect in ${redirects}; do IFS='^' read -r path ver <<<"\${redirect}" echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\nversioned: true\\n---\\n" > "\${path/\${ver}/_index.md}" @@ -609,8 +772,12 @@ done for x in "${url_src_dst_vers}"; do IFS='^' read -r _ _ dst _ <<<"\${x}" + title="\${dst%/*}" + title="\$(tc \${title##*/})" while [[ -n "\${dst}" ]]; do - touch "\${dst}/_index.md" + if [[ ! -f "\${dst}/_index.md" ]]; then + echo -e "---title: \${title}\\n---\\n\\n# \${title}\\n\\n{{< section >}}" > "\${dst}/_index.md" + fi dst="\${dst%/*}" done done @@ -630,33 +797,39 @@ ${PODMAN} run \ --env=HUGO_REFLINKSERRORLEVEL=${HUGO_REFLINKSERRORLEVEL} \ --init \ --interactive \ - --name=${DOCS_CONTAINER} \ - --platform=linux/amd64 \ --publish=${DOCS_HOST_PORT}:3002 \ --publish=3003:3003 \ --rm \ --tty \ ${volumes} \ - ${DOCS_IMAGE} \ - /entrypoint + ${DOCS_IMAGE} EOF - await_build http://localhost:3003 & - if [ -n "${DEBUG}" ]; then - ${cmd} + if [ -n "${ENTER}" ]; then + ${cmd} /bin/bash + elif [ -n "${DEBUG}" ]; then + await_build http://localhost:3003 & + + debg "${cmd} /entrypoint" + ${cmd} /entrypoint else - ${cmd} 2>&1| sed \ - -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \ - -e '/^hugo server/ d' \ - -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \ - -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \ - -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \ - -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \ - -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \ - -e '/website-proxy/ d' \ - -e '/rm -rf dist*/ d' \ - -e '/Press Ctrl+C to stop/ d' \ - -e '/make/ d' || echo + await_build http://localhost:3003 & + + ${cmd} /entrypoint 2>&1\ + | sed -u \ + -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \ + -e '/^hugo server/ d' \ + -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \ + -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \ + -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \ + -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \ + -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \ + -e '/website-proxy/ d' \ + -e '/rm -rf dist*/ d' \ + -e '/Press Ctrl+C to stop/ d' \ + -e '/make/ d' \ + -e '/WARNING: The manual_mount source directory/ d' \ + -e '/docs\/_index.md .* not found/ d' fi ;; esac