Create V3 docs and move current docs to olddocs folder

.github/workflows/check-generated-md.yml

@@ -6,7 +6,7 @@ on:
      - opened
      - synchronize
    paths:
-     - 'docs/**'
+     - 'olddocs/**'
 
 jobs:
  check-generated-markdown:
@@ -20,10 +20,10 @@ jobs:
          python-version: '3.10'
      - name: Install dependencies
        run: |
-         cd docs/gen_edot_col_components
+         cd olddocs/gen_edot_col_components
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Check Markdown Generation
        run: |
-         cd docs
+         cd olddocs
          make check-md-gen

.github/workflows/check-links.yml

@@ -22,11 +22,11 @@ jobs:
           cache-version: 0
       - name: Install Dependencies
         run: |
-          cd docs
+          cd olddocs
           make install
       - name: Check Links
         run: |
-          cd docs
+          cd olddocs
           make check-links
 
   # This job detects if any files changed in a given path
@@ -42,7 +42,7 @@ jobs:
         with:
           filters: |
             docs:
-              - 'docs/**'
+              - 'olddocs/**'
 
       - name: Set output
         run: echo "docs_changed=${{ steps.filter.outputs.docs }}" >> $GITHUB_OUTPUT

.github/workflows/docs-build.yml

@@ -0,0 +1,19 @@
+---
+name: docs-build
+
+on:
+  push:
+    branches:
+        - main
+  pull_request_target: ~
+
+jobs:
+  docs-preview:
+    uses: elastic/docs-builder/.github/workflows/preview-build.yml@main
+    with:
+      path-pattern: docs/**
+    permissions:
+      id-token: write
+      deployments: write
+      contents: read
+      pull-requests: read

.github/workflows/docs-cleanup.yml

@@ -0,0 +1,15 @@
+---
+name: docs-cleanup
+
+on:
+  pull_request_target:
+    types:
+      - closed
+
+jobs:
+  docs-preview:
+    uses: elastic/docs-builder/.github/workflows/preview-cleanup.yml@main
+    permissions:
+      contents: none
+      id-token: write
+      deployments: write

.github/workflows/jekyll.yml

@@ -41,19 +41,19 @@ jobs:
           cache-version: 0 # Increment this number if you need to re-download cached gems
       - name: Install Dependencies
         run: |
-          cd docs
+          cd olddocs
           bundle install
       - name: Build site
         # Outputs to the './_site' directory by default
         run: |
-          cd docs
+          cd olddocs
           bundle exec jekyll build
       - name: Upload artifact
         id: deployment
         # Automatically uploads an artifact from the './_site' directory by default
         uses: actions/upload-pages-artifact@v3
         with:
-          path: docs/_site
+          path: olddocs/_site
 
   # Deployment job
   deploy:

docs/docset.yml

@@ -0,0 +1,69 @@
+project: 'EDOT docs'
+cross_links:
+  - apm-agent-android
+  - apm-agent-dotnet
+  - apm-agent-go
+  - apm-agent-ios
+  - apm-agent-java
+  - apm-agent-nodejs
+  - apm-agent-php
+  - apm-agent-python
+  - apm-agent-ruby
+  - apm-agent-rum-js
+  - apm-aws-lambda
+  - apm-k8s-attacher
+  - apm-server
+  - cloud
+  - cloud-on-k8s
+  - docs-content
+  - elasticsearch
+  - kibana
+toc:
+  - toc: reference
+# - toc: release-notes
+subs:
+  edot: Elastic Distribution of OpenTelemetry
+  ecloud:   "Elastic Cloud"
+  ech:   "Elastic Cloud Hosted"
+  ess:   "Elasticsearch Service"
+  ece:   "Elastic Cloud Enterprise"
+  serverless-full:   "Elastic Cloud Serverless"
+  security-app:   "Elastic Security app"
+  stack-manage-app:   "Stack Management"
+  stack-monitor-app:   "Stack Monitoring"
+  rules-ui:   "Rules"
+  connectors-ui:   "Connectors"
+  connectors-feature:   "Actions and Connectors"
+  hosted-ems:   "Elastic Maps Server"
+  data-sources:   "data views"
+  agent:   "Elastic Agent"
+  agents:   "Elastic Agents"
+  fleet:   "Fleet"
+  fleet-server:   "Fleet Server"
+  package-manager:   "Elastic Package Manager"
+  stack:   "Elastic Stack"
+  es:   "Elasticsearch"
+  kib:   "Kibana"
+  ls:   "Logstash"
+  security-features:   "security features"
+  stack-security-features:   "Elastic Stack security features"
+  endpoint-sec:   "Endpoint Security"
+  swimlane:   "Swimlane"
+  sn:   "ServiceNow"
+  sn-itsm:   "ServiceNow ITSM"
+  sn-itom:   "ServiceNow ITOM"
+  sn-sir:   "ServiceNow SecOps"
+  ibm-r:   "IBM Resilient"
+  webhook:   "Webhook"
+  webhook-cm:   "Webhook - Case Management"
+  opsgenie:   "Opsgenie"
+  bedrock:   "Amazon Bedrock"
+  gemini:   "Google Gemini"
+  hive:   "TheHive"
+  report-features:   "reporting features"
+  ml:   "machine learning"
+  ccs:   "cross-cluster search"
+  anomaly-job:   "anomaly detection job"
+  observability:   "Observability"
+  kib-repo:   "https://github.com/elastic/kibana/"
+  kib-pull: "https://github.com/elastic/kibana/pull/"

docs/reference/_snippets/guided-instructions.md

@@ -0,0 +1,6 @@
+Use the **Add data** screen in Elastic Observability to generate install commands that are already configured with the values you need.
+
+1. Open Elastic Observability.
+2. Go to **Add data**.
+3. Select what you want to monitor.
+4. Follow the instructions.

docs/reference/_snippets/retrieve-credentials.md

@@ -0,0 +1,9 @@
+Retrieve your Elasticsearch URL and your API key:
+
+1. Retrieve the Elasticsearch URL for your Elastic Cloud deployment:
+
+    1. Go to the [Elastic Cloud console](https://cloud.elastic.co/).
+    2. Next to your deployment, select **Manage**.
+    3. Under **Applications** next to **Elasticsearch**, select **Copy endpoint**.
+
+2. Create an API Key following [these instructions](docs-content://deploy-manage/api-keys/elasticsearch-api-keys.md).

docs/reference/architecture/hosts_vms.md

@@ -0,0 +1,66 @@
+---
+navigation_title: Hosts / VMs environments
+description: Recommended EDOT architecture for host or virtual machine environments.
+applies_to:
+  stack:
+  serverless:
+    observability:
+products:
+  - cloud-serverless
+  - observability
+---
+
+# Hosts and VMs environments
+
+On host or virtual machine environments, deploy local, per-host OpenTelemetry Collector instances, here referred to as OTel Collector in Agent Mode.
+
+![VM-Edge](./../images/arch-vm-edge.png)
+
+These collectors have two main purposes:
+
+1.  The collection of local logs and infrastructure metrics. Refer to [this sample config file](https://github.com/elastic/elastic-agent/blob/main/internal/pkg/otel/samples/linux/managed_otlp/platformlogs_hostmetrics.yml) for recommended collector receiver configurations for hostmetrics and logs.
+2.  Enriching application telemetry from OTel SDKs that run within the instrumented applications on corresponding hosts with resource information.
+
+## Deployment scenarios
+
+See the recommended architectures per Elastic deployment scenarios:
+
+:::{note}
+Elastic's Observability solution is technically compatible with edge setups that are fully based on upstream OTel components as long as the ingestion path follows the recommendations outlined in the following sections.
+:::
+
+### Elastic Cloud Serverless
+
+Elastic Cloud Serverless provides a managed OTLP endpoint for ingestion of OpenTelemetry data.
+
+![VM-Serverless](./../images/arch-vm-serverless.png)
+
+Users can send their OTel data from the [edge setup](#hosts--vms-environments) in OTel-native format through OTLP without any additional requirements for self-managed preprocessing of data.
+
+### Elastic Cloud Hosted
+
+As of Elastic Stack version <STACK_VERSION> on Elastic Cloud Hosted (ECH), you need to run a self-hosted EDOT Collector in Gateway Mode to ingest OTel data from the [edge setup](#hosts--vms-environments) in OTel-native format into the Elastic-hosted Elasticsearch.
+
+![VM-ECH](./../images/arch-vm-ech.png)
+
+The EDOT Collector in Gateway mode enriches and pre-aggregates the data for a seamless experience in the Elastic Observability solution before ingesting it directly into Elasticsearch.
+
+If required, users can build their custom, EDOT-like collector [following these instructions](../edot-collector/custom-collector#build-a-custom-edot-like-collector).
+
+:::{note}
+The EDOT Gateway Collector does not send data through Elastic's Integration / APM Server on ECH to ingest data into Elasticsearch.
+:::
+
+:::{important}
+If self-managing an EDOT Gateway is not a valid option for you, refer to [Elastic's classic ingestion path for OTel data on ECH](https://www.elastic.co/guide/en/observability/current/apm-open-telemetry.html).
+:::
+
+### Self-managed
+
+In a self-managed deployment scenario, you need to host an EDOT Collector in Gateway mode that pre-processes and ingests OTel data from the [edge setup](#hosts--vms-environments) into the self-managed Elastic Stack.
+
+![VM-self-managed](./../images/arch-vm-self-managed.png)
+
+:::{note}
+Compared to [Elastic's classic ingestion paths](https://www.elastic.co/guide/en/observability/current/apm-open-telemetry.html) for OTel data, with the EDOT Gateway Collector there is no need for an APM Server anymore.
+:::

docs/reference/architecture/index.md

@@ -0,0 +1,19 @@
+---
+navigation_title: Reference Architecture
+description: Recommended architectures for EDOT with different Elastic deployment options.
+applies_to:
+  stack:
+  serverless:
+    observability:
+products:
+  - cloud-serverless
+  - observability
+---
+
+# Reference architecture
+
+The following sections outline the recommended architectures for Elastic Distributions of OpenTelemetry (EDOT) with different Elastic deployment options.
+
+- [Hosts and VMs](hosts_vms.md)
+- [Kubernetes](k8s.md)
+

docs/reference/architecture/k8s.md

@@ -0,0 +1,87 @@
+---
+navigation_title: Kubernetes environments
+description: Recommended EDOT architecture for Kubernetes environments.
+applies_to:
+  stack:
+  serverless:
+    observability:
+products:
+  - cloud-serverless
+  - observability
+---
+
+# Kubernetes Environments
+
+The recommended OTel architecture for Kubernetes clusters includes a set of OpenTelemetry collectors in different modes. The following diagram shows the different modes:
+
+![K8s-Cluster](./../images/arch-k8s-cluster.png)
+
+## Daemon mode
+
+The Collector in Daemon mode is deployed on each Kubernetes node to collect nodes-local logs and host metrics.
+
+The daemon collector also receives telemetry from applications instrumented with OTel SDKs and running on corresponding nodes.
+
+That collector enriches the application telemetry with resource information such as host and Kubernetes metadata.
+
+All data is then being sent through OTLP to the OTel or EDOT Gateway Collector.
+
+## Cluster mode
+
+The Collector in Cluster mode collects Kubernetes cluster-level metrics and sends them to the OTel or EDOT Gateway Collector using OTLP.
+
+## Gateway mode
+
+The OTel or EDOT Collector in Gateway mode gathers the OTel data from all other collectors and ingests it into the Elastic backend.
+
+For self-managed and Elastic Cloud Hosted deployment models the Gateway collector does some additional pre-processing of data.
+
+## Deployment scenarios
+
+See the recommended architectures per Elastic deployment scenarios:
+
+::::{note}
+Elastic's Observability solution is technically compatible with setups that are fully based on upstream OTel components, as long as the ingestion path follows the recommendations outlined in following sub-sections for the different Elastic deployment options.
+::::
+
+### Elastic Cloud Serverless
+
+Elastic Cloud Serverless provides a managed OTLP endpoint for ingestion of OpenTelemetry data.
+
+![K8s-Serverless](./../images/arch-k8s-serverless.png)
+
+For a Kubernetes setup, that means the Gateway Collector passes through the OTel data in native format using the OTLP protocol to the managed OTLP endpoint. There is no need for the Gateway Collector to do any Elastic-specific pre-processing.
+
+### Elastic Cloud Hosted
+
+With Elastic Cloud Hosted (ECH), OTel data is being directly ingested into the Elastic-hosted Elasticsearch instance.
+
+![K8s-ECH](./../images/arch-k8s-ech.png)
+
+The Gateway Collector needs to do some preprocessing, aggregation of metrics and, finally, it uses the Elasticsearch exporter to ingest data into ECH. 
+
+While the Daemon and Cluster collectors, as well as the OTel SDKs, can stay fully vendor agnostic or upstream, the Gateway Collector needs to be either an EDOT Collector or a [custom, EDOT-like collector](../edot-collector/custom-collector) containing the [required components and pre-processing pipelines](../edot-collector/config/default-config-k8s#direct-ingestion-into-elasticsearch).
+
+If required, users can build their custom, EDOT-like collector [following these instructions](../edot-collector/custom-collector#build-a-custom-edot-like-collector).
+
+::::{note}
+The EDOT Gateway Collector does not send data through Elastic's Integration / APM Server on ECH to ingest data into Elasticsearch.
+::::
+
+::::{important}
+If self-managing an EDOT Gateway is not a valid option for you, refer to [Elastic's classic ingestion path for OTel data on ECH](https://www.elastic.co/guide/en/observability/current/apm-open-telemetry.html).
+::::
+
+### Self-managed
+
+With a self-managed scenario the Gateway Collector ingests data directly into the self-managed Elasticsearch instance.
+
+![K8s-self-managed](./../images/arch-k8s-self-managed.png)
+
+The Gateway Collector does some preprocessing and aggregation of OTel data before ingesting it into Elasticsearch. 
+
+While the Daemon and Cluster collectors, as well as the OTel SDKs, can stay fully vendor agnostic or upstream, the Gateway Collector needs to be either an EDOT Collector or a [custom, EDOT-like collector](../edot-collector/custom-collector) containing the [required components and pre-processing pipelines](../edot-collector/config/default-config-k8s#direct-ingestion-into-elasticsearch).
+
+::::{note}
+Compared to [Elastic's classic ingestion paths](https://www.elastic.co/guide/en/observability/current/apm-open-telemetry.html) for OTel data, with the EDOT Gateway Collector there is no need for an APM Server anymore.
+::::