Updates Docker and Kubernetes to use EDOT Collector instead of APM Server

docker/README.md

@@ -1,8 +1,10 @@
 # Running your own Elastic Stack with Docker
 
-If you'd like to start Elastic locally, you can use the provided
-[docker-compose-elastic.yml](docker-compose-elastic.yml) file. This starts
-Elasticsearch, Kibana, and APM Server and only requires Docker installed.
+If you would like to start a local Elastic Stack with Docker, use
+[docker-compose-elastic.yml](docker-compose-elastic.yml).
+
+This starts Elasticsearch, Kibana and Elastic Distribution of OpenTelemetry
+(EDOT) Collector.
 
 Note: If you haven't checked out this repository, all you need is one file:
 ```bash
@@ -26,3 +28,21 @@ Clean up when finished, like this:
 ```bash
 docker compose -f docker-compose-elastic.yml down
 ```
+
+## OpenTelemetry
+
+### Metrics
+
+If your application only sends logs or traces, you can skip this section.
+
+EDOT Collector supports delta, not cumulative metrics. Applications that send
+OpenTelemetry metrics using the official OTEL SDK need to export this variable:
+```bash
+OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=delta
+```
+
+Alternatively, you can use [EDOT language SDKs][edot-sdks] which set this by
+default.
+
+---
+[edot-sdks]: https://github.com/elastic/opentelemetry?tab=readme-ov-file#edot-sdks--agents

docker/docker-compose-elastic.yml

@@ -1,8 +1,69 @@
+# This is a Docker Compose file that runs a local Elastic Stack comprised of
+# Elasticsearch, Kibana and Elastic Distribution of OpenTelemetry (EDOT)
+# Collector.
 name: elastic-stack
 
+configs:
+  # This is the minimal yaml configuration needed to listen on all interfaces
+  # for OTLP logs, metrics and traces, exporting to Elasticsearch.
+  edot-collector-config:
+    content: |
+      receivers:
+        otlp:
+          protocols:
+            grpc:
+              endpoint: 0.0.0.0:4317
+            http:
+              endpoint: 0.0.0.0:4318
+
+      connectors:
+        elasticapm:
+
+      processors:
+        elastictrace:
+
+      exporters:
+        elasticsearch:
+          endpoint: http://elasticsearch:9200
+          user: elastic
+          password: elastic
+          mapping:
+            mode: otel
+          logs_dynamic_index:
+            enabled: true
+          metrics_dynamic_index:
+            enabled: true
+          traces_dynamic_index:
+            enabled: true
+          flush:
+              bytes: 1048576  # apm-server default instead of 5000000
+              interval: 1s  # apm-server default instead of 30s
+
+      service:
+        pipelines:
+          traces:
+            receivers: [otlp]
+            processors: [elastictrace]
+            exporters: [elasticapm, elasticsearch]
+
+          metrics:
+            receivers: [otlp]
+            processors: []
+            exporters: [elasticsearch]
+
+          metrics/aggregated:
+            receivers: [elasticapm]
+            processors: []
+            exporters: [elasticsearch]
+
+          logs:
+            receivers: [otlp]
+            processors: []
+            exporters: [elasticapm, elasticsearch]
+
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0
     container_name: elasticsearch
     ports:
       - 9200:9200
@@ -38,7 +99,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0
     container_name: elasticsearch_settings
     restart: 'no'
     # gen-ai assistants in kibana save state in a way that requires system
@@ -53,7 +114,7 @@ services:
       '
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.17.2
+    image: docker.elastic.co/kibana/kibana:9.0.0
     container_name: kibana
     depends_on:
       elasticsearch_settings:
@@ -76,27 +137,22 @@ services:
       retries: 300
       interval: 1s
 
-  apm-server:
-    image: docker.elastic.co/apm/apm-server:8.17.2
-    container_name: apm-server
+  otel-collector:
+    image: docker.elastic.co/elastic-agent/elastic-otel-collector:9.0.0
+    container_name: otel-collector
     depends_on:
       elasticsearch:
         condition: service_healthy
-    command: >
-      apm-server
-        -E apm-server.kibana.enabled=true
-        -E apm-server.kibana.host=http://kibana:5601
-        -E apm-server.kibana.username=elastic
-        -E apm-server.kibana.password=elastic
-        -E output.elasticsearch.hosts=["http://elasticsearch:9200"]
-        -E output.elasticsearch.username=elastic
-        -E output.elasticsearch.password=elastic
-    cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"]
-    cap_drop: ["ALL"]
+    command: [
+      "--config=/etc/otelcol-contrib/config.yaml",
+    ]
+    configs:
+      - source: edot-collector-config
+        target: /etc/otelcol-contrib/config.yaml
     ports:
-      - 8200:8200
+      - "4317:4317"  # grpc
+      - "4318:4318"  # http
     healthcheck:
-      test: ["CMD-SHELL", "bash -c 'echo -n > /dev/tcp/127.0.0.1/8200'"]
+      test: ["CMD-SHELL", "bash -c 'echo -n > /dev/tcp/127.0.0.1/4317'"]
       retries: 300
-      interval: 1s
-
+      interval: 1s

example-apps/chatbot-rag-app/README.md

@@ -22,8 +22,9 @@ Copy [env.example](env.example) to `.env` and fill in values noted inside.
 ## Installing and connecting to Elasticsearch
 
 There are a number of ways to install Elasticsearch. Cloud is best for most
-use-cases. We also have [docker-compose-elastic.yml][docker-compose-elastic],
-that starts Elasticsearch, Kibana, and APM Server on your laptop in one step.
+use-cases. We also have [docker-compose-elastic.yml][docker-compose],
+that starts Elasticsearch, Kibana, and Elastic Distribution of OpenTelemetry
+(EDOT) Collector on your laptop in one step.
 
 Once you decided your approach, edit your `.env` file accordingly.
 
@@ -84,8 +85,8 @@ copied to a file name `.env` and updated with `ELASTICSEARCH_URL` and
 For example, if you started your Elastic Stack with [k8s-manifest-elastic.yml][k8s-manifest-elastic],
 you would update these values:
 ```
-ELASTICSEARCH_URL=http://elasticsearch:9200
-OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
+ELASTICSEARCH_URL=http://elasticsearch.default.svc:9200
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.default.svc:4318
 ```
 
 Then, import your `.env` file as a configmap like this:
@@ -132,6 +133,7 @@ kubectl port-forward deployment.apps/chatbot-rag-app 4000:4000 &
 Clean up when finished, like this:
 ```bash
 kubectl delete -f k8s-manifest.yml
+kubectl delete configmap chatbot-rag-app-env
 ```
 
 ### Run with Python
@@ -197,16 +199,16 @@ prefix `python` with `opentelemetry-instrument` to enable OpenTelemetry.
 dotenv run -- opentelemetry-instrument python api/app.py
 ```
 
-[env.example](env.example) defaults to use Elastic APM server, started by
-[docker-compose-elastic.yml](../../docker). If you start your Elastic stack
-this way, you can access Kibana like this, authenticating with the username
+[env.example](env.example) defaults to use an OpenTelemetry Collector,
+specifically Elastic Distribution of OpenTelemetry (EDOT) Collector, if you
+started your Elastic Stack with [docker-compose-elastic.yml][docker-compose].
+If you did, you can access Kibana like this, authenticating with the username
 "elastic" and password "elastic":
 
 http://localhost:5601/app/apm/traces?rangeFrom=now-15m&rangeTo=now
 
-Under the scenes, chatbot-rag-app is automatically instrumented by the Elastic
-Distribution of OpenTelemetry (EDOT) Python. You can see more details about
-EDOT Python [here](https://github.com/elastic/elastic-otel-python).
+Under the scenes, chatbot-rag-app is automatically instrumented by EDOT Python.
+You can see more details about EDOT Python [here][edot-python].
 
 OpenTelemetry support for LLM providers not included in EDOT Python are provided
 by the [Langtrace Python SDK](https://docs.langtrace.ai/sdk/python_sdk).
@@ -260,5 +262,6 @@ docker compose up --build --force-recreate
 ---
 [loader-docs]: https://python.langchain.com/docs/how_to/#document-loaders
 [install-es]: https://www.elastic.co/search-labs/tutorials/install-elasticsearch
-[docker-compose-elastic]: ../../docker/docker-compose-elastic.yml
+[docker-compose]: ../../docker/docker-compose-elastic.yml
+[edot-python]: https://github.com/elastic/elastic-otel-python
 [k8s-manifest-elastic]: ../../k8s/k8s-manifest-elastic.yml

example-apps/chatbot-rag-app/data/index_data.py

@@ -139,6 +139,7 @@ def main():
         if isinstance(e, ApiError) and e.status_code != 408:
             raise
         warn(f"Error occurred, will retry after ML jobs complete: {e}")
+        time.sleep(2)  # Checking too fast could return no tasks and fail again
         await_ml_tasks()
         es.indices.delete(index=INDEX, ignore_unavailable=True)
         store.add_documents(list(docs))

example-apps/chatbot-rag-app/env.example

@@ -72,12 +72,13 @@ OTEL_SDK_DISABLED=true
 # Assign the service name that shows up in Kibana
 OTEL_SERVICE_NAME=chatbot-rag-app
 
-# Default to send logs, traces and metrics to an Elastic APM server accessible
-# via localhost.
+# Default to send logs, traces and metrics to an OpenTelemetry collector,
+# accessible via localhost. For example, Elastic Distribution of OpenTelemetry
+# (EDOT) Collector.
 #
-# When running inside Kubernetes, set to http://elasticsearch.default.svc:9200
+# When running inside Kubernetes, set to http://otel-collector.default.svc:4318
 # or similar.
-OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
 
 # Change to 'false' to hide prompt and completion content

k8s/README.md

@@ -1,8 +1,10 @@
 # Running your own Elastic Stack with Kubernetes
 
-If you'd like to start Elastic with Kubernetes, you can use the provided
-[manifest-elastic.yml](manifest-elastic.yml) file. This starts
-Elasticsearch, Kibana, and APM Server in an existing Kubernetes cluster.
+If you would like to start a local Elastic Stack with Kubernetes, use
+[manifest-elastic.yml](manifest-elastic.yml).
+
+This starts Elasticsearch, Kibana and Elastic Distribution of OpenTelemetry
+(EDOT) Collector.
 
 Note: If you haven't checked out this repository, all you need is one file:
 ```bash
@@ -28,7 +30,7 @@ Elastic Stack version can take a long time due to image pulling.
 kubectl wait --for=condition=available --timeout=10m \
   deployment/elasticsearch \
   deployment/kibana \
-  deployment/apm-server
+  deployment/otel-collector
 ```
 
 Next, forward the kibana port:
@@ -45,3 +47,21 @@ Clean up when finished, like this:
 ```bash
 kubectl delete -f k8s-manifest-elastic.yml
 ```
+
+## OpenTelemetry
+
+### Metrics
+
+If your application only sends logs or traces, you can skip this section.
+
+EDOT Collector supports delta, not cumulative metrics. Applications that send
+OpenTelemetry metrics using the official OTEL SDK need to export this variable:
+```bash
+OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=delta
+```
+
+Alternatively, you can use [EDOT language SDKs][edot-sdks] which set this by
+default.
+
+---
+[edot-sdks]: https://github.com/elastic/opentelemetry?tab=readme-ov-file#edot-sdks--agents