Skip to content

Commit 7fbcf34

Browse files
smithclaysmithclay
committed
add kafka to logs example
1 parent f074eff commit 7fbcf34

File tree

5 files changed

+160
-0
lines changed

5 files changed

+160
-0
lines changed

‎kafka/connect-plugins/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
confluentinc*
2+
*.zip

‎kafka/connect-plugins/.gitkeep‎

Whitespace-only changes.

‎kafka/docker-compose.yml‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Copyright 2023 The original authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
version: '3.8'
16+
services:
17+
kafka:
18+
image: "confluentinc/cp-kafka:7.5.0"
19+
ports:
20+
- "9092:9092"
21+
container_name: kafka
22+
environment:
23+
KAFKA_NODE_ID: 101
24+
# random cluster ID used for formatting LOG_DIR for KRaft
25+
CLUSTER_ID: 'xtzWWN4bTjitpL3kfd9s5g'
26+
KAFKA_CONTROLLER_QUORUM_VOTERS: '101@kafka:29093'
27+
KAFKA_PROCESS_ROLES: 'broker,controller'
28+
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092'
29+
KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://0.0.0.0:9092,CONTROLLER://kafka:29093'
30+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
31+
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
32+
KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
33+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
34+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
35+
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
36+
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
37+
CONFLUENT_METRICS_ENABLE: 'false'
38+
KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
39+
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
40+
KAFKA_SUPER_USERS: 'User:anonymous'
41+
networks:
42+
- kafka-platform
43+
44+
schema-registry:
45+
image: confluentinc/cp-schema-registry:7.5.0
46+
hostname: schema-registry
47+
container_name: schema-registry
48+
ports:
49+
- "8081:8081"
50+
environment:
51+
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:29092
52+
SCHEMA_REGISTRY_HOST_NAME: schema-registry
53+
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
54+
depends_on:
55+
- kafka
56+
networks:
57+
- kafka-platform
58+
akhq:
59+
image: tchiotludo/akhq
60+
hostname: akhq
61+
container_name: akhq
62+
depends_on:
63+
- kafka
64+
ports:
65+
- "8087:8080"
66+
environment:
67+
AKHQ_CONFIGURATION: |
68+
akhq:
69+
connections:
70+
docker-kafka-server:
71+
properties:
72+
bootstrap.servers: "kafka:29092"
73+
schema-registry:
74+
url: "http://schema-registry:8081"
75+
connect:
76+
- name: "connect"
77+
url: "http://connect:8083"
78+
networks:
79+
- kafka-platform
80+
81+
connect:
82+
image: confluentinc/cp-kafka-connect:7.5.0
83+
container_name: connect
84+
depends_on:
85+
- kafka
86+
volumes:
87+
- ./connect-plugins:/connect-plugins
88+
ports:
89+
- "8083:8083"
90+
- "8000:8000"
91+
environment:
92+
CONNECT_BOOTSTRAP_SERVERS: 'kafka:29092'
93+
CONNECT_REST_ADVERTISED_HOST_NAME: connect
94+
CONNECT_REST_PORT: 8083
95+
CONNECT_GROUP_ID: kafka-connect
96+
CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs
97+
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
98+
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
99+
CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets
100+
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
101+
CONNECT_STATUS_STORAGE_TOPIC: _connect-status
102+
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
103+
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
104+
CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter
105+
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
106+
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
107+
CONNECT_PLUGIN_PATH: "/usr/local/share/kafka/plugins,/usr/share/filestream-connectors,/connect-plugins"
108+
networks:
109+
- kafka-platform
110+
111+
networks:
112+
kafka-platform:
113+
driver: bridge

‎kafka/plugin-config.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"name": "elasticsearch-sink",
3+
"config": {
4+
"connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
5+
"tasks.max": "1",
6+
"topics": "example-topic",
7+
"key.ignore": "true",
8+
"schema.ignore": "true",
9+
"connection.url": "https://logingest.lightstep.com:443",
10+
"connection.username": "lightstep",
11+
"connection.password": "your-access-token",
12+
"type.name": "_doc",
13+
"name": "elasticsearch-sink",
14+
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
15+
"value.converter.schemas.enable": "false"
16+
}
17+
}

‎kafka/readme.md‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
### Stream logs from Kafka
2+
3+
Streams Kafka topics to ServiceNow Cloud Observability logs
4+
5+
#### Run
6+
7+
1) Download and unzip https://www.confluent.io/hub/confluentinc/kafka-connect-elasticsearch into `connect-plugins/`
8+
9+
2) Run Kafka
10+
11+
```
12+
docker-compose up
13+
14+
# wait a few minutes...
15+
```
16+
17+
3) Configure connector
18+
19+
```
20+
curl -X POST http://localhost:8083/connectors -H 'Content-Type: application/json' -d @plugin-config.json
21+
```
22+
23+
4) Send some data
24+
25+
```
26+
docker exec -i kafka bash -c "echo '{\"userId\": \"1\", \"action\": \"login\"}' | /usr/bin/kafka-console-producer --broker-list kafka:9092 --topic example-topic"
27+
28+
```

0 commit comments

Comments
 (0)