feat(mongo_stream_source_plugin): added examples forlder, cleanup

usedatabrew · Nov 21, 2023 · 407d169 · 407d169
1 parent d7ada7c
commit 407d169
Show file tree

Hide file tree

Showing 3 changed files with 110 additions and 9 deletions.
diff --git a/config/example.yaml b/config/example.yaml
@@ -16,23 +16,77 @@ service:
   stream_schema:
     - stream: flights
       columns:
-        - name: _id
-          databrewType: UUID
-          nativeConnectorType: ObjectID
+        - name: flight_id
+          databrewType: Int32
+          nativeConnectorType: integer
           pk: true
           nullable: false
-        - name: message
+        - name: flight_number
+          databrewType: Int32
+          nativeConnectorType: integer
+          pk: false
+          nullable: true
+        - name: departure_airport
+          databrewType: String
+          nativeConnectorType: character varying
+          pk: false
+          nullable: true
+        - name: arrival_airport
+          databrewType: String
+          nativeConnectorType: character varying
+          pk: false
+          nullable: true
+        - name: departure_date
+          databrewType: Date32
+          nativeConnectorType: date
+          pk: false
+          nullable: true
+        - name: arrival_date
+          databrewType: Date32
+          nativeConnectorType: date
+          pk: false
+          nullable: true
+        - name: departure_time
           databrewType: String
-          nativeConnectorType: String
+          nativeConnectorType: character varying
           pk: false
+          nullable: true
+        - name: arrival_time
+          databrewType: String
+          nativeConnectorType: character varying
+          pk: false
+          nullable: true
+        - name: flight_duration
+          databrewType: Float64
+          nativeConnectorType: double precision
+          pk: false
+          nullable: true
+        - name: flight_status
+          databrewType: String
+          nativeConnectorType: character varying
+          pk: false
+          nullable: true
+        - name: id
+          databrewType: Int32
+          nativeConnectorType: integer
+          pk: true
           nullable: false
 
 source:
-  driver: mongo_stream
+  driver: postgres_cdc
   config:
-    uri: mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+1.6.2
-    database: test
+    host: databrew-testing-instance.postgres.database.azure.com
+    slot_name: databrew_replication_slot_15_11910
+    user: postgres
+    password: Lorem123
+    port: 5432
+    schema: public
     stream_snapshot: true
+    snapshot_memory_safety_factor: 0.1
+    snapshot_batch_size: 5000
+    database: mocks
+    tables:
+      - flights
 
 processors:
   - driver: openai
@@ -46,4 +100,12 @@ processors:
 
 sink:
   driver: stdout
-  config: { }
+  config:
+    brokers:
+      - "pkc-ygz0p.norwayeast.azure.confluent.cloud:9092"
+    sasl: true
+    sasl_user: 2BUOXTEG5AKL4IEC
+    sasl_password: stm1jZufR0njS4hVHnYgX4zrziADac/BZUGv2qh7Z0RBU3alrNTbxdMDob0p0aLg
+    sasl_mechanism: PLAIN
+    bind_topic_to_stream: true
+    topic_name: llsobgnp-mangust
diff --git a/docker-compose.yaml → examples/mongo_strream/docker-compose.yaml b/docker-compose.yaml → examples/mongo_strream/docker-compose.yaml
diff --git a/examples/mongo_strream/example.yaml b/examples/mongo_strream/example.yaml
@@ -0,0 +1,39 @@
+service:
+  id: 123
+  pipeline_id: 1234
+  enable_etcd_registry: true
+  etcd:
+    host: http://137.135.132.105:2379
+  enable_influx: true
+  influx:
+    host: https://eu-central-1-1.aws.cloud2.influxdata.com
+    token: dwWFG-NXl8uy0dmiptLjN1-B1-5JIlyEnwCHVxmPgY-6o6f62-G09dtYDg1SEYk6mL3wGG_7DIMVer8Yoe-e9Q==
+    org: DataBrew, Inc.
+    bucket: databrew_ingestor
+    group_name: org_11
+    pipeline_id: 1233
+  reload_on_restart: false
+  stream_schema:
+    - stream: test
+      columns:
+        - name: _id
+          databrewType: UUID
+          nativeConnectorType: ObjectID
+          pk: true
+          nullable: false
+        - name: message
+          databrewType: String
+          nativeConnectorType: String
+          pk: false
+          nullable: false
+
+source:
+  driver: mongo_stream
+  config:
+    uri: mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+1.6.2
+    database: test
+    stream_snapshot: true
+
+sink:
+  driver: stdout
+  config: { }