[DOP-21387] Rebuild documentation structure

.env.docker

@@ -2,7 +2,7 @@
 POSTGRES_DB=data_rentgen
 POSTGRES_USER=data_rentgen
 POSTGRES_PASSWORD=changeme
-POSTGRES_INITDB_ARGS: '--encoding=UTF-8 --lc-collate=C --lc-ctype=C'
+POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
 
 # Init Kafka
 KAFKA_CFG_NODE_ID=0
@@ -20,15 +20,18 @@ KAFKA_CLIENT_PASSWORDS=changeme
 KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN,SCRAM-SHA-256
 
 # Common backend config
-DATA_RENTGEN__LOGGING__PRESET=colored
 DATA_RENTGEN__DATABASE__URL=postgresql+asyncpg://data_rentgen:changeme@db:5432/data_rentgen
+DATA_RENTGEN__LOGGING__PRESET=colored
 
 # See Backend -> Server -> Configuration documentation
 DATA_RENTGEN__SERVER__DEBUG=false
 
 # See Backend -> Consumer -> Configuration documentation
-DATA_RENTGEN__KAFKA__BOOTSTRAP_SERVERS=kafka:9092
+DATA_RENTGEN__KAFKA__BOOTSTRAP_SERVERS=broker:9092
 DATA_RENTGEN__KAFKA__SECURITY__TYPE=scram-sha256
 DATA_RENTGEN__KAFKA__SECURITY__USER=data_rentgen
 DATA_RENTGEN__KAFKA__SECURITY__PASSWORD=changeme
 DATA_RENTGEN__KAFKA__COMPRESSION=zstd
+
+# See Frontend -> UI
+DATA_RENTGEN__UI__API_BROWSER_URL=http://localhost:8000

.env.local

@@ -9,3 +9,5 @@ export DATA_RENTGEN__KAFKA__SECURITY__PASSWORD=changeme
 export DATA_RENTGEN__KAFKA__COMPRESSION=zstd
 
 export DATA_RENTGEN__SERVER__DEBUG=true
+
+export DATA_RENTGEN__UI__API_BROWSER_URL=http://localhost:8000

README.rst

@@ -31,24 +31,26 @@
 What is Data.Rentgen?
 ---------------------
 
-Data.Rentgen is a DataLineage service compatible with `OpenLineage <https://openlineage.io/>`_ specification.
+Data.Rentgen is a Data Motion Lineage service, compatible with `OpenLineage <https://openlineage.io/>`_ specification.
 
-**Note**: service is under active development, and is not ready to use.
+**Note**: service is under active development, and is not ready to use yet.
 
 Goals
 -----
 
-* Collect lineage events produced by OpenLineage clients & integrations (Spark, Airflow, Flink, custom ones).
+* Collect lineage events produced by OpenLineage clients & integrations (Spark, Airflow).
+* Support consuming large amounts of lineage events, by using Kafka as event buffer and storing data in tables partitioned by event timestamp.
 * Store operation-grained events (instead of job grained `Marquez <https://marquezproject.ai/>`_), for better detalization.
-* Provide API for run ↔ dataset lineage, as well as parent run → children run lineage.
-* Support handling large amounts of lineage events, using Kafka as event buffer and storing data in tables partitioned by event timestamp.
+* Provide API for building run ↔ dataset lineage, as well as parent run → children run lineage.
+* Ability to build lineage graph with specific time boundaries (unlike Marquez there lineage is build only for last job run).
+* Ability to build lineage graph with different granularity. e.g. merge all individual Spark operations into Spark applicationId or Spark applicationName.
 
 Non-goals
 ---------
 
-* This is **not** a data catalog. Use `Datahub <https://datahubproject.io/>`_ or `OpenMetadata <https://open-metadata.org/>`_ instead.
-* Static dataset → dataset lineage (like view → table) is not supported.
-* Currently column-level lineage is not supported.
+* This is **not** a Data Catalog. Use `Datahub <https://datahubproject.io/>`_ or `OpenMetadata <https://open-metadata.org/>`_ instead.
+* Static Data Lineage like view → table is not supported.
+* Currently column-level lineage is collected by OpenLineage, but not yet consumed by Data.Rentgen.
 
 .. documentation
 

data_rentgen/consumer/settings/__init__.py

@@ -37,7 +37,7 @@ class ConsumerApplicationSettings(BaseSettings):
         DATA_RENTGEN__LOGGING__PRESET=json
     """
 
-    database: DatabaseSettings = Field(description=":ref:`Database settings <configuration-consumer-database>`")
+    database: DatabaseSettings = Field(description=":ref:`Database settings <configuration-database>`")
     kafka: KafkaSettings = Field(
         description=":ref:`Kafka settings <configuration-consumer-kafka>`",
     )

data_rentgen/consumer/settings/kafka.py

@@ -25,7 +25,7 @@ class KafkaSettings(BaseModel):
     )
     security: KafkaSecuritySettings = Field(
         default_factory=KafkaSecuritySettings,
-        description=":ref:`Kafka security settings <configuration-consumer-kafka-security>`",
+        description="Kafka security settings",
     )
     compression: KafkaCompression | None = Field(
         default=None,

data_rentgen/server/settings/__init__.py

@@ -37,7 +37,7 @@ class ServerApplicationSettings(BaseSettings):
         DATA_RENTGEN__SERVER__DEBUG=True
     """
 
-    database: DatabaseSettings = Field(description=":ref:`Database settings <configuration-server-database>`")
+    database: DatabaseSettings = Field(description=":ref:`Database settings <configuration-database>`")
     logging: LoggingSettings = Field(
         default_factory=LoggingSettings,
         description=":ref:`Logging settings <configuration-server-logging>`",

docker-compose.yml

@@ -40,7 +40,7 @@ services:
       db-migration:
         condition: service_completed_successfully
 
-  kafka:
+  broker:
     image: bitnami/kafka:3.7
     restart: unless-stopped
     env_file: .env.docker
@@ -60,7 +60,7 @@ services:
     restart: unless-stopped
     env_file: .env.docker
     depends_on:
-      kafka:
+      broker:
         condition: service_healthy
       db-migration:
         condition: service_completed_successfully
@@ -74,6 +74,22 @@ services:
       db:
         condition: service_healthy
 
+  frontend:
+    image: mtsrus/data-rentgen-ui:develop
+    restart: unless-stopped
+    env_file: .env.docker
+    ports:
+      - 3000:3000
+    healthcheck:
+      test: [CMD-SHELL, curl, -f, http://localhost:3000/]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+    depends_on:
+      server:
+        condition: service_healthy
+
 volumes:
   postgres_data:
   kafka_data: