next version

.gitignore

@@ -0,0 +1,3 @@
+
+.vscode
+.DS_STORE

README.md

@@ -78,13 +78,8 @@ Takes BERTopic output and saves it to a sparql endpoint
     }}
 ```
 
-
-
 ### transform
 
-
-
-
 ## poc-ai-bertopic
 
 initial loading of the model. Using Roberta as the embedding model.
@@ -164,8 +159,6 @@ cli scripts for Named Entity Recognition
     }}
 ```
 
-
-
 ## poc-ai-airflow-zeroshot
 
 ### scripts/save.py

dag-tasks/bertopic_retrain.dot

@@ -0,0 +1,8 @@
+digraph {
+    rankdir=LR
+
+    load_bertopic -> retrain_and_save
+    retrain_and_save -> restart
+    restart -> transform
+    transform -> {save_transform, save_topics}
+}

dag-tasks/bertopic_transform.dot

@@ -0,0 +1,6 @@
+digraph {
+    rankdir=LR
+
+    load_bertopic -> transform
+    transform -> save_bertopic
+}

dag-tasks/dags.dot

@@ -0,0 +1,69 @@
+digraph G {
+rankdir=LR
+beautify = true
+
+    start [shape = "diamond"]
+    start -> {
+        load_zeroshot 
+        load_taxo 
+        load_bertopic_rs 
+        load_bertopic_ts 
+        load_ner 
+        load_embed
+    }
+
+    subgraph cluster_bertopic_rs {
+        label = "bertopic_retrain_and_save"
+        cluster = true
+
+        load_bertopic_rs -> retrain_and_save
+        retrain_and_save -> restart
+        restart -> transform_rs
+        transform_rs -> { save_transform save_topics }
+    }
+
+    subgraph cluster_bertopic_ts {
+        label = "bertopic_transform"
+        cluster = true
+
+        load_bertopic_ts -> transform_ts
+        transform_ts -> save_bertopic
+    }
+
+    subgraph cluster_embed {
+        label = "embed"
+        cluster = true
+
+        load_embed -> embed
+        embed -> save_embed
+    }
+
+
+    subgraph cluster_zeroshot {
+        label="zeroshot";
+        cluster=true;
+
+        {load_zeroshot load_taxo} -> zs_bbc
+        zs_bbc -> save_zeroshot
+    }
+
+    subgraph cluster_ner {
+        label = "ner";
+        cluster = true;
+
+        load_ner -> ner
+        ner -> save_ner
+    }
+
+    end [shape="diamond"]   
+    { 
+        save_ner 
+        save_zeroshot
+        save_embed 
+        save_bertopic 
+        save_transform 
+        save_topics
+    } -> end
+
+
+}