[#] NextDotID#159 basenames loading jobs

ZhongFuze · Sep 3, 2024 · 28c5bce · 28c5bce
1 parent 4ba03cd
commit 28c5bce
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 0 deletions.
diff --git a/src/config/tdb/migrations/run_loading_jobs.gsql b/src/config/tdb/migrations/run_loading_jobs.gsql
@@ -1,3 +1,36 @@
+CREATE LOADING JOB Load_Basenames FOR GRAPH SocialGraph {
+LOAD /home/tigergraph/shared_data/basenames/ethereum.Identities.csv"
+    TO VERTEX Identities VALUES ($"primary_id", $"id", REDUCE(ignore_if_exists($"uuid")), $"platform", $"identity", _, _, _, REDUCE(min($"created_at")), REDUCE(min($"added_at")), REDUCE(max($"updated_at")), _, _, REDUCE(or($"reverse"))) USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/basenames.Identities.csv"
+    TO VERTEX Identities VALUES ($"primary_id", $"id", REDUCE(ignore_if_exists($"uuid")), $"platform", $"identity", $"display_name", _, _, REDUCE(min($"created_at")), REDUCE(min($"added_at")), REDUCE(max($"updated_at")), _, REDUCE(max($"expired_at")), REDUCE(or($"reverse"))) USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/IdentitiesGraph.csv"
+    TO VERTEX IdentitiesGraph VALUES ($"primary_id", $"id", $"updated_nanosecond") USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/DomainCollection.csv"
+    TO VERTEX DomainCollection VALUES ($"primary_id", $"id", REDUCE(max($"updated_at"))) USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/Hold_Identity.csv"
+    TO EDGE Hold_Identity VALUES ($"from", $"to", $"source", REDUCE(ignore_if_exists($"uuid")), _, $"id", REDUCE(min($"created_at")), REDUCE(max($"updated_at")), $"fetcher", REDUCE(max($"expired_at"))) USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/Hold_Contract.csv"
+    TO EDGE Hold_Contract VALUES ($"from", $"to", $"source", _, $"id", REDUCE(ignore_if_exists($"uuid")), REDUCE(min($"created_at")), REDUCE(max($"updated_at")), $"fetcher", REDUCE(max($"expired_at"))) USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/Resolve.csv"
+    TO EDGE Resolve VALUES ($"from", $"to", $"source", $"system", $"name", REDUCE(ignore_if_exists($"uuid")), REDUCE(max($"updated_at")), $"fetcher") USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/Reverse_Resolve.csv"
+    TO EDGE Reverse_Resolve VALUES ($"from", $"to", $"source", $"system", $"name", REDUCE(ignore_if_exists($"uuid")), REDUCE(max($"updated_at")), $"fetcher") USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/PartOfIdentitiesGraph_Reverse.csv"
+    TO EDGE PartOfIdentitiesGraph VALUES ($"from", $"to") USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+
+LOAD /home/tigergraph/shared_data/basenames/PartOfCollection.csv"
+    TO EDGE PartOfCollection VALUES ($"from", $"to", $"platform", $"name", $"tld", $"status") USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";
+}
+
+
 CREATE LOADING JOB Load_DBExport_SocialGraph FOR GRAPH SocialGraph {
   LOAD "/home/tigergraph/shared_data/export_graphs/GlobalTypes/Identities.csv"
     TO VERTEX Identities VALUES ($"primary_id", $"id", $"uuid", $"platform", $"identity", $"display_name", $"profile_url", $"avatar_url", $"created_at", $"added_at", $"updated_at", $"uid", $"expired_at", $"reverse") USING SEPARATOR = "\t", EOL = "\n", HEADER = "true";

diff --git a/src/tigergraph/vertex/identity_graph.rs b/src/tigergraph/vertex/identity_graph.rs
@@ -347,6 +347,12 @@ impl IdentityGraph {
                                     let duration_since_update =
                                         current_time.signed_duration_since(updated_at);
                                     // Check if the difference is greater than 2 hours
+                                    tracing::trace!(
+                                        "updated_at {}, duration_since_update {}, current_time {}",
+                                        updated_at,
+                                        duration_since_update,
+                                        current_time
+                                    );
                                     if duration_since_update > Duration::hours(2) {
                                         tracing::info!("Basenames refetching...");
                                         return Ok(None);