Compression optimization (#200)

# Goal The goal of this PR is to optimize the number of initial pages created
ProjectLibertyLabs · May 2, 2024 · 0afd737 · 0afd737
1 parent 0d03e91
commit 0afd737
Show file tree

Hide file tree

Showing 10 changed files with 33 additions and 27 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bridge/common/Cargo.toml b/bridge/common/Cargo.toml
@@ -1,12 +1,12 @@
 [package]
 name = "dsnp-graph-sdk-common"
-version = "1.1.1"
+version = "1.1.2"
 edition = "2021"
 license = "Apache-2.0"
 publish = false
 
 [dependencies]
-dsnp-graph-core = { version = "1.1.1", path = "../../core" }
-dsnp-graph-config = { version = "1.1.1", path = "../../config" }
+dsnp-graph-core = { version = "1.1.2", path = "../../core" }
+dsnp-graph-config = { version = "1.1.2", path = "../../config" }
 libc = "0.2.153"
 protobuf = { version = "3.4.0", features = ["with-bytes"] }
diff --git a/bridge/ffi/Cargo.toml b/bridge/ffi/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "dsnp-graph-sdk-ffi"
-version = "1.1.1"
+version = "1.1.2"
 edition = "2021"
 license = "Apache-2.0"
 publish = false
@@ -10,8 +10,8 @@ name = "dsnp_graph_sdk_ffi"
 crate-type = ["staticlib"]
 
 [dependencies]
-dsnp-graph-core = { version = "1.1.1", path = "../../core" }
-dsnp-graph-config = { version = "1.1.1", path = "../../config" }
+dsnp-graph-core = { version = "1.1.2", path = "../../core" }
+dsnp-graph-config = { version = "1.1.2", path = "../../config" }
 libc = "0.2.153"
 lazy_static = "1.4.0"
 anyhow = "1.0.80"
diff --git a/bridge/jni/Cargo.toml b/bridge/jni/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "dsnp-graph-sdk-jni"
-version = "1.1.1"
+version = "1.1.2"
 edition = "2021"
 license = "Apache-2.0"
 publish = false
@@ -10,9 +10,9 @@ name = "dsnp_graph_sdk_jni"
 crate-type = ["cdylib"]
 
 [dependencies]
-dsnp-graph-core = { version = "1.1.1", path = "../../core" }
-dsnp-graph-config = { version = "1.1.1", path = "../../config" }
-dsnp-graph-sdk-common = { version = "1.1.1", path = "../common" }
+dsnp-graph-core = { version = "1.1.2", path = "../../core" }
+dsnp-graph-config = { version = "1.1.2", path = "../../config" }
+dsnp-graph-sdk-common = { version = "1.1.2", path = "../common" }
 jni = "0.21.1"
 cfg-if = "1.0.0"
 log = { version = "^0.4.21", features = ["std", "max_level_debug", "release_max_level_debug"] }

diff --git a/bridge/node/Cargo.toml b/bridge/node/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "dsnp-graph-sdk-node"
-version = "1.1.1"
+version = "1.1.2"
 edition = "2021"
 license = "Apache-2.0"
 publish = false
@@ -11,8 +11,8 @@ name = "dsnp_graph_sdk_node"
 crate-type = ["cdylib"]
 
 [dependencies]
-dsnp-graph-core = { version = "1.1.1", path = "../../core" }
-dsnp-graph-config = { version = "1.1.1", path = "../../config" }
+dsnp-graph-core = { version = "1.1.2", path = "../../core" }
+dsnp-graph-config = { version = "1.1.2", path = "../../config" }
 neon = { version = "1.0.0", default-features = false, features = ["napi-6"] }
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.114"

diff --git a/bridge/node/package-lock.json b/bridge/node/package-lock.json
diff --git a/bridge/node/package.json b/bridge/node/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@dsnp/graph-sdk",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "author": "Amplica Labs",
   "license": "Apache-2.0",
   "description": "dsnp-graph-sdk-node",
@@ -55,6 +55,6 @@
   },
   "homepage": "https://github.com/LibertyDSNP/graph-sdk/bridge/node/README.md",
   "customProperties": {
-    "uploadedBinariesVersion": "v1.1.1"
+    "uploadedBinariesVersion": "v1.1.2"
   }
 }
diff --git a/config/Cargo.toml b/config/Cargo.toml
@@ -5,7 +5,7 @@ homepage = "https://spec.dsnp.org"
 license = "Apache-2.0"
 publish = false
 repository = "https://github.com/libertyDSNP/graph-sdk/"
-version = "1.1.1"
+version = "1.1.2"
 
 [lib]
 name = "dsnp_graph_config"

diff --git a/core/Cargo.toml b/core/Cargo.toml
@@ -5,7 +5,7 @@ homepage = "https://spec.dsnp.org"
 license = "Apache-2.0"
 publish = false
 repository = "https://github.com/libertyDSNP/graph-sdk/"
-version = "1.1.1"
+version = "1.1.2"
 
 [lib]
 name = "dsnp_graph_core"
@@ -16,7 +16,7 @@ doctest = false
 anyhow = "1.0.69"
 apache-avro = { version = "0.16.0", features = ["snappy"] }
 dryoc = "0.5.3"
-dsnp-graph-config = { version = "1.1.1", path = "../config" }
+dsnp-graph-config = { version = "1.1.2", path = "../config" }
 lazy_static = "1.4.0"
 log = { version = "^0.4.21", features = ["std", "max_level_debug", "release_max_level_debug"] }
 log-result-proc-macro = { path = "../log-result-proc-macro" }

diff --git a/core/src/graph/graph.rs b/core/src/graph/graph.rs
@@ -256,13 +256,15 @@ impl Graph {
 			})
 			.collect();
 
-		let ids_to_add: Vec<DsnpUserId> = updates
+		let mut ids_to_add: Vec<DsnpUserId> = updates
 			.iter()
 			.filter_map(|event| match event {
 				UpdateEvent::Add { dsnp_user_id, .. } => Some(*dsnp_user_id),
 				_ => None,
 			})
 			.collect();
+		// helps with the compression
+		ids_to_add.sort();
 
 		// First calculate pages that have had connections removed. Later, we will
 		// prefer to use these pages first to add new connections, so as to minimize
@@ -665,6 +667,10 @@ impl Graph {
 			.iter()
 			.filter(|c| !ids_to_add.contains(&c.user_id))
 		{
+			// This timestamp condition is eliminating PRID checks for 2 categories of connections
+			// 1. Connections that are just added but not included in `ids_to_add` list
+			// 2. Connections that are added less than `max_allowed_stale_days` since we need some time for the other
+			// side of the connection to act on it
 			if duration_days_since(c.since) > max_allowed_stale_days &&
 				!self
 					.user_key_manager