Add android documents syncing

qb-rs · Sep 1, 2024 · 7172bd2 · 7172bd2
1 parent 17792f0
commit 7172bd2
Show file tree

Hide file tree

Showing 27 changed files with 781 additions and 134 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/drawio/android_arch.drawio b/drawio/android_arch.drawio
@@ -0,0 +1,122 @@
+<mxfile host="Electron" modified="2024-09-01T15:53:19.838Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/24.5.3 Chrome/124.0.6367.207 Electron/30.0.6 Safari/537.36" etag="CAL87vl9NIMcoEYHAsm9" version="24.5.3" type="device">
+  <diagram name="Page-1" id="EDsVvEjms7b2LsurbVCz">
+    <mxGraphModel dx="683" dy="432" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-1" value="Documents Provider (Kotlin)" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="120" y="80" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-2" value="Flutter Engine" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="120" y="200" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-3" value="documents_provider.dart" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="120" y="240" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-4" value="spawns" style="endArrow=classic;html=1;rounded=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="160" y="120" as="sourcePoint" />
+            <mxPoint x="160" y="200" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-8" value="service.dart" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="440" y="240" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-12" value="Application (Flutter)" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="440" y="80" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-14" value="spawns" style="endArrow=classic;html=1;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="480.6" y="120" as="sourcePoint" />
+            <mxPoint x="480" y="200" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-15" value="Background Service" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="440" y="200" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-16" value="qb-daemon" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="440" y="360" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-17" value="frb" style="shape=flexArrow;endArrow=classic;startArrow=classic;html=1;rounded=0;entryX=0.801;entryY=1.04;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1">
+          <mxGeometry width="100" height="100" relative="1" as="geometry">
+            <mxPoint x="600" y="360" as="sourcePoint" />
+            <mxPoint x="600" y="280" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-19" value="spawns" style="endArrow=classic;html=1;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="480.6" y="280" as="sourcePoint" />
+            <mxPoint x="480" y="360" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-20" value="qb-ext-android" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="440" y="400" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-21" value="Files" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="120" y="400" width="200" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-23" value="" style="endArrow=classic;startArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="tGjq85jMf3p_U8cP3Jxx-21">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="390" y="470" as="sourcePoint" />
+            <mxPoint x="440" y="420" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-25" value="frb" style="shape=flexArrow;endArrow=classic;startArrow=classic;html=1;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="100" height="100" relative="1" as="geometry">
+            <mxPoint x="160" y="520" as="sourcePoint" />
+            <mxPoint x="280" y="520" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-27" value="Communicate via flutter rust bridge" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="160" y="550" width="120" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-28" value="Communicate via flutter background service" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="320" y="550" width="120" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-29" value="fbs" style="shape=flexArrow;endArrow=classic;startArrow=classic;html=1;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="100" height="100" relative="1" as="geometry">
+            <mxPoint x="320" y="520" as="sourcePoint" />
+            <mxPoint x="440" y="520" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-30" value="Communicate via MethodChannel" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+          <mxGeometry x="480" y="550" width="120" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-31" value="MC" style="shape=flexArrow;endArrow=classic;startArrow=classic;html=1;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="100" height="100" relative="1" as="geometry">
+            <mxPoint x="480" y="520" as="sourcePoint" />
+            <mxPoint x="600" y="520" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-33" value="fbs" style="shape=flexArrow;endArrow=classic;html=1;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="320" y="259.8" as="sourcePoint" />
+            <mxPoint x="440" y="259.8" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-35" value="fbs" style="shape=flexArrow;endArrow=classic;startArrow=classic;html=1;rounded=0;entryX=0.808;entryY=1.064;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" target="tGjq85jMf3p_U8cP3Jxx-12">
+          <mxGeometry width="100" height="100" relative="1" as="geometry">
+            <mxPoint x="600" y="200" as="sourcePoint" />
+            <mxPoint x="700" y="100" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-36" value="MC" style="shape=flexArrow;endArrow=classic;html=1;rounded=0;" edge="1" parent="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="280.8" y="120" as="sourcePoint" />
+            <mxPoint x="280" y="200" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="tGjq85jMf3p_U8cP3Jxx-37" value="" style="endArrow=classic;startArrow=classic;html=1;rounded=0;exitX=0;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="tGjq85jMf3p_U8cP3Jxx-1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="80" y="100" as="sourcePoint" />
+            <mxPoint x="120" y="419.66" as="targetPoint" />
+            <Array as="points">
+              <mxPoint x="80" y="100" />
+              <mxPoint x="80" y="420" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>
diff --git a/qb-core/src/change.rs b/qb-core/src/change.rs
@@ -133,7 +133,7 @@ impl QBChangeMap {
     }
 
     /// Append another changemap to this map.
-    pub fn append(&mut self, other: Self) {
+    pub fn append_map(&mut self, other: Self) {
         if other.head > self.head {
             self.head = other.head;
         }
@@ -144,6 +144,13 @@ impl QBChangeMap {
         }
     }
 
+    /// Append entries to this map.
+    pub fn append(&mut self, entries: Vec<(QBResource, QBChange)>) {
+        for entry in entries {
+            self.push(entry);
+        }
+    }
+
     /// Returns whether this changemap is empty.
     #[inline(always)]
     pub fn is_empty(&self) -> bool {
@@ -173,7 +180,7 @@ impl QBChangeMap {
     }
 
     /// Push an entry.
-    pub fn push(&mut self, resource: QBResource, change: QBChange) {
+    pub fn push(&mut self, (resource, change): (QBResource, QBChange)) {
         let new_change = self.register(&change);
         let entries = self.entries(resource);
         entries.push(change);

diff --git a/qb-core/src/fs.rs b/qb-core/src/fs.rs
@@ -9,7 +9,7 @@ pub mod wrapper;
 use std::{ffi::OsString, path::Path};
 
 use thiserror::Error;
-use tracing::{debug, warn};
+use tracing::{debug, info, warn};
 
 use table::QBFileTable;
 use tree::{QBFileTree, TreeFile};
@@ -91,6 +91,7 @@ pub enum QBFSChangeKind {
 }
 
 /// struct describing a text or binary diff of a file
+#[derive(Debug)]
 pub enum QBFileDiff {
     /// binary file
     Binary(Vec<u8>),
@@ -277,6 +278,7 @@ impl QBFS {
         let contents = self.wrapper.read(&path).await?;
         let hash = QBHash::compute(&contents);
 
+        info!("TREE: {} - {}", path.as_ref(), self.tree);
         let file = self
             .tree
             .get_or_insert_mut(&path, TreeFile::default().into())

diff --git a/qb-core/src/fs/tree.rs b/qb-core/src/fs/tree.rs
@@ -10,7 +10,7 @@ use std::{
 
 use bitcode::{Decode, Encode};
 use itertools::Itertools;
-use tracing::warn;
+use tracing::{info, warn};
 
 use crate::{
     change::QBChange,
@@ -62,7 +62,7 @@ impl QBFileTreeNode {
         if let QBFileTreeNode::File(val) = self {
             return val;
         }
-        panic!("error while unpacking")
+        panic!("error while unpacking: {:?} is not File", self);
     }
 
     /// unwrap file
@@ -71,7 +71,7 @@ impl QBFileTreeNode {
         if let QBFileTreeNode::File(val) = self {
             return val;
         }
-        panic!("error while unpacking")
+        panic!("error while unpacking: {:?} is not File", self);
     }
 
     /// unwrap mutable dir
@@ -80,7 +80,7 @@ impl QBFileTreeNode {
         if let QBFileTreeNode::Dir(val) = self {
             return val;
         }
-        panic!("error while unpacking")
+        panic!("error while unpacking: {:?} is not Dir", self);
     }
 
     /// unwrap dir
@@ -89,7 +89,7 @@ impl QBFileTreeNode {
         if let QBFileTreeNode::Dir(val) = self {
             return val;
         }
-        panic!("error while unpacking")
+        panic!("error while unpacking: {:?} is not Dir", self);
     }
 }
 
@@ -233,6 +233,13 @@ impl QBFileTree {
         }
     }
 
+    /// Process changes that were applied to the underlying file system
+    pub fn notify_changes<'a>(&mut self, changes: impl Iterator<Item = &'a QBFSChange>) {
+        for change in changes {
+            self.notify_change(change);
+        }
+    }
+
     fn get_tree(&self, path: impl AsRef<QBPath>) -> HashSet<Compare> {
         let idx = match self.index(&path) {
             Some(idx) => idx,

diff --git a/qb-core/src/path.rs b/qb-core/src/path.rs
@@ -202,6 +202,7 @@ impl QBPath {
     /// Convert into string
     #[inline]
     pub fn to_string(&self, root: &str) -> String {
+        assert!(!root.ends_with('/'));
         let path = &self.0;
         format!("{root}{path}")
     }
@@ -269,6 +270,7 @@ pub struct QBResource {
 #[derive(
     Encode, Decode, Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash,
 )]
+#[serde(rename_all = "lowercase")]
 pub enum QBResourceKind {
     /// a file
     File,

diff --git a/qb-daemon/src/master.rs b/qb-daemon/src/master.rs
@@ -227,7 +227,7 @@ impl QBMaster {
                 // Apply changes to changelog
                 let mut changemap = local.clone();
                 _ = changemap.merge(remote).unwrap();
-                self.changemap.append(changemap);
+                self.changemap.append_map(changemap);
 
                 // find the new common hash
                 let new_common = self.changemap.head().clone();

diff --git a/qb-ext-local/src/lib.rs b/qb-ext-local/src/lib.rs
@@ -96,7 +96,7 @@ impl Runner {
                 // Apply changes
                 let mut changemap = local.clone();
                 let changes = changemap.merge(remote).unwrap();
-                self.fs.changemap.append(changemap);
+                self.fs.changemap.append_map(changemap);
                 let fschanges = self.fs.to_fschanges(changes);
                 self.watcher_skip.append(
                     &mut fschanges
@@ -171,37 +171,56 @@ impl Runner {
             return;
         }
 
-        let change = match event.kind {
+        let entries = match event.kind {
             EventKind::Modify(ModifyKind::Data(_)) => {
                 let kind = self.fs.diff(&resource).await.unwrap();
                 match kind {
                     Some(QBFileDiff::Text(diff)) => {
-                        QBChange::new(self.recorder.record(), QBChangeKind::UpdateText(diff))
+                        vec![(
+                            resource,
+                            QBChange::new(self.recorder.record(), QBChangeKind::UpdateText(diff)),
+                        )]
                     }
                     Some(QBFileDiff::Binary(contents)) => {
-                        QBChange::new(self.recorder.record(), QBChangeKind::UpdateBinary(contents))
+                        vec![(
+                            resource,
+                            QBChange::new(
+                                self.recorder.record(),
+                                QBChangeKind::UpdateBinary(contents),
+                            ),
+                        )]
                     }
                     None => return,
                 }
             }
             EventKind::Modify(ModifyKind::Name(RenameMode::To)) => {
                 let ts = self.recorder.record();
                 let previouspath = self.trackers.remove(&event.tracker().unwrap()).unwrap();
-                self.fs.changemap.push(
-                    QBResource::new(previouspath, resource.kind.clone()),
-                    QBChange::new(ts.clone(), QBChangeKind::RenameFrom),
-                );
-                QBChange::new(ts, QBChangeKind::RenameTo)
+                vec![
+                    (
+                        QBResource::new(previouspath, resource.kind.clone()),
+                        QBChange::new(ts.clone(), QBChangeKind::RenameFrom),
+                    ),
+                    (resource, QBChange::new(ts, QBChangeKind::RenameTo)),
+                ]
             }
             EventKind::Remove(..) => {
                 info!("DELETE {}", resource);
-                QBChange::new(self.recorder.record(), QBChangeKind::Delete)
+                vec![(
+                    resource,
+                    QBChange::new(self.recorder.record(), QBChangeKind::Delete),
+                )]
             }
-            EventKind::Create(..) => QBChange::new(self.recorder.record(), QBChangeKind::Create),
+            EventKind::Create(..) => vec![(
+                resource,
+                QBChange::new(self.recorder.record(), QBChangeKind::Create),
+            )],
             _ => panic!("this should not happen"),
         };
 
-        self.fs.changemap.push(resource, change);
+        let fschanges = self.fs.to_fschanges(entries.clone());
+        self.fs.tree.notify_changes(fschanges.iter());
+        self.fs.changemap.append(entries);
     }
 
     fn should_sync(&mut self) -> bool {
@@ -247,7 +266,7 @@ impl Runner {
                             info!("stopping...");
                             break
                         }
-                        _ => unimplemented!(),
+                        _ => unimplemented!("unknown message: {msg:?}"),
                     }
                 },
                 Some(Ok(event)) = watcher_rx.recv() => {

diff --git a/qb-ext-tcp/src/lib.rs b/qb-ext-tcp/src/lib.rs
@@ -57,7 +57,7 @@ impl Runner {
                             info!("stopping...");
                             break;
                         }
-                        _ => unimplemented!(),
+                        _ => unimplemented!("unknown message: {msg:?}"),
                     }
                 }
             }

diff --git a/qb-ext/Cargo.toml b/qb-ext/Cargo.toml
@@ -6,6 +6,7 @@ edition.workspace = true
 [dependencies]
 tokio = { version = "1.39.2", features = ["sync"] }
 serde = { version = "1.0.204", features = ["derive"] }
+serde_bytes = "0.11.15"
 simdutf8 = "0.1.4"
 bitcode = "0.6.3"
 hex = "0.4.3"

diff --git a/qb-ext/src/control.rs b/qb-ext/src/control.rs
@@ -40,6 +40,7 @@ impl QBCId {
         Self(rng.gen::<u64>())
     }
 
+    /// The root control handle.
     pub fn root() -> Self {
         Self(0)
     }
@@ -57,6 +58,8 @@ impl QBCId {
         Ok(Self(u64::from_be_bytes(id_bytes)))
     }
 
+    /// Returns whether this control handle is the root (useful, when the
+    /// daemon is embedded inside an application and can only have one).
     pub fn is_root(&self) -> bool {
         self.0 == 0
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -57,7 +57,7 @@ impl Runner { @@
                                 info!("stopping...");
                                 break;
                             }
-                            _ => unimplemented!(),
+                            _ => unimplemented!("unknown message: {msg:?}"),
                         }
                     }
                 }
@@ Expand Down @@