Skip to content

Commit f80d448

Browse files
gsovereigntygsovereignty
committed
problem: getting replies is slow
1 parent ec11729 commit f80d448

File tree

6 files changed

+138
-125
lines changed

6 files changed

+138
-125
lines changed

‎src/lib/snort_workers/main.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ export function UpdatePubkey(pubkey:string) {
2424

2525
}
2626

27-
export function PushEvent(e: NostrEvent) {
27+
export function PushEvent(e: NostrEvent[]) {
2828
if (worker) {
2929
let cmd = new Command("push_event")
3030
cmd.event = e

‎src/lib/snort_workers/master_worker.ts

Lines changed: 120 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,26 @@ import type { NostrEvent } from '@nostr-dev-kit/ndk';
66
import WorkerPubkeys from './live_subs?worker';
77
import WorkerEvents from './fetch_events?worker';
88
import { seedRelays } from '@/snort_workers/seed_relays';
9+
import type { Nostr } from 'nostr-tools';
910

1011
let workerData = new WorkerData();
1112
let workerDataStore = writable(workerData);
1213

14+
const sys = new NostrSystem({
15+
checkSigs: false
16+
// automaticOutboxModel: true,
17+
// buildFollowGraph: true,
18+
});
19+
20+
let connecting = false
21+
22+
async function connect() {
23+
if (!connecting) {
24+
connecting = true
25+
seedRelays.forEach((r) => sys.ConnectToRelay(r, { read: true, write: false }));
26+
}
27+
}
28+
1329
workerDataStore.subscribe((data) => {
1430
let fed = new FrontendData();
1531
fed.basePubkey = data.ourPubkey();
@@ -64,32 +80,31 @@ onmessage = (m: MessageEvent<Command>) => {
6480
});
6581
}
6682
if (m.data.command == 'push_event') {
67-
68-
workerDataStore.update(current=>{
69-
console.log(m.data.event)
70-
current.events.set(m.data.event!.id, m.data.event!)
71-
return current
72-
})
83+
let map = new Map<string, NostrEvent>()
84+
if (m.data.event) {
85+
for (let e of m.data.event) {
86+
map.set(e.id, e)
87+
}
88+
if (map.size > 0) {
89+
console.log(89, map)
90+
updateReplies(map)
91+
console.log(91)
92+
}
93+
}
7394
}
7495
};
7596

7697
//connect to seed relays, get our follows and relays.
7798
async function start(pubkey?: string, pubkeys?: string[]) {
99+
connect()
78100
return new Promise((resolve, reject) => {
79101
if (pubkey) {
80102
workerData.setOurPubkey(pubkey);
81103
} else {
82104
pubkey = workerData.ourPubkey();
83105
}
84-
const sys = new NostrSystem({
85-
checkSigs: false
86-
// automaticOutboxModel: true,
87-
// buildFollowGraph: true,
88-
});
106+
89107
(async () => {
90-
await sys.Init();
91-
seedRelays.forEach((r) => sys.ConnectToRelay(r, { read: true, write: false }));
92-
93108
const rb = new RequestBuilder('fetch-initial-data');
94109
let _pukeys: string[] = [];
95110
if (pubkeys) {
@@ -155,90 +170,96 @@ async function start(pubkey?: string, pubkeys?: string[]) {
155170

156171
let permaSub: Worker | undefined = undefined;
157172

158-
async function PermaSub(pubkeys: string[]) {
159-
if (pubkeys.length > 0) {
160-
if (permaSub) {
161-
permaSub.terminate();
173+
function updateReplies(newEvents?:Map<string, NostrEvent>) {
174+
workerDataStore.update((current) => {
175+
if (newEvents) {
176+
current.events = new Map([...newEvents, ...current.events]);
162177
}
163-
permaSub = new WorkerPubkeys();
164-
permaSub.onmessage = (x: MessageEvent<Map<string, NostrEvent>>) => {
165-
workerDataStore.update((current) => {
166-
current.events = new Map([...x.data, ...current.events]);
167-
//console.log(current.events.size)
168-
let printed = 0;
169-
let printedID = new Set<string>();
170-
for (let [id, e] of current.events) {
171-
current.missingEvents.delete(id);
172-
let tagsForEvent = new tagSplits(e);
173-
if (tagsForEvent.unknown.size > 0) {
174-
//tell user that there's an unhandled tag
175-
if (printed < 20 && !printedID.has(tagsForEvent.id)) {
176-
printed++;
177-
printedID.add(tagsForEvent.id);
178-
//console.log('unknown tag detected', printed, tagsForEvent.rawEvent);
179-
}
178+
//console.log(current.events.size)
179+
let printed = 0;
180+
let printedID = new Set<string>();
181+
for (let [id, e] of current.events) {
182+
current.missingEvents.delete(id);
183+
let tagsForEvent = new tagSplits(e);
184+
if (tagsForEvent.unknown.size > 0) {
185+
//tell user that there's an unhandled tag
186+
if (printed < 20 && !printedID.has(tagsForEvent.id)) {
187+
printed++;
188+
printedID.add(tagsForEvent.id);
189+
//console.log('unknown tag detected', printed, tagsForEvent.rawEvent);
190+
}
191+
}
192+
tagsForEvent.roots.forEach((r) => {
193+
if (!current.events.has(r)) {
194+
current.missingEvents.add(r);
195+
} else {
196+
current.roots.add(r);
197+
}
198+
});
199+
if (
200+
(tagsForEvent.replies.size != 1 && tagsForEvent.unlabelled.size > 1) ||
201+
tagsForEvent.replies.size > 1
202+
) {
203+
//we don't know which tag is the _real_ reply (parent), let's try and find out
204+
let possibleParents = new Map<string, NostrEvent>();
205+
let possibleReplyTags = new Set([...tagsForEvent.unlabelled, ...tagsForEvent.replies]);
206+
let numMissing = 0;
207+
for (let _id of possibleReplyTags) {
208+
let _event = current.events.get(_id);
209+
if (_event) {
210+
possibleParents.set(_id, _event);
180211
}
181-
tagsForEvent.roots.forEach((r) => {
182-
if (!current.events.has(r)) {
183-
current.missingEvents.add(r);
184-
} else {
185-
current.roots.add(r);
186-
}
187-
});
188-
if (
189-
(tagsForEvent.replies.size != 1 && tagsForEvent.unlabelled.size > 1) ||
190-
tagsForEvent.replies.size > 1
191-
) {
192-
//we don't know which tag is the _real_ reply (parent), let's try and find out
193-
let possibleParents = new Map<string, NostrEvent>();
194-
let possibleReplyTags = new Set([...tagsForEvent.unlabelled, ...tagsForEvent.replies]);
195-
let numMissing = 0;
196-
for (let _id of possibleReplyTags) {
197-
let _event = current.events.get(_id);
198-
if (_event) {
199-
possibleParents.set(_id, _event);
200-
}
201-
if (!_event) {
202-
current.missingEvents.add(_id);
203-
numMissing++;
204-
}
205-
}
206-
if (numMissing == 0 && possibleParents.size > 0) {
207-
let allTaggedEvents = new Set<string>();
208-
for (let [_, e] of possibleParents) {
209-
let splits = new tagSplits(e);
210-
for (let id of splits.All()) {
211-
allTaggedEvents.add(id);
212-
}
213-
}
214-
let tagsThatAreNotInTaggedEvents = new Set<string>();
215-
for (let id of possibleReplyTags) {
216-
if (!allTaggedEvents.has(id)) {
217-
tagsThatAreNotInTaggedEvents.add(id);
218-
}
219-
}
220-
if (tagsThatAreNotInTaggedEvents.size == 1) {
221-
//console.log("found mistagged reply")
222-
tagsForEvent.replies = new Set([tagsThatAreNotInTaggedEvents][0]);
223-
}
224-
//if more than one in replies: find all the tagged events and see which tag among all these events is unique (the unique one is probably the reply, and the repeated one(s) are the root or further up in the thread)
225-
//console.log('implement me');
226-
} else {
227-
//console.log(missing)
228-
//todo: fetch missing events by ID
212+
if (!_event) {
213+
current.missingEvents.add(_id);
214+
numMissing++;
215+
}
216+
}
217+
if (numMissing == 0 && possibleParents.size > 0) {
218+
let allTaggedEvents = new Set<string>();
219+
for (let [_, e] of possibleParents) {
220+
let splits = new tagSplits(e);
221+
for (let id of splits.All()) {
222+
allTaggedEvents.add(id);
229223
}
230224
}
231-
if (tagsForEvent.replies.size == 1) {
232-
let existing = current.replies.get([...tagsForEvent.replies][0]);
233-
if (!existing) {
234-
existing = new Set();
225+
let tagsThatAreNotInTaggedEvents = new Set<string>();
226+
for (let id of possibleReplyTags) {
227+
if (!allTaggedEvents.has(id)) {
228+
tagsThatAreNotInTaggedEvents.add(id);
235229
}
236-
existing.add(tagsForEvent.id);
237-
current.replies.set([...tagsForEvent.replies][0], existing);
238230
}
231+
if (tagsThatAreNotInTaggedEvents.size == 1) {
232+
//console.log("found mistagged reply")
233+
tagsForEvent.replies = new Set([tagsThatAreNotInTaggedEvents][0]);
234+
}
235+
//if more than one in replies: find all the tagged events and see which tag among all these events is unique (the unique one is probably the reply, and the repeated one(s) are the root or further up in the thread)
236+
//console.log('implement me');
237+
} else {
238+
//console.log(missing)
239+
//todo: fetch missing events by ID
239240
}
240-
return current;
241-
});
241+
}
242+
if (tagsForEvent.replies.size == 1) {
243+
let existing = current.replies.get([...tagsForEvent.replies][0]);
244+
if (!existing) {
245+
existing = new Set();
246+
}
247+
existing.add(tagsForEvent.id);
248+
current.replies.set([...tagsForEvent.replies][0], existing);
249+
}
250+
}
251+
return current;
252+
});
253+
}
254+
255+
async function PermaSub(pubkeys: string[]) {
256+
if (pubkeys.length > 0) {
257+
if (permaSub) {
258+
permaSub.terminate();
259+
}
260+
permaSub = new WorkerPubkeys();
261+
permaSub.onmessage = (x: MessageEvent<Map<string, NostrEvent>>) => {
262+
updateReplies(x.data)
242263
};
243264
let cmd = new Command('sub_to_pubkeys');
244265
cmd.pubkeys = pubkeys;
@@ -252,37 +273,26 @@ let numberOfMissingEvents = derived(workerDataStore, ($wds) => {
252273

253274
//let fetchEventsWorker: Worker | undefined = undefined;
254275

255-
const missingEventSys = new NostrSystem({
256-
checkSigs: false
257-
// automaticOutboxModel: true,
258-
// buildFollowGraph: true,
259-
});
260276

261277

262278
let q: QueryLike
263279

264280

265-
266-
let fmeStarted = false;
267-
function fmeStart() {
268-
if (!fmeStarted) {
269-
fmeStarted = true;
270-
seedRelays.forEach((r) => missingEventSys.ConnectToRelay(r, { read: true, write: false }));
271-
}
272-
}
273-
274281
numberOfMissingEvents.subscribe((n) => {
275282
if (n > 0) {
276-
fmeStart()
277283
const rb = new RequestBuilder('fetch-missing-events');
278284
rb.withFilter().ids([...workerData.missingEvents])
279285
rb.withOptions({ leaveOpen: false });
280286
if (q) {q.cancel()}
281-
q = missingEventSys.Query(rb);
287+
q = sys.Query(rb);
282288
q.on('event', (evs): void => {
283-
evs.forEach(e=>{
284-
workerData.events.set(e.id, getNostrEvent(e))
285-
})
289+
let m = new Map<string, NostrEvent>()
290+
for (let e of evs) {
291+
m.set(e.id, e)
292+
}
293+
if (m.size > 0) {
294+
updateReplies(m)
295+
}
286296
})
287297

288298
// console.log(248, n)

‎src/lib/snort_workers/types.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ export class Command {
55
pubkey?: string;
66
pubkeys?: string[];
77
events?: string[];
8-
event?: NostrEvent;
8+
event?: NostrEvent[];
99
constructor(command: 'start' | 'sub_to_pubkeys' | 'fetch_events' | 'push_event') {
1010
this.command = command;
1111
}

‎src/lib/snort_workers/utils.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,5 +66,8 @@ export class tagSplits {
6666
if (this.replies.size == 0 && this.unlabelled.size == 1) {
6767
this.replies.add([...this.unlabelled][0]);
6868
}
69+
if (this.replies.size == 0 && this.roots.size == 1) {
70+
this.replies.add([...this.roots][0]);
71+
}
6972
}
7073
}

‎src/lib/views/messages/Messages.svelte

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838
let ev = $fds.rawEvents.get(id);
3939
if (ev) {
4040
fullSet.set(id, ev);
41+
} else {
42+
throw new Error("this should not happen")
4143
}
4244
}
4345
}
@@ -150,6 +152,8 @@
150152
<h3>HUMBLE HORSE</h3>
151153
<h6>Release Name: "Giddy Up"</h6>
152154
Events in memory: {$FrontendDataStore.rawEvents.size}<br />
155+
<Button onClick={()=>{console.log($FrontendDataStore.replies.get($threadParentID))}}>Print root event data</Button>
156+
153157
</div>
154158
</div>
155159
</ChatLayout>

‎src/lib/views/messages/RenderKind1AsThreadHead.svelte

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212
export let note: NostrEvent;
1313
export let store: Writable<FrontendData>;
1414
15-
16-
1715
let top: HTMLDivElement;
1816
let q: QueryLike;
1917
@@ -22,17 +20,15 @@
2220
// ID should be unique to the use case, this is important as all data fetched from this ID will be merged into the same NoteStore
2321
const rb = new RequestBuilder(`get-${note.id}`);
2422
rb.withFilter().tag('e', [note.id]).kinds([1]);
25-
rb.withOptions({leaveOpen: false})
26-
console.log(26)
27-
console.log(rb)
28-
console.log(28)
23+
rb.withOptions({ leaveOpen: false });
2924
q = System.Query(rb);
3025
// basic usage using "onEvent", fired every 100ms
3126
q.on('event', (evs) => {
32-
console.log(35, evs);
33-
evs.forEach(e=>{
34-
PushEvent(e)
35-
})
27+
if (evs.length > 0) {
28+
console.log(35, evs);
29+
PushEvent(evs);
30+
}
31+
3632
// something else..
3733
});
3834
})();
@@ -43,11 +39,11 @@
4339
})();
4440
});
4541
46-
onDestroy(()=>{
42+
onDestroy(() => {
4743
if (q) {
48-
q.cancel()
44+
q.cancel();
4945
}
50-
})
46+
});
5147
5248
$: childrenCount = $store?.replies.get(note.id) ? $store.replies.get(note.id)!.size : 0;
5349
</script>

0 commit comments

Comments
 (0)