Skip to content

Commit

Permalink
Improve fetch metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
miladsoft committed Sep 16, 2024
1 parent 635aed4 commit 1b7bcb5
Show file tree
Hide file tree
Showing 7 changed files with 257 additions and 207 deletions.
1 change: 1 addition & 0 deletions src/app/components/explore/explore.component.html
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ <h2 class="text-xl font-semibold">Explore Projects</h2>
<img
class="object-cover"
[src]="project?.banner || 'images/pages/profile/cover.jpg'"
onerror="this.onerror=null; this.src='/images/pages/profile/cover.jpg';"
alt="Card cover image"
/>
</div>
Expand Down
16 changes: 6 additions & 10 deletions src/app/components/explore/explore.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ export class ExploreComponent implements OnInit, OnDestroy {
this.projects.forEach(project => this.subscribeToProjectMetadata(project));
}


this._indexedDBService.getMetadataStream()
.pipe(takeUntil(this._unsubscribeAll))
.subscribe((updatedMetadata) => {
Expand All @@ -96,15 +95,11 @@ export class ExploreComponent implements OnInit, OnDestroy {
this.projects = [...this.projects, ...projects];
this.filteredProjects = [...this.projects];


for (let i = 0; i < projects.length; i += this.metadataLoadLimit) {
const batch = projects.slice(i, i + this.metadataLoadLimit);
await Promise.all(batch.map(project => this.loadMetadataForProject(project)));
}
const pubkeys = projects.map(project => project.nostrPubKey);
await this.metadataService.fetchMetadataForMultipleKeys(pubkeys);

this.stateService.setProjects(this.projects);


this.projects.forEach(project => this.subscribeToProjectMetadata(project));
}
this.loading = false;
Expand All @@ -117,6 +112,7 @@ export class ExploreComponent implements OnInit, OnDestroy {
});
}


async loadMetadataForProject(project: Project): Promise<void> {
try {
const metadata = await this.metadataService.fetchMetadataWithCache(project.nostrPubKey);
Expand All @@ -136,13 +132,13 @@ export class ExploreComponent implements OnInit, OnDestroy {
displayName: metadata.name,
about: metadata.about,
picture: metadata.picture,
banner:metadata.banner
banner: metadata.banner
};

const index = this.projects.findIndex(p => p.projectIdentifier === project.projectIdentifier);
if (index !== -1) {
this.projects[index] = updatedProject;
this.projects = [...this.projects];
this.projects = [...this.projects];
}

this.filteredProjects = [...this.projects];
Expand All @@ -154,7 +150,7 @@ export class ExploreComponent implements OnInit, OnDestroy {
.pipe(takeUntil(this._unsubscribeAll))
.subscribe((updatedMetadata: any) => {
if (updatedMetadata && updatedMetadata.pubkey === project.nostrPubKey) {
this.updateProjectMetadata(project, updatedMetadata);
this.updateProjectMetadata(project, updatedMetadata.metadata);
}
});
}
Expand Down
4 changes: 2 additions & 2 deletions src/app/components/profile/profile.component.html
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
<div>
<img
class="h-40 object-cover lg:h-80"
src="images/pages/profile/cover.jpg"
alt="Cover image"
[src]="metadata?.banner" onerror="this.onerror=null; this.src='/images/pages/profile/cover.jpg';" alt="{{metadata?.display_name || metadata?.name || 'Banner'}}"

/>
</div>

Expand Down
3 changes: 0 additions & 3 deletions src/app/services/indexed-db.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ export class IndexedDBService {
async saveUserMetadata(pubkey: string, metadata: any): Promise<void> {
try {
await localForage.setItem(pubkey, metadata);
console.log('Metadata saved successfully!');
this.metadataSubject.next({ pubkey, metadata });
} catch (error) {
console.error('Error saving metadata to IndexedDB:', error);
Expand All @@ -46,7 +45,6 @@ export class IndexedDBService {
async removeUserMetadata(pubkey: string): Promise<void> {
try {
await localForage.removeItem(pubkey);
console.log(`Metadata for pubkey ${pubkey} removed successfully!`);
this.metadataSubject.next({ pubkey, metadata: null });
} catch (error) {
console.error('Error removing metadata from IndexedDB:', error);
Expand All @@ -56,7 +54,6 @@ export class IndexedDBService {
async clearAllMetadata(): Promise<void> {
try {
await localForage.clear();
console.log('All metadata cleared successfully!');
this.metadataSubject.next(null);
} catch (error) {
console.error('Error clearing all metadata:', error);
Expand Down
57 changes: 44 additions & 13 deletions src/app/services/metadata.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export class MetadataService {


getMetadataStream(): Observable<any> {
return this.metadataSubject.asObservable();
return this.metadataSubject.asObservable().pipe(throttleTime(2000));
}


Expand All @@ -31,6 +31,44 @@ export class MetadataService {
this.processQueue();
}

async fetchMetadataForMultipleKeys(pubkeys: string[]): Promise<void> {
const filter: Filter = {
kinds: [0],
authors: pubkeys,
};

try {
await this.relayService.ensureConnectedRelays();
const connectedRelays = this.relayService.getConnectedRelays();

if (connectedRelays.length === 0) {
console.error('No relays are connected.');
return;
}

const sub = this.relayService.getPool().subscribeMany(connectedRelays, [filter], {
onevent: async (event: NostrEvent) => {
if (event.kind === 0) {
try {
const metadata = JSON.parse(event.content);
await this.indexedDBService.saveUserMetadata(event.pubkey, metadata);
this.metadataSubject.next({ pubkey: event.pubkey, metadata });
} catch (error) {
console.error('Error parsing metadata:', error);
}
}
},
oneose: () => {
}
});

setTimeout(() => {
sub.close();
}, 10 * 60 * 1000);
} catch (error) {
console.error('Failed to fetch metadata for multiple keys:', error);
}
}

private async processQueue(): Promise<void> {
if (this.isProcessingQueue || this.requestQueue.size === 0) {
Expand All @@ -43,33 +81,28 @@ export class MetadataService {
const batch = Array.from(this.requestQueue).slice(0, this.maxRequestsPerBatch);
this.requestQueue = new Set(Array.from(this.requestQueue).slice(this.maxRequestsPerBatch));


await Promise.all(batch.map(async (pubkey) => {
try {
const updatedMetadata = await this.fetchMetadataRealtime(pubkey);
if (updatedMetadata) {
await this.indexedDBService.saveUserMetadata(pubkey, updatedMetadata);
this.metadataSubject.next(updatedMetadata);
console.log(`Metadata updated for user: ${pubkey}`);
}
}
} catch (error) {
console.error(`Failed to update metadata for user: ${pubkey}`, error);
}
}));


await new Promise(resolve => setTimeout(resolve, this.requestDelay));
}

this.isProcessingQueue = false;
}


async fetchMetadataWithCache(pubkey: string): Promise<any> {
const metadata = await this.indexedDBService.getUserMetadata(pubkey);
if (metadata) {
this.metadataSubject.next(metadata);
console.log('Metadata loaded from IndexedDB');
} else {
this.enqueueRequest(pubkey);
}
Expand All @@ -78,7 +111,6 @@ export class MetadataService {
return metadata;
}


private subscribeToMetadataUpdates(pubkey: string): void {
this.relayService.ensureConnectedRelays().then(() => {
const filter: Filter = { authors: [pubkey], kinds: [0] };
Expand All @@ -90,20 +122,19 @@ export class MetadataService {
const updatedMetadata = JSON.parse(event.content);
await this.indexedDBService.saveUserMetadata(pubkey, updatedMetadata);
this.metadataSubject.next(updatedMetadata);
console.log('Real-time metadata update saved to IndexedDB');
} catch (error) {
} catch (error) {
console.error('Error parsing updated metadata:', error);
}
}
},
oneose() {
console.log('Real-time metadata subscription closed.');
},
oneose(){},
});
});
}




async fetchMetadataRealtime(pubkey: string): Promise<any> {
await this.relayService.ensureConnectedRelays();
const connectedRelays = this.relayService.getConnectedRelays();
Expand Down
12 changes: 1 addition & 11 deletions src/app/services/projects.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export interface ProjectStats {
})
export class ProjectsService {
private offset = 0;
private limit = 9;
private limit = 45;
private totalProjects = 0;
private loading = false;
private projects: Project[] = [];
Expand Down Expand Up @@ -56,8 +56,6 @@ export class ProjectsService {
? `${indexerUrl}api/query/Angor/projects?offset=${this.offset}&limit=${this.limit}`
: `${indexerUrl}api/query/Angor/projects?limit=${this.limit}`;

console.log(`Fetching projects from URL: ${url}`);

try {
const response = await this.http
.get<Project[]>(url, { observe: 'response' })
Expand All @@ -66,14 +64,12 @@ export class ProjectsService {
if (!this.totalProjectsFetched && response && response.headers) {
const paginationTotal = response.headers.get('pagination-total');
this.totalProjects = paginationTotal ? +paginationTotal : 0;
console.log(`Total projects: ${this.totalProjects}`);
this.totalProjectsFetched = true;

this.offset = Math.max(this.totalProjects - this.limit, 0);
}

const newProjects = response?.body || [];
console.log('New projects received:', newProjects);

if (!newProjects || newProjects.length === 0) {
this.noMoreProjects = true;
Expand All @@ -89,8 +85,6 @@ export class ProjectsService {

if (uniqueNewProjects.length > 0) {
this.projects = [...this.projects, ...uniqueNewProjects];
console.log(`${uniqueNewProjects.length} new projects added`);

this.offset = Math.max(this.offset - this.limit, 0);
return uniqueNewProjects;
} else {
Expand All @@ -110,8 +104,6 @@ export class ProjectsService {
fetchProjectStats(projectIdentifier: string): Observable<ProjectStats> {
const indexerUrl = this.indexerService.getPrimaryIndexer(this.selectedNetwork);
const url = `${indexerUrl}api/query/Angor/projects/${projectIdentifier}/stats`;
console.log(`Fetching project stats from URL: ${url}`);

return this.http.get<ProjectStats>(url).pipe(
catchError((error) => {
console.error(
Expand All @@ -126,8 +118,6 @@ export class ProjectsService {
fetchProjectDetails(projectIdentifier: string): Observable<Project> {
const indexerUrl = this.indexerService.getPrimaryIndexer(this.selectedNetwork);
const url = `${indexerUrl}api/query/Angor/projects/${projectIdentifier}`;
console.log(`Fetching project details from URL: ${url}`);

return this.http.get<Project>(url).pipe(
catchError((error) => {
console.error(
Expand Down
Loading

0 comments on commit 1b7bcb5

Please sign in to comment.