feat: implement replacement of encrypted room metadata & fix database… (

#37)

.dockerignore

@@ -5,14 +5,21 @@ node_modules
 .coverage
 seeds
 test
-!test/integration
 .eslint*
 .nvmrc
 .nyc_output
 docker-compose.yml
+docker-compose.local.yml
+mongo-rs-init.sh
+mongo-rs-init.local.sh
 Dockerfile
 *.env
 *.md
 .nostr
 postgresql.conf
-test/integration/docker-compose.yml
+test/integration/docker-compose.yml
+deployment
+docs
+.vscode
+.git-hooks
+commitlint.config.js

README.md

@@ -80,7 +80,7 @@ Todo
 Clone repository and enter directory:
 
 ```sh
-git clone git@github.com:Guakamoli/denostr.git && cd denostr
+git clone https://github.com/Guakamoli/denostr.git --depth 1 && cd denostr
 ```
 
 Create `.env` file inside denostr project folder

docker-compose.local.yml

@@ -0,0 +1,82 @@
+services:
+  denostr:
+    build: .
+    container_name: denostr
+    environment:
+      RELAY_PORT: 8008
+      NOSTR_CONFIG_DIR: /app/.nostr
+      MONGO_URI: "mongodb://denostr-db:27017/denostr?replicaSet=rs0"
+      MONGO_MIN_POOL_SIZE: 4
+      MONGO_MAX_POOL_SIZE: 16
+      DEBUG: "*"
+    user: deno:deno
+    volumes:
+      - ${PWD}/.nostr:/app/.nostr
+    ports:
+      - 8008:8008
+    depends_on:
+      denostr-db:
+        condition: service_healthy
+      denostr-db-rs-init:
+        condition: service_completed_successfully
+    restart: on-failure
+    networks:
+      default:
+  denostr-db:
+    image: mongo:5.0
+    container_name: denostr-db
+    networks:
+      default:
+    restart: always
+    ports:
+      - '27017:27017'
+    volumes:
+    - db:/data/db
+    entrypoint: ["mongod", "--bind_ip_all", "--replSet", "rs0"]
+    healthcheck:
+      test: echo 'db.runCommand("ping").ok' | mongo --host 127.0.0.1 --port 27017 --quiet 1
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+  denostr-db-rs-init:
+    image: mongo:5.0
+    container_name: denostr-db-rs-init
+    environment:
+      MONGO_URI: mongodb://denostr-db:27017/
+    command: bash /data/mongo-rs-init.sh
+    volumes:
+    - ${PWD}/mongo-rs-init.local.sh:/data/mongo-rs-init.sh
+    networks:
+      default:
+    depends_on:
+      denostr-db:
+        condition: service_healthy
+  denostr-db-express:
+    image: mongo-express:latest
+    container_name: denostr-db-express
+    environment:
+      ME_CONFIG_OPTIONS_EDITORTHEME: ambiance
+      ME_CONFIG_MONGODB_SERVER: denostr-db
+      ME_CONFIG_MONGODB_PORT: 27017
+    restart: always
+    ports:
+      - '8081:8081'
+    networks:
+      default:
+    depends_on:
+      denostr-db:
+        condition: service_healthy
+      denostr-db-rs-init:
+        condition: service_completed_successfully
+
+networks:
+  default:
+    name: denostr
+    ipam:
+      driver: default
+      config:
+        - subnet: 10.10.10.0/24
+
+volumes:
+  db:

docker-compose.yml

@@ -0,0 +1,89 @@
+services:
+  denostr:
+    image: ghcr.io/guakamoli/denostr:v0.0.8-worker
+    container_name: denostr
+    environment:
+      RELAY_PORT: 8008
+      NOSTR_CONFIG_DIR: /app/.nostr
+      WORKER_TYPE: worker
+      MONGO_URI: "mongodb://denostr-db0:27017,denostr-db1:27017/denostr?authSource=admin&replicaSet=rs0"
+      MONGO_READ_REPLICA_ENABLED: true
+      MONGO_MIN_POOL_SIZE: 16
+      MONGO_MAX_POOL_SIZE: 32
+      DEBUG: "*"
+    user: deno:deno
+    volumes:
+      - ${PWD}/.nostr/settings.yaml:/app/.nostr/settings.yaml
+    ports:
+      - 8008:8008
+    depends_on:
+      denostr-db0:
+        condition: service_healthy
+      denostr-db1:
+        condition: service_healthy
+      denostr-db-rs-init:
+        condition: service_completed_successfully
+    restart: on-failure
+    networks:
+      default:
+  denostr-db0:
+    image: mongo:5.0
+    container_name: denostr-db0
+    networks:
+      default:
+    restart: always
+    ports:
+      - '27017:27017'
+    volumes:
+    - db0:/data/db
+    entrypoint: ["mongod", "--bind_ip_all", "--replSet", "rs0"]
+    healthcheck:
+      test: echo 'db.runCommand("ping").ok' | mongo --host 127.0.0.1 --port 27017 --quiet 1
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+  denostr-db1:
+    image: mongo:5.0
+    container_name: denostr-db1
+    networks:
+      default:
+    restart: always
+    ports:
+      - '27018:27017'
+    volumes:
+    - db1:/data/db
+    entrypoint: ["mongod", "--bind_ip_all", "--replSet", "rs0"]
+    healthcheck:
+      test: echo 'db.runCommand("ping").ok' | mongo --host 127.0.0.1 --port 27017 --quiet 1
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+  denostr-db-rs-init:
+    image: mongo:5.0
+    container_name: denostr-db-rs-init
+    environment:
+      MONGO_URI: mongodb://denostr-db0:27017,denostr-db1:27017
+    command: bash /data/mongo-rs-init.sh
+    volumes:
+    - ${PWD}/mongo-rs-init.sh:/data/mongo-rs-init.sh
+    networks:
+      default:
+    depends_on:
+      denostr-db0:
+        condition: service_healthy
+      denostr-db1:
+        condition: service_healthy
+
+networks:
+  default:
+    name: denostr
+    ipam:
+      driver: default
+      config:
+        - subnet: 10.10.10.0/24
+
+volumes:
+  db0:
+  db1:

mongo-rs-init.local.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+mongo --quiet $MONGO_URI <<EOF
+var config = {
+    "_id": "rs0",
+    "version": 1,
+    "members": [
+        {
+            "_id": 1,
+            "host": "denostr-db:27017",
+            "priority": 1
+        },
+    ]
+};
+rs.initiate(config, { force: true });
+rs.status();
+EOF

mongo-rs-init.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+mongo --quiet $MONGO_URI <<EOF
+var config = {
+    "_id": "rs0",
+    "version": 1,
+    "members": [
+        {
+            "_id": 1,
+            "host": "denostr-db0:27017",
+            "priority": 10
+        },
+        {
+            "_id": 2,
+            "host": "denostr-db1:27017",
+            "priority": 1
+        }
+    ]
+};
+rs.initiate(config, { force: true });
+rs.status();
+EOF

src/constants/base.ts

@@ -14,6 +14,11 @@ export enum EventKinds {
     CHANNEL_MUTE_USER = 44,
     CHANNEL_RESERVED_FIRST = 45,
     CHANNEL_RESERVED_LAST = 49,
+    // Encrypted Chanels
+    ENCRYPTED_CHANNEL_MEGOLM_SESSION = 104,
+    ENCRYPTED_CHANNEL_CREATION = 140,
+    ENCRYPTED_CHANNEL_METADATA = 141,
+    ENCRYPTED_CHANNEL_MESSAGE = 142,
     // Relay-only
     RELAY_INVITE = 50,
     INVOICE_UPDATE = 402,

src/database/models/Events.ts

@@ -2,6 +2,7 @@ import mongoose from 'npm:mongoose'
 
 import { Tag } from '../../@types/base.ts'
 import { getMasterDbClient, getReadReplicaDbClient } from '../client.ts'
+import { Buffer } from 'Buffer'
 
 export interface EventInput {
     event_id: Buffer
@@ -24,32 +25,37 @@ export interface EventDocument extends EventInput, mongoose.Document {
 
 const EventSchema = new mongoose.Schema({
     event_id: {
-        type: mongoose.Schema.Types.Buffer,
+        type: Buffer,
         require: true,
     },
     event_pubkey: {
-        type: mongoose.Schema.Types.Buffer,
+        type: Buffer,
+        require: true,
     },
     event_kind: {
         type: Number,
+        require: true,
     },
     event_created_at: {
         type: Number,
-        default: Date.now(),
+        require: true,
     },
     event_content: {
         type: String,
+        require: true,
     },
     event_tags: {
         type: [[String]],
+        require: true,
     },
     event_signature: {
-        type: mongoose.Schema.Types.Buffer,
+        type: Buffer,
+        require: true,
     },
     event_delegator: {
-        type: mongoose.Schema.Types.Buffer,
+        type: Buffer,
     },
-    event_deduplication: [String],
+    event_deduplication: [mongoose.Schema.Types.Mixed],
     first_seen: { type: Date },
     deleted_at: { type: Date },
 })
@@ -67,6 +73,15 @@ EventSchema.index({ 'event_kind': 1 }, {
 EventSchema.index({ 'event_signature': 1 }, {
     background: true,
 })
+EventSchema.index({ 'event_tags.0.0': 1 }, {
+    background: true,
+})
+EventSchema.index({ 'event_tags.0.1': 1 }, {
+    background: true,
+})
+EventSchema.index({ 'remote_address': 1 }, {
+    background: true,
+})
 
 export const EventsModel = (dbClient: mongoose.Connection) =>
     dbClient.model<EventDocument>(

src/database/types/IEvent.ts

@@ -11,7 +11,7 @@ export interface IEvent extends IRecord {
     event_tags: Tag[][]
     event_signature: Buffer
     event_delegator?: Buffer | null
-    event_deduplication?: string[] | null
+    event_deduplication?: (string | number)[] | null
     first_seen: Date
     deleted_at?: Date
     expires_at?: number

src/repositories/event-repository.ts

@@ -11,6 +11,7 @@ import { IEvent } from '../database/types/index.ts'
 import { createLogger } from '../factories/logger-factory.ts'
 import { isGenericTagQuery } from '../utils/filter.ts'
 import { toBuffer } from '../utils/transform.ts'
+import { isChannelMetadata } from '../utils/event.ts'
 
 const toNumber = (input: number) => Number(input)
 
@@ -124,8 +125,8 @@ export class EventRepository implements IEventRepository {
         const row = applySpec({
             event_id: pipe(prop('id'), toBuffer),
             event_pubkey: pipe(prop('pubkey'), toBuffer),
-            event_created_at: prop('created_at'),
-            event_kind: prop('kind'),
+            event_created_at: pipe(prop('created_at'), toNumber),
+            event_kind: pipe(prop('kind'), toNumber),
             event_tags: pipe(prop('tags'), toJSON),
             event_content: prop('content'),
             event_signature: pipe(prop('sig'), toBuffer),
@@ -160,7 +161,7 @@ export class EventRepository implements IEventRepository {
     public async upsert(event: Event): Promise<number> {
         debug('upserting event: %o', event)
 
-        const row = applySpec<IEvent>({
+        const row: IEvent = applySpec({
             event_id: pipe(prop('id'), toBuffer),
             event_pubkey: pipe(prop('pubkey'), toBuffer),
             event_created_at: prop('created_at'),
@@ -191,10 +192,12 @@ export class EventRepository implements IEventRepository {
         })(event)
 
         const extraFilter: any = {}
-        if (Number(row.event_kind) === 41 && Array.isArray(row.event_tags) && row.event_tags.length > 0) {
-            const tags = [...new Set(row.event_tags.reduce((p: string[][], v: string[]) => [...p, ...v], []))]
-            extraFilter['event_tags.0.0'] = EventTags.Event;
-            extraFilter['event_tags'] = { $elemMatch: { $elemMatch: { $in: tags.filter(tag => tag !== EventTags.Event) } } }
+        if (isChannelMetadata(row.event_kind) && Array.isArray(row.event_tags) && row.event_tags.length > 0) {
+            if (Array.isArray(row.event_tags[0]) && row.event_tags[0].length >= 2) {
+                const [, parentId] = row.event_tags[0];
+                extraFilter['event_tags.0.0'] = EventTags.Event;
+                extraFilter['event_tags.0.1'] = parentId;
+            }
         }
 
         const query = masterEventsModel
@@ -207,6 +210,7 @@ export class EventRepository implements IEventRepository {
                         { event_kind: { $eq: 0 } },
                         { event_kind: { $eq: 3 } },
                         { event_kind: { $eq: 41 } },
+                        { event_kind: { $eq: 141 } },
                         { event_kind: { $gte: 10000, $lt: 20000 } },
                         { event_kind: { $gte: 30000, $lt: 40000 } },
                     ],