From 53b5ea490bb4e5ed5ce267f8083e289fbea906df Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 24 Oct 2025 11:20:55 -0400 Subject: [PATCH 01/46] Start of v2 --- .github/workflows/main.yml | 23 +++- .gitignore | 2 + README.md | 29 ++--- config.nims | 21 ++++ nim.cfg | 8 ++ pkger.json | 3 + pkger/.gitignore | 1 + pkger/deps.json | 49 ++++++++ src/objs.nim | 89 +++++++++++++++ src/proto2.nim | 221 +++++++++++++++++++++++++++++++++++++ tests/tproto2.nim | 135 ++++++++++++++++++++++ 11 files changed, 565 insertions(+), 16 deletions(-) create mode 100644 nim.cfg create mode 100644 pkger.json create mode 100644 pkger/.gitignore create mode 100644 pkger/deps.json create mode 100644 src/objs.nim create mode 100644 src/proto2.nim create mode 100644 tests/tproto2.nim diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e3c1e83..c9ae502 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,10 +11,9 @@ jobs: strategy: matrix: version: - - binary:1.6.18 + - binary:2.2.4 os: - ubuntu-latest - # - macOS-latest steps: - uses: actions/checkout@v1 - uses: iffy/install-nim@v5 @@ -55,4 +54,24 @@ jobs: - uses: actions/checkout@v1 - run: docker build --file docker/singleuser.Dockerfile . - run: docker build --file docker/multiuser.Dockerfile . + + binaries: + strategy: + matrix: + version: + - binary:2.2.4 + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - uses: actions/checkout@v1 + - uses: iffy/install-nim@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + version: ${{ matrix.version }} + - name: Update nimble + run: nimble install -y nimble + - name: Install deps diff --git a/.gitignore b/.gitignore index 06f0478..062926b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ bucketsrelay.sqlite _tests !TODO fly.toml +relay.sqlite +libs diff --git a/README.md b/README.md index 8b54453..ff6962c 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,26 @@ This repository contains the open source code for the [Buckets](https://www.budgetwithbuckets.com) relay server, which allows users to share budget data between their devices in an end-to-end encrypted way. +You can use the publicly available relay at + ## Quickstart - single user mode If you want to run the relay on your own computer with only one user account, do the following: 1. Install [Nim](https://nim-lang.org/) -2. Build the relay: +2. Get the code: ``` git clone https://github.com/buckets/relay.git buckets-relay.git cd buckets-relay.git +``` + +3. Install dependencies + +``` +nimble install https://github.com/iffy/pkger/ +``` + nimble singleuserbins ``` @@ -37,8 +47,6 @@ Register users via `brelay adduser ...` or through the web interface. Registration-related emails are sent through [Postmark](https://postmarkapp.com/). Set `POSTMARK_API_KEY` to your Postmark key to use it. Otherwise, disable emails with `-d:nopostmark`. -Users can authenticate with their Buckets license if you set an environment variable `AUTH_LICENSE_PUBKEY=` - ## Security - You should ensure that connections to this relay server are made with TLS. @@ -73,14 +81,12 @@ fly secrets set RELAY_USERNAME='someusername' RELAY_PASSWORD='somepassword' ```sh fly launch --dockerfile docker/multiuser.Dockerfile -fly secrets set POSTMARK_API_KEY='your key' AUTH_LICENSE_PUBKEY='the key' LICENSE_HASH_SALT='choose something here' +fly secrets set POSTMARK_API_KEY='your key' ``` | Variable | Description | |---|---| | `POSTMARK_API_KEY` | API key from [Postmark](https://postmarkapp.com/) | -| `AUTH_LICENSE_PUBKEY` | RSA public key of Buckets licenses. If empty, license authentication is disabled. | -| `LICENSE_HASH_SALT` | A hashing salt for the case when a license needs to be disabled. Any random, but consistent value is fine. | ## Protocol @@ -90,12 +96,7 @@ In summary, devices connect with websockets and exchange messages. Messages sent ### Authentication -Clients authenticate with the server in two ways: - -1. With a relay account via HTTP Basic authentication. This is used to group together a user's various clients and prevent abuse. -2. With a public/private key. This is used to identify and connect individual clients. - -A single relay account can have multiple public/private keys; typically one for each device. +Clients authenticate with the server with a public/private key. A single person may have multiple public/private keys; typically one for each device. ### Client Commands @@ -103,7 +104,7 @@ Clients send the following commands: | Command | Description | |--------------|-------------| -| `Iam` | In response to a `Who` event, proves that this client has the private key for their public key. | +| `Iam` | In response to a `Who` event, proves that this client has the private key | | `Connect` | Asks the server for a connection to another client identified by the client's public key. | | `Disconnect` | Asks the server to disconnect a connection to another client. | | `SendData` | Sends bytes to another client. | @@ -130,7 +131,7 @@ The relay server sends the following events: Authentication happens like this: 1. On connection, server sends `Who(challenge=ABCD...)` -2. Client responds with `Iam(pubkey=MYPK..., signature=SIGN...)` +2. Client responds with `Iam(username=USER..., password=PASS..., pubkey=MYPK..., signature=SIGN...)` 3. If the signature is correct, server sends `Authenticated` ``` diff --git a/config.nims b/config.nims index 1a49a70..e5ed4ac 100644 --- a/config.nims +++ b/config.nims @@ -1,3 +1,24 @@ # See LICENSE.md for licensing switch("gc", "orc") switch("threads", "on") + +import os +const ROOT = currentSourcePath.parentDir() +switch("dynlibOverride", "libsodium") + +const archsegment = block: + when hostCPU == "i386": + "x32" + elif hostCPU == "arm64": + "arm64" + else: + "x64" + +when defined(macosx): + switch("passL", ROOT/"libs"/"libsodium"/"macos"/archsegment/"libsodium.a") +elif defined(linux): + switch("cincludes", ROOT/"libs"/"libsodium"/"linux"/archsegment/"include") + switch("clibdir", ROOT/"libs"/"libsodium"/"linux"/archsegment/"lib") + switch("passL", "-lsodium") +elif defined(windows): + switch("passL", ROOT/"libs"/"libsodium"/"win"/archsegment/"libsodium.a") \ No newline at end of file diff --git a/nim.cfg b/nim.cfg new file mode 100644 index 0000000..518c89e --- /dev/null +++ b/nim.cfg @@ -0,0 +1,8 @@ + +### PKGER START - DO NOT EDIT BELOW ######### +--path:"pkger/lazy/libsodium" +--path:"pkger/lazy/lowdb/src" +--path:"pkger/lazy/ws/src" +--path:"pkger/lazy/nimja/src" +--path:"pkger/lazy/db_connector/src" +### PKGER END - DO NOT EDIT ABOVE ########### diff --git a/pkger.json b/pkger.json new file mode 100644 index 0000000..15f1ae6 --- /dev/null +++ b/pkger.json @@ -0,0 +1,3 @@ +{ + "dir": "pkger" +} diff --git a/pkger/.gitignore b/pkger/.gitignore new file mode 100644 index 0000000..36f068a --- /dev/null +++ b/pkger/.gitignore @@ -0,0 +1 @@ +lazy diff --git a/pkger/deps.json b/pkger/deps.json new file mode 100644 index 0000000..ebd7bec --- /dev/null +++ b/pkger/deps.json @@ -0,0 +1,49 @@ +{ + "pinned": { + "libsodium": { + "pkgname": "libsodium", + "parent": "", + "src": { + "url": "https://github.com/FedericoCeratto/nim-libsodium", + "kind": "git" + }, + "sha": "0258efe4e7f48e22daedf26f70e3efbe830abfb5" + }, + "lowdb": { + "pkgname": "lowdb", + "parent": "", + "src": { + "url": "https://github.com/PhilippMDoerner/lowdb", + "kind": "git" + }, + "sha": "c8bd2c631ed923b25c0bab4cd1ccbe84bf56400b" + }, + "ws": { + "pkgname": "ws", + "parent": "", + "src": { + "url": "https://github.com/treeform/ws", + "kind": "git" + }, + "sha": "cbb8f763b436669392d10baec2a45778395395cc" + }, + "nimja": { + "pkgname": "nimja", + "parent": "", + "src": { + "url": "https://github.com/enthus1ast/nimja", + "kind": "git" + }, + "sha": "438fc1f6c654c69cd85c9a5d9cbe45a48f507f8c" + }, + "db_connector": { + "pkgname": "db_connector", + "parent": "", + "src": { + "url": "https://github.com/nim-lang/db_connector", + "kind": "git" + }, + "sha": "74aef399e5c232f95c9fc5c987cebac846f09d62" + } + } +} \ No newline at end of file diff --git a/src/objs.nim b/src/objs.nim new file mode 100644 index 0000000..b6be9bb --- /dev/null +++ b/src/objs.nim @@ -0,0 +1,89 @@ +# Copyright (c) One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +## These are the objects used for the protocol + +import std/strformat +import std/base64 +import std/hashes + +type + PublicKey* = distinct string + SecretKey* = distinct string + + MessageKind* = enum + Who + Okay + Error + Note + + CommandKind* = enum + Iam + PublishNote + FetchNote + + ErrorCode* = enum + Generic = 0 + + RelayMessage* = object + case kind*: MessageKind + of Who: + who_challenge*: string + of Okay: + ok_cmd*: CommandKind + of Error: + err_code*: ErrorCode + err_message*: string + of Note: + note_data*: string + + RelayCommand* = object + case kind*: CommandKind + of Iam: + iam_pubkey*: PublicKey + iam_signature*: string + of PublishNote: + pub_topic*: string + pub_data*: string + of FetchNote: + fetch_topic*: string + +template b64encode(x: string): string = base64.encode(x) + +proc `$`*(k: PublicKey): string = b64encode(k.string) +proc hash*(p: PublicKey): Hash {.borrow.} +proc `==`*(a,b: PublicKey): bool {.borrow.} + +proc abbr*(s: string, size = 6): string = + if s.len > size: + result.add s.substr(0, size) & "..." + else: + result.add(s) + +proc abbr*(a: PublicKey): string = abbr($a) + +proc `$`*(msg: RelayMessage): string = + result.add "(" & $msg.kind & " " + case msg.kind + of Who: + result.add "challenge=" & b64encode(msg.who_challenge).abbr + of Okay: + result.add &"cmd={msg.ok_cmd}" + of Error: + result.add &"code={msg.err_code} msg={msg.err_message}" + of Note: + result.add msg.note_data + result.add ")" + +proc `$`*(cmd: RelayCommand): string = + result.add "(" & $cmd.kind & " " + case cmd.kind + of Iam: + result.add &"{cmd.iam_pubkey.abbr} sig={cmd.iam_signature.b64encode.abbr}" + of PublishNote: + result.add &"'{cmd.pub_topic}' data={cmd.pub_data.b64encode}" + of FetchNote: + result.add &"'{cmd.fetch_topic}'" + result.add ")" \ No newline at end of file diff --git a/src/proto2.nim b/src/proto2.nim new file mode 100644 index 0000000..1616f91 --- /dev/null +++ b/src/proto2.nim @@ -0,0 +1,221 @@ +# Copyright (c) One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +import std/logging +import std/options +import std/strutils +import std/strformat +import std/tables + +import lowdb/sqlite +import libsodium/sodium + +import ./objs; export objs + +const LOG_COMMS = not defined(release) + +type + KeyPair* = tuple + pk: PublicKey + sk: SecretKey + + Relay*[T] = object + db: DbConn + clients: TableRef[PublicKey, RelayConnection[T]] + + RelayConnection*[T] = ref object + sender*: T + pubkey*: PublicKey ## The authenticated pubkey + challenge: string + +#------------------------------------------------------------------- +# Database +#------------------------------------------------------------------- +func strval*(dbval: sqlite.DbValue): string = + case dbval.kind + of dvkString: + dbval.s + of dvkNull: + "" + else: + raise ValueError.newException("Can't get string from " & $dbval.kind) + +template patch(db: untyped, applied: seq[string], name: string, body: untyped): untyped = + block: + if name notin applied: + info name, " - applying..." + db.exec(sql"BEGIN") + try: + body + db.exec(sql"INSERT INTO _schema_patches (name) VALUES (?)", name) + db.exec(sql"COMMIT") + except: + error name, " - error applying patch: " & getCurrentExceptionMsg() + db.exec(sql"ROLLBACK") + raise + else: + debug name, " - applied" + +proc updateSchema*(db: DbConn) = + ## Upgrade the schema + db.exec(sql"""CREATE TABLE IF NOT EXISTS _schema_patches ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE, + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP + )""") + + var applied: seq[string] + for row in db.getAllRows(sql"SELECT name FROM _schema_patches"): + applied.add(row[0].strval) + + info "Already applied patches: ", applied.join(",") + + db.patch(applied, "initial"): + db.exec(sql"""CREATE TABLE note ( + topic TEXT PRIMARY KEY, + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + data BLOB DEFAULT '' + )""") + + #----------- in-memory stuff + db.exec(sql"""CREATE TEMPORARY TABLE note_sub ( + topic TEXT PRIMARY KEY, + pubkey TEXT NOT NULL + )""") + db.exec(sql"CREATE INDEX note_sub_pubkey ON note_sub(pubkey)") + +#------------------------------------------------------------------- +# Relay code +#------------------------------------------------------------------- + +proc newRelay*[T](db: DbConn): Relay[T] = + result.db = db + result.clients = newTable[PublicKey, RelayConnection[T]]() + db.updateSchema() + +template sendMessage*[T](conn: RelayConnection[T], msg: RelayMessage) = + when LOG_COMMS: + info "[" & conn.pubkey.abbr & "] <- " & $msg + conn.sender.sendMessage(msg) + +template sendError*[T](conn: RelayConnection[T], msg: string) = + conn.sendMessage(RelayMessage( + kind: Error, + err_code: Generic, + err_message: msg, + )) + +template sendOkay*[T](conn: RelayConnection[T], cmd: CommandKind) = + conn.sendMessage(RelayMessage( + kind: Okay, + ok_cmd: cmd, + )) + +proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = + new(result) + result.sender = client + result.challenge = randombytes(32) + result.sendMessage(RelayMessage( + kind: Who, + who_challenge: result.challenge, + )) + +#------------------------------------------------------------------- +# pub/sub notes +#------------------------------------------------------------------- + +proc addNoteSub(relay: Relay, topic: string, pubkey: PublicKey) = + ## Record that a pubkey is subscribed to a topic + try: + relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic, pubkey.string) + info &"[{pubkey.abbr}] sub {topic}" + except: + raise ValueError.newException("Topic already subscribed") + +proc getNoteSub(relay: Relay, topic: string): Option[PublicKey] = + let orow = relay.db.getRow(sql"SELECT pubkey FROM note_sub WHERE topic = ?", topic) + if orow.isSome: + return some(orow.get()[0].s.PublicKey) + +proc popNote(relay: Relay, topic: string): Option[string] = + let db = relay.db + db.exec(sql"BEGIN") + try: + let orow = db.getRow(sql"SELECT data FROM note WHERE topic=?", topic) + if orow.isSome: + let row = orow.get() + result = some(row[0].strval) + info &"[note] pop {topic}" + db.exec(sql"DELETE FROM note WHERE topic=?", topic) + else: + debug &"[note] dne {topic}" + db.exec(sql"COMMIT") + except: + warn &"[note] error " & getCurrentExceptionMsg() + db.exec(sql"ROLLBACK") + +proc delNoteSub(relay: Relay, topic: string) = + relay.db.exec(sql"DELETE FROM note_sub WHERE topic = ?", topic) + info &"[note] del {topic}" + +#------------------------------------------------------------------- +# relay command handling +#------------------------------------------------------------------- + +proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: RelayCommand) = + when LOG_COMMS: + info "[" & conn.pubkey.abbr & "] DO " & $cmd + case cmd.kind + of Iam: + try: + crypto_sign_verify_detached(cmd.iam_pubkey.string, conn.challenge, cmd.iam_signature) + except: + conn.challenge = "" # disable authentication + conn.sendError "Invalid signature" + return + conn.pubkey = cmd.iam_pubkey + relay.clients[conn.pubkey] = conn + info &"[{conn.pubkey.abbr}] connected" + conn.sendOkay cmd.kind + of PublishNote: + let opubkey = relay.getNoteSub(cmd.pub_topic) + if opubkey.isSome: + # someone is waiting + var other_conn = relay.clients[opubkey.get()] + conn.sendOkay cmd.kind + other_conn.sendMessage(RelayMessage( + kind: Note, + note_data: cmd.pub_data, + )) + relay.delNoteSub(cmd.pub_topic) + else: + # no one is waiting + relay.db.exec(sql"INSERT INTO note (topic, data) VALUES (?, ?)", + cmd.pub_topic, + cmd.pub_data, + ) + conn.sendOkay cmd.kind + of FetchNote: + let odata = relay.popNote(cmd.fetch_topic) + if odata.isSome(): + # the note is already here + conn.sendMessage(RelayMessage( + kind: Note, + note_data: odata.get(), + )) + else: + # the note isn't here yet + relay.addNoteSub(cmd.fetch_topic, conn.pubkey) + +#------------------------------------------------------------------- +# Utilities +#------------------------------------------------------------------- +proc genkeys*(): KeyPair = + let (pk, sk) = crypto_sign_keypair() + result = (pk.PublicKey, sk.SecretKey) + +proc sign*(key: SecretKey, message: string): string = + ## Sign a message with the given secret key + result = crypto_sign_detached(key.string, message) diff --git a/tests/tproto2.nim b/tests/tproto2.nim new file mode 100644 index 0000000..35b0b14 --- /dev/null +++ b/tests/tproto2.nim @@ -0,0 +1,135 @@ +import std/deques +import std/logging +import std/unittest +import std/os + +import lowdb/sqlite +import proto2 + +if getEnv("SHOW_LOGS") != "": + var L = newConsoleLogger() + addHandler(L) +else: + echo "set SHOW_LOGS=something to see logs" + + +#--------------------------------- +# In-memory TestClient +#--------------------------------- +type + TestClient* = ref object + received: Deque[RelayMessage] + pk: PublicKey + sk: SecretKey + +proc newTestClient*(): TestClient = + new(result) + result.received = initDeque[RelayMessage]() + +proc newTestClient*(keys: KeyPair): TestClient = + new(result) + result = newTestClient() + result.pk = keys.pk + result.sk = keys.sk + +proc sendMessage*(c: var TestClient, msg: RelayMessage) = + c.received.addLast(msg) + +proc pop*(c: var TestClient): RelayMessage = + c.received.popFirst() + +#--------------------------------- +# Test utilities +#--------------------------------- +proc testRelay(): Relay[TestClient] = + newRelay[TestClient](open(":memory:", "", "", "")) + +proc pop(conn: var RelayConnection[TestClient]): RelayMessage = + conn.sender.pop() + +proc pop(conn: var RelayConnection[TestClient], expected: MessageKind): RelayMessage = + try: + result = conn.sender.pop() + except IndexDefect: + raise IndexDefect.newException("Error getting message of kind: " & $expected) + doAssert result.kind == expected + +proc pk(conn: var RelayConnection[TestClient]): PublicKey = conn.sender.pk +proc sk(conn: var RelayConnection[TestClient]): SecretKey = conn.sender.sk + +proc authenticatedConn(relay: Relay): RelayConnection[TestClient] = + let client = newTestClient(genkeys()) + var conn = relay.initAuth(client) + let who = conn.pop() + doAssert who.kind == Who + let sig = client.sk.sign(who.who_challenge) + relay.handleCommand(conn, RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: client.pk)) + let ok = conn.pop() + doAssert ok.kind == Okay + doAssert ok.ok_cmd == Iam + return conn + +#--------------------------------- +# End of TestClient +#--------------------------------- + +test "auth": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + checkpoint "who?" + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + check who.who_challenge != "" + echo $who + + checkpoint "iam" + let signature = aclient.sk.sign(who.who_challenge) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + discard alice.pop(Okay) + +test "PublishNote": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "somedata", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + let data = bob.pop(Note) + check data.note_data == "somedata" + +test "fetch prior to pub": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "heyo", + )) + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "heyo", + pub_data: "foo", + )) + check alice.pop(Okay).ok_cmd == PublishNote + let note = alice.pop(Note) + check note.note_data == "foo" + +test "iam twice": check false +test "iam invalid sig": check false +test "publish max size topic": check false +test "publish max size data": check false +test "publish expiration": check false +test "fetch dne topic": check false +test "fetch note again": check false +test "limit number of simultaneous fetches": check false +test "sub then disconnect, the pub": check false From e246ca1605b0bcfa05b330c0d688bd97874b9d25 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Mon, 27 Oct 2025 15:29:44 -0400 Subject: [PATCH 02/46] Auth and notes work --- README.md | 123 +++++++++------------- src/objs.nim | 14 ++- src/proto2.nim | 126 ++++++++++++++++------ tests/tproto2.nim | 261 +++++++++++++++++++++++++++++++++++----------- 4 files changed, 357 insertions(+), 167 deletions(-) diff --git a/README.md b/README.md index ff6962c..1be61ca 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ This repository contains the open source code for the [Buckets](https://www.budg You can use the publicly available relay at -## Quickstart - single user mode +## Quickstart -If you want to run the relay on your own computer with only one user account, do the following: +If you want to run the relay on your own computer, do the following: 1. Install [Nim](https://nim-lang.org/) 2. Get the code: @@ -24,28 +24,18 @@ cd buckets-relay.git ``` nimble install https://github.com/iffy/pkger/ +pkger fetch ``` -nimble singleuserbins -``` +4. Run the server: -3. Run the server: +TODO: ```sh -RELAY_USERNAME=someusername -RELAY_PASSWORD=somepassword -bin/brelay server +nim r src/brelay.nim server ``` -This will launch the relay on the default port. Run `brelay --help` for more options. - -## Multi-user mode - -If instead of `nimble singleuserbins` you run `nimble multiuserbins` the server will be built in multi-user mode. - -Register users via `brelay adduser ...` or through the web interface. - -Registration-related emails are sent through [Postmark](https://postmarkapp.com/). Set `POSTMARK_API_KEY` to your Postmark key to use it. Otherwise, disable emails with `-d:nopostmark`. +This will launch the relay on the default port. Run with `--help` for more options. ## Security @@ -55,6 +45,8 @@ Registration-related emails are sent through [Postmark](https://postmarkapp.com/ ## Development +TODO: + To run the server locally: ```sh @@ -65,29 +57,20 @@ nimble run brelay server If you'd like to run a relay server on [fly.io](https://fly.io/), sign up for the service then do one of the following. If you'd like to host somewhere else, you could use the Dockerfiles in [docker/](./docker/) as a starting point. +TODO: + ### Single-user mode ```sh fly launch --dockerfile docker/singleuser.Dockerfile -fly secrets set RELAY_USERNAME='someusername' RELAY_PASSWORD='somepassword' ``` -| Variable | Description | -|---|---| -| `RELAY_USERNAME` | Username or email you'll use to authenticate to the relay. | -| `RELAY_PASSWORD` | Password you'll use to authenticate to the relay. | - ### Multi-user mode ```sh fly launch --dockerfile docker/multiuser.Dockerfile -fly secrets set POSTMARK_API_KEY='your key' ``` -| Variable | Description | -|---|---| -| `POSTMARK_API_KEY` | API key from [Postmark](https://postmarkapp.com/) | - ## Protocol Relay clients communicate with the relay server using the following protocol. See [./src/bucketsrelay/proto.nim](./src/bucketsrelay/proto.nim) for more information, and [./src/bucketsrelay/stringproto.nim](./src/bucketsrelay/stringproto.nim) for encoding details. @@ -102,12 +85,12 @@ Clients authenticate with the server with a public/private key. A single person Clients send the following commands: -| Command | Description | -|--------------|-------------| -| `Iam` | In response to a `Who` event, proves that this client has the private key | -| `Connect` | Asks the server for a connection to another client identified by the client's public key. | -| `Disconnect` | Asks the server to disconnect a connection to another client. | -| `SendData` | Sends bytes to another client. | +| Command | Description | +|----------------|-------------| +| `Iam` | In response to a `Who` event, proves that this client has the private key | +| `PublishNote` | Send a few bytes to another client addressed by topic (good for key exchange) | +| `FetchNote` | Request a note addressed by topic | +| `SendData` | Store/forward bytes to another client, addressed by relay-authenticated public key | ### Server Events @@ -115,14 +98,11 @@ The relay server sends the following events: | Event | Description | |-----------------|-------------| +| `Okay` | Sent when certain commands succeed | +| `Error` | Sent when commands fail | | `Who` | Challenge for authenticating a client's public/private keys | -| `Authenticated` | Sent when a client successfully completes authentication | -| `Connected` | Sent when a client has connected to another client | -| `Disconnected` | Sent when a client has been disconnected from another client | -| `Data` | Data payload from another, connected client | -| `Entered` | Sent when a client within the same user account has authenticated to the relay | -| `Exited` | Sent when a client within the same user account has disconnected from the relay | -| `ErrorEvent` | Sent when errors happen with authentication, connection or message sending | +| `Note` | Data payload of a note requested by `FetchNote` | +| `Data` | Data payload from another client, addressed by relay-authenticated public key | ### Sequences and Usage @@ -131,8 +111,8 @@ The relay server sends the following events: Authentication happens like this: 1. On connection, server sends `Who(challenge=ABCD...)` -2. Client responds with `Iam(username=USER..., password=PASS..., pubkey=MYPK..., signature=SIGN...)` -3. If the signature is correct, server sends `Authenticated` +2. Client responds with `Iam(pubkey=MYPK..., signature=SIGN...)` +3. If the signature is correct, server sends `Okay(cmd=Iam)` ``` Client Relay @@ -143,49 +123,50 @@ Client Relay │ Iam │ ├────────────────►│ │ │ - │ Authenticated │ + │ Okay │ │◄────────────────┤ │ │ ``` -#### Client-to-client connection +#### Notes + +After authenticating, clients can send each other short notes, addressed by a string *topic*. Each note expires after a time and will only ever been sent to one client who. The `FetchNote` command may be sent before or after the note is published. It works like this: + +1. Alice sends `PublishNote(topic=apple, data=something)` +2. Bob sends `FetchNote(topic=apple)` +3. Server sends to Bob `Note(topic=apple, data=something)` + +``` +Alice Relay Bob + │ │ │ + ├───────Authenticated─|─Authenticated──────┤ + │ │ │ + │ PublishNote(apple) | │ + ├────────────────────►│ FetchNote(apple) | + │ │◄───────────────────┤ + │ │ │ + │ │ Note(apple) │ + │ │───────────────────►│ + │ │ │ +``` + +#### Data + +After authenticating, clients may send data to be stored and forwarded to clients next time they connect. Stored data expires after a time. In other words, the transport is unreliable by design. -After authenticating, clients connect to each other and send data like this: +Here's how it works: -1. Alice sends `Connect(pubkey=BOBPK)` -2. Bob sends `Connect(pubkey=ALICEPK)` -3. Server sends Alice `Connected(pubkey=BOBPK)` -4. Server sends Bob `Connected(pubkey=ALICEPK)` -5. Alice sends data with `SendData(data=hello, pubkey=BOBPK)` -6. Server sends Bob data with `Data(data=hello, sender=ALICEPK)` +1. Alice sends `SendData(dst=BOBPK, data=hello)` +2. Server sends to Bob `Data(src=ALICEPK, data=hello)` ``` Alice Relay Bob │ │ │ ├───Authenticated─┼─Authenticated───┤ │ │ │ - │Connect(Bob) │ │ - ├────────────────►│ Connect(Alice) │ - │ │◄────────────────┤ - │ │ │ - │ Connected(Bob) │ Connected(Alice)│ - │◄────────────────┼────────────────►│ - │ │ │ │SendData(Bob) │ │ ├────────────────►│ Data(Alice) │ │ ├────────────────►│ │ │ │ ``` -#### Same-user presence notifications - -The relay server will announce client presence to all clients that use the same HTTP Auth credentials. For example, if both Alice and Bob signed in as `alicenbob@example.com` the following would happen: - -1. Alice finishes authenticating -2. Bob finishes authenticating -3. Server sends Alice `Entered(pubkey=BOBPK)` -4. Server sends Bob `Entered(pubkey=ALICEPK)` -5. Alice disconnects -6. Server sends Bob `Exited(pubkey=ALICEPK)` - - diff --git a/src/objs.nim b/src/objs.nim index b6be9bb..8d1c0d0 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -3,7 +3,9 @@ # This work is licensed under the terms of the MIT license. # For a copy, see LICENSE.md in this repository. -## These are the objects used for the protocol +## These are the objects used for the protocol. +## This file should be kept free of dependencies other than the stdlib +## as it's meant to be referenced by outside libraries. import std/strformat import std/base64 @@ -26,6 +28,7 @@ type ErrorCode* = enum Generic = 0 + TooLarge = 1 RelayMessage* = object case kind*: MessageKind @@ -36,7 +39,9 @@ type of Error: err_code*: ErrorCode err_message*: string + err_cmd*: CommandKind of Note: + note_topic*: string note_data*: string RelayCommand* = object @@ -50,6 +55,11 @@ type of FetchNote: fetch_topic*: string +const + MAX_TOPIC_SIZE* = 512 + MAX_NOTE_SIZE* = 8096 + RELAY_NOTE_DURATION* = 5 * 24 * 60 * 60 + template b64encode(x: string): string = base64.encode(x) proc `$`*(k: PublicKey): string = b64encode(k.string) @@ -72,7 +82,7 @@ proc `$`*(msg: RelayMessage): string = of Okay: result.add &"cmd={msg.ok_cmd}" of Error: - result.add &"code={msg.err_code} msg={msg.err_message}" + result.add &"cmd={msg.err_cmd} code={msg.err_code} msg={msg.err_message}" of Note: result.add msg.note_data result.add ")" diff --git a/src/proto2.nim b/src/proto2.nim index 1616f91..3e5177b 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -3,11 +3,13 @@ # This work is licensed under the terms of the MIT license. # For a copy, see LICENSE.md in this repository. +import std/base64 import std/logging import std/options import std/strutils import std/strformat import std/tables +import std/times import lowdb/sqlite import libsodium/sodium @@ -15,6 +17,7 @@ import libsodium/sodium import ./objs; export objs const LOG_COMMS = not defined(release) +const TESTMODE = defined(testmode) and not defined(release) type KeyPair* = tuple @@ -30,6 +33,15 @@ type pubkey*: PublicKey ## The authenticated pubkey challenge: string +when TESTMODE: + var TIME_SKEW = 0 + proc skewTime*(seconds: int) = + TIME_SKEW += seconds + proc skewTime*(dur: Duration) = + TIME_SKEW += dur.inSeconds() + proc resetSkew*() = + TIME_SKEW = 0 + #------------------------------------------------------------------- # Database #------------------------------------------------------------------- @@ -42,6 +54,15 @@ func strval*(dbval: sqlite.DbValue): string = else: raise ValueError.newException("Can't get string from " & $dbval.kind) +proc toDB*(p: PublicKey): string = + base64.encode(p.string) + +proc fromDB*(t: typedesc[PublicKey], v: string): PublicKey = + base64.decode(v).PublicKey + +proc fromDB*(t: typedesc[PublicKey], v: DbBlob): PublicKey = + base64.decode(v.string).PublicKey + template patch(db: untyped, applied: seq[string], name: string, body: untyped): untyped = block: if name notin applied: @@ -90,7 +111,24 @@ proc updateSchema*(db: DbConn) = # Relay code #------------------------------------------------------------------- +proc `$`*[T](conn: RelayConnection[T]): string = + result = "RelayConnectiong(" + result &= &"pubkey={conn.pubkey.abbr} " + result &= &"sender={conn.sender}" + if conn.challenge != "": + result &= " cha=" & base64.encode(conn.challenge) + result &= ")" + +proc `$`*[T](tab: TableRef[PublicKey, RelayConnection[T]]): string = + result = "TableRef(" + for key in tab.keys(): + let val = tab[key] + result.add &"{key}: {val}, " + result &= ")" + proc newRelay*[T](db: DbConn): Relay[T] = + when TESTMODE: + resetSkew() result.db = db result.clients = newTable[PublicKey, RelayConnection[T]]() db.updateSchema() @@ -100,11 +138,12 @@ template sendMessage*[T](conn: RelayConnection[T], msg: RelayMessage) = info "[" & conn.pubkey.abbr & "] <- " & $msg conn.sender.sendMessage(msg) -template sendError*[T](conn: RelayConnection[T], msg: string) = +template sendError*[T](conn: RelayConnection[T], msg: string, cmd: CommandKind, code: ErrorCode) = conn.sendMessage(RelayMessage( kind: Error, - err_code: Generic, + err_code: code, err_message: msg, + err_cmd: cmd, )) template sendOkay*[T](conn: RelayConnection[T], cmd: CommandKind) = @@ -122,25 +161,40 @@ proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = who_challenge: result.challenge, )) +proc disconnect*[T](relay: Relay[T], conn: RelayConnection[T]) = + relay.db.exec(sql"DELETE FROM note_sub WHERE pubkey=?", conn.pubkey.toDB) + relay.clients.del(conn.pubkey) + #------------------------------------------------------------------- # pub/sub notes #------------------------------------------------------------------- +proc delExpiredNotes(relay: Relay) = + let offset = when TESTMODE: + -RELAY_NOTE_DURATION + TIME_SKEW + else: + -RELAY_NOTE_DURATION + let offstring = &"{offset} seconds" + relay.db.exec(sql"DELETE FROM note WHERE created <= datetime('now', ?)", offstring) + proc addNoteSub(relay: Relay, topic: string, pubkey: PublicKey) = ## Record that a pubkey is subscribed to a topic try: - relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic, pubkey.string) + relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic, pubkey.toDB) info &"[{pubkey.abbr}] sub {topic}" except: raise ValueError.newException("Topic already subscribed") proc getNoteSub(relay: Relay, topic: string): Option[PublicKey] = + ## Return a PublicKey who is listening for a note by topic. + relay.delExpiredNotes() let orow = relay.db.getRow(sql"SELECT pubkey FROM note_sub WHERE topic = ?", topic) if orow.isSome: - return some(orow.get()[0].s.PublicKey) + return some(PublicKey.fromDB(orow.get()[0].s)) proc popNote(relay: Relay, topic: string): Option[string] = let db = relay.db + relay.delExpiredNotes() db.exec(sql"BEGIN") try: let orow = db.getRow(sql"SELECT data FROM note WHERE topic=?", topic) @@ -172,42 +226,50 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay try: crypto_sign_verify_detached(cmd.iam_pubkey.string, conn.challenge, cmd.iam_signature) except: - conn.challenge = "" # disable authentication - conn.sendError "Invalid signature" + conn.sendError("Invalid signature", cmd.kind, Generic) return conn.pubkey = cmd.iam_pubkey relay.clients[conn.pubkey] = conn - info &"[{conn.pubkey.abbr}] connected" + conn.challenge = "" # disable authentication + info &"[{conn.pubkey}] connected" conn.sendOkay cmd.kind of PublishNote: - let opubkey = relay.getNoteSub(cmd.pub_topic) - if opubkey.isSome: - # someone is waiting - var other_conn = relay.clients[opubkey.get()] - conn.sendOkay cmd.kind - other_conn.sendMessage(RelayMessage( - kind: Note, - note_data: cmd.pub_data, - )) - relay.delNoteSub(cmd.pub_topic) + if cmd.pub_topic.len > MAX_TOPIC_SIZE: + conn.sendError("Topic too long", cmd.kind, TooLarge) + elif cmd.pub_data.len > MAX_NOTE_SIZE: + conn.sendError("Data too long", cmd.kind, TooLarge) else: - # no one is waiting - relay.db.exec(sql"INSERT INTO note (topic, data) VALUES (?, ?)", - cmd.pub_topic, - cmd.pub_data, - ) - conn.sendOkay cmd.kind + let opubkey = relay.getNoteSub(cmd.pub_topic) + if opubkey.isSome: + # someone is waiting + var other_conn = relay.clients[opubkey.get()] + conn.sendOkay cmd.kind + other_conn.sendMessage(RelayMessage( + kind: Note, + note_data: cmd.pub_data, + )) + relay.delNoteSub(cmd.pub_topic) + else: + # no one is waiting + relay.db.exec(sql"INSERT INTO note (topic, data) VALUES (?, ?)", + cmd.pub_topic, + cmd.pub_data, + ) + conn.sendOkay cmd.kind of FetchNote: - let odata = relay.popNote(cmd.fetch_topic) - if odata.isSome(): - # the note is already here - conn.sendMessage(RelayMessage( - kind: Note, - note_data: odata.get(), - )) + if cmd.fetch_topic.len > MAX_TOPIC_SIZE: + conn.sendError("Topic too large", cmd.kind, TooLarge) else: - # the note isn't here yet - relay.addNoteSub(cmd.fetch_topic, conn.pubkey) + let odata = relay.popNote(cmd.fetch_topic) + if odata.isSome(): + # the note is already here + conn.sendMessage(RelayMessage( + kind: Note, + note_data: odata.get(), + )) + else: + # the note isn't here yet + relay.addNoteSub(cmd.fetch_topic, conn.pubkey) #------------------------------------------------------------------- # Utilities diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 35b0b14..8602af4 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -1,7 +1,8 @@ import std/deques import std/logging -import std/unittest import std/os +import std/strutils +import std/unittest import lowdb/sqlite import proto2 @@ -22,6 +23,8 @@ type pk: PublicKey sk: SecretKey +proc `$`*(tc: TestClient): string = $tc[] + proc newTestClient*(): TestClient = new(result) result.received = initDeque[RelayMessage]() @@ -54,11 +57,15 @@ proc pop(conn: var RelayConnection[TestClient], expected: MessageKind): RelayMes raise IndexDefect.newException("Error getting message of kind: " & $expected) doAssert result.kind == expected +proc msgCount(conn: var RelayConnection[TestClient]): int = + conn.sender.received.len + proc pk(conn: var RelayConnection[TestClient]): PublicKey = conn.sender.pk proc sk(conn: var RelayConnection[TestClient]): SecretKey = conn.sender.sk +proc keys(conn: var RelayConnection[TestClient]): KeyPair = (conn.sender.pk, conn.sender.sk) -proc authenticatedConn(relay: Relay): RelayConnection[TestClient] = - let client = newTestClient(genkeys()) +proc authenticatedConn(relay: Relay, keys: KeyPair): RelayConnection[TestClient] = + let client = newTestClient(keys) var conn = relay.initAuth(client) let who = conn.pop() doAssert who.kind == Who @@ -69,67 +76,197 @@ proc authenticatedConn(relay: Relay): RelayConnection[TestClient] = doAssert ok.ok_cmd == Iam return conn +proc authenticatedConn(relay: Relay): RelayConnection[TestClient] = + relay.authenticatedConn(genkeys()) + #--------------------------------- # End of TestClient #--------------------------------- -test "auth": - let relay = testRelay() - let aclient = newTestClient(genkeys()) +suite "Auth": + test "basic": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + checkpoint "who?" + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + check who.who_challenge != "" + echo $who + + checkpoint "iam" + let signature = aclient.sk.sign(who.who_challenge) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + discard alice.pop(Okay) + + test "iam twice": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + checkpoint "who?" + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + check who.who_challenge != "" + + checkpoint "iam" + let signature = aclient.sk.sign(who.who_challenge) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + discard alice.pop(Okay) + + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + check alice.pop().kind == Error + + test "iam invalid sig": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + let signature = aclient.sk.sign(who.who_challenge & "garbage") + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + +suite "PublishNote": + + test "basic": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "somedata", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + let data = bob.pop(Note) + check data.note_data == "somedata" + check data.note_topic == "sometopic" + + test "fetch first": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "heyo", + )) + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "heyo", + pub_data: "foo", + )) + check alice.pop(Okay).ok_cmd == PublishNote + let note = alice.pop(Note) + check note.note_data == "foo" + check note.note_topic == "heyo" + + test "publish max size topic": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "h".repeat(MAX_TOPIC_SIZE + 1), + pub_data: "foo", + )) + block: + let err = alice.pop(Error) + check err.err_code == TooLarge + check err.err_cmd == PublishNote + + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "a".repeat(MAX_TOPIC_SIZE + 1), + )) + block: + let err = alice.pop(Error) + check err.err_code == TooLarge + check err.err_cmd == FetchNote - checkpoint "who?" - var alice = relay.initAuth(aclient) - let who = alice.pop(Who) - check who.who_challenge != "" - echo $who - - checkpoint "iam" - let signature = aclient.sk.sign(who.who_challenge) - relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) - discard alice.pop(Okay) - -test "PublishNote": - let relay = testRelay() - var alice = relay.authenticatedConn() - var bob = relay.authenticatedConn() - - relay.handleCommand(alice, RelayCommand( - kind: PublishNote, - pub_topic: "sometopic", - pub_data: "somedata", - )) - let ok = alice.pop(Okay) - check ok.ok_cmd == PublishNote - - relay.handleCommand(bob, RelayCommand( - kind: FetchNote, - fetch_topic: "sometopic", - )) - let data = bob.pop(Note) - check data.note_data == "somedata" - -test "fetch prior to pub": - let relay = testRelay() - var alice = relay.authenticatedConn() - relay.handleCommand(alice, RelayCommand( - kind: FetchNote, - fetch_topic: "heyo", - )) - relay.handleCommand(alice, RelayCommand( - kind: PublishNote, - pub_topic: "heyo", - pub_data: "foo", - )) - check alice.pop(Okay).ok_cmd == PublishNote - let note = alice.pop(Note) - check note.note_data == "foo" - -test "iam twice": check false -test "iam invalid sig": check false -test "publish max size topic": check false -test "publish max size data": check false -test "publish expiration": check false -test "fetch dne topic": check false -test "fetch note again": check false -test "limit number of simultaneous fetches": check false -test "sub then disconnect, the pub": check false + test "publish max size data": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "topic", + pub_data: "a".repeat(MAX_NOTE_SIZE + 1), + )) + let err = alice.pop(Error) + check err.err_code == TooLarge + check err.err_cmd == PublishNote + + test "expiration": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "topic", + pub_data: "a", + )) + check alice.pop(Okay).ok_cmd == PublishNote + + skewTime(RELAY_NOTE_DURATION) + skewTime(1) + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "topic", + )) + check alice.msgCount == 0 + + test "fetch note again": + let relay = testRelay() + var alice = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "somedata", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + let data = alice.pop(Note) + check data.note_data == "somedata" + + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + check alice.msgCount == 0 + + test "sub then disconnect, the pub": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "foo", + )) + relay.disconnect(bob) + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "foo", + pub_data: "bar", + )) + + var bob2 = relay.authenticatedConn(bob.keys) + check bob2.msgCount == 0 + relay.handleCommand(bob2, RelayCommand( + kind: FetchNote, + fetch_topic: "foo" + )) + let data = bob2.pop(Note) + check data.note_data == "bar" From bbc8e6b0e54d012acf0a9483ab41e56cb091f9de Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Mon, 27 Oct 2025 16:22:39 -0400 Subject: [PATCH 03/46] Message sending works --- README.md | 2 +- src/objs.nim | 31 +++++++++--- src/proto2.nim | 123 ++++++++++++++++++++++++++++++++++++++++----- tests/tproto2.nim | 124 ++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 255 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 1be61ca..d355d0e 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Alice Relay Bob #### Data -After authenticating, clients may send data to be stored and forwarded to clients next time they connect. Stored data expires after a time. In other words, the transport is unreliable by design. +After authenticating, clients may send data to be stored and forwarded to clients next time they connect. Stored data expires after a time. Messages sent to unknown public keys will be dropped without notice. In other words, the transport is unreliable by design. Here's how it works: diff --git a/src/objs.nim b/src/objs.nim index 8d1c0d0..11f9e1b 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -20,11 +20,7 @@ type Okay Error Note - - CommandKind* = enum - Iam - PublishNote - FetchNote + Data ErrorCode* = enum Generic = 0 @@ -43,6 +39,15 @@ type of Note: note_topic*: string note_data*: string + of Data: + data_src*: PublicKey + data_val*: string + + CommandKind* = enum + Iam + PublishNote + FetchNote + SendData RelayCommand* = object case kind*: CommandKind @@ -54,11 +59,17 @@ type pub_data*: string of FetchNote: fetch_topic*: string + of SendData: + dst*: PublicKey + data*: string const - MAX_TOPIC_SIZE* = 512 - MAX_NOTE_SIZE* = 8096 + RELAY_MAX_TOPIC_SIZE* = 512 + RELAY_MAX_NOTE_SIZE* = 4096 RELAY_NOTE_DURATION* = 5 * 24 * 60 * 60 + RELAY_MAX_MESSAGE_SIZE* = 100_000 + RELAY_MESSAGE_DURATION* = 30 * 24 * 60 * 60 + RELAY_PUBKEY_MEMORY_SECONDS* = 60 * 24 * 60 * 60 template b64encode(x: string): string = base64.encode(x) @@ -84,7 +95,9 @@ proc `$`*(msg: RelayMessage): string = of Error: result.add &"cmd={msg.err_cmd} code={msg.err_code} msg={msg.err_message}" of Note: - result.add msg.note_data + result.add &"{msg.note_data.b64encode.abbr} ({msg.note_data.len})" + of Data: + result.add &"src={msg.data_src.abbr} data={msg.data_val.b64encode.abbr} ({msg.data_val.len})" result.add ")" proc `$`*(cmd: RelayCommand): string = @@ -96,4 +109,6 @@ proc `$`*(cmd: RelayCommand): string = result.add &"'{cmd.pub_topic}' data={cmd.pub_data.b64encode}" of FetchNote: result.add &"'{cmd.fetch_topic}'" + of SendData: + result.add &"dst={cmd.dst.abbr} data={cmd.data.b64encode.abbr} ({cmd.data.len})" result.add ")" \ No newline at end of file diff --git a/src/proto2.nim b/src/proto2.nim index 3e5177b..ccab791 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -99,6 +99,21 @@ proc updateSchema*(db: DbConn) = created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, data BLOB DEFAULT '' )""") + db.exec(sql"CREATE INDEX note_created ON note(created)") + db.exec(sql"""CREATE TABLE message ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + src TEXT NOT NULL, + dst TEXT NOT NULL, + data BLOB NOT NULL + )""") + db.exec(sql"CREATE INDEX message_created ON message(created)") + db.exec(sql"CREATE INDEX message_dst ON message(dst)") + db.exec(sql"""CREATE TABLE known_pubkey ( + pubkey TEXT PRIMARY KEY, + last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP + )""") + db.exec(sql"CREATE INDEX known_pubkey_last_seen ON known_pubkey(last_seen)") #----------- in-memory stuff db.exec(sql"""CREATE TEMPORARY TABLE note_sub ( @@ -180,7 +195,7 @@ proc delExpiredNotes(relay: Relay) = proc addNoteSub(relay: Relay, topic: string, pubkey: PublicKey) = ## Record that a pubkey is subscribed to a topic try: - relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic, pubkey.toDB) + relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic.DbBlob, pubkey.toDB) info &"[{pubkey.abbr}] sub {topic}" except: raise ValueError.newException("Topic already subscribed") @@ -188,7 +203,7 @@ proc addNoteSub(relay: Relay, topic: string, pubkey: PublicKey) = proc getNoteSub(relay: Relay, topic: string): Option[PublicKey] = ## Return a PublicKey who is listening for a note by topic. relay.delExpiredNotes() - let orow = relay.db.getRow(sql"SELECT pubkey FROM note_sub WHERE topic = ?", topic) + let orow = relay.db.getRow(sql"SELECT pubkey FROM note_sub WHERE topic = ?", topic.DbBlob) if orow.isSome: return some(PublicKey.fromDB(orow.get()[0].s)) @@ -197,12 +212,12 @@ proc popNote(relay: Relay, topic: string): Option[string] = relay.delExpiredNotes() db.exec(sql"BEGIN") try: - let orow = db.getRow(sql"SELECT data FROM note WHERE topic=?", topic) + let orow = db.getRow(sql"SELECT data FROM note WHERE topic=?", topic.DbBlob) if orow.isSome: let row = orow.get() - result = some(row[0].strval) + result = some(row[0].b.string) info &"[note] pop {topic}" - db.exec(sql"DELETE FROM note WHERE topic=?", topic) + db.exec(sql"DELETE FROM note WHERE topic=?", topic.DbBlob) else: debug &"[note] dne {topic}" db.exec(sql"COMMIT") @@ -211,9 +226,59 @@ proc popNote(relay: Relay, topic: string): Option[string] = db.exec(sql"ROLLBACK") proc delNoteSub(relay: Relay, topic: string) = - relay.db.exec(sql"DELETE FROM note_sub WHERE topic = ?", topic) + relay.db.exec(sql"DELETE FROM note_sub WHERE topic = ?", topic.DbBlob) info &"[note] del {topic}" + +#------------------------------------------------------------------- +# send/receive data +#------------------------------------------------------------------- + +proc forgetOldPubkeys(relay: Relay) = + let offset = when TESTMODE: + -RELAY_PUBKEY_MEMORY_SECONDS + TIME_SKEW + else: + -RELAY_PUBKEY_MEMORY_SECONDS + let offstring = &"{offset} seconds" + relay.db.exec(sql"DELETE FROM known_pubkey WHERE last_seen <= datetime('now', ?)", offstring) + +proc rememberPubkey(relay: Relay, pubkey: PublicKey) = + relay.db.exec(sql""" + INSERT OR REPLACE INTO known_pubkey (pubkey, last_seen) + VALUES (?, CURRENT_TIMESTAMP)""", pubkey.toDB) + +proc isKnown(relay: Relay, pubkey: PublicKey): bool = + relay.forgetOldPubkeys() + let orow = relay.db.getRow(sql"SELECT last_seen FROM known_pubkey WHERE pubkey = ?", pubkey.toDB) + return orow.isSome() + +proc delExpiredMessages(relay: Relay) = + let offset = when TESTMODE: + -RELAY_MESSAGE_DURATION + TIME_SKEW + else: + -RELAY_MESSAGE_DURATION + let offstring = &"{offset} seconds" + relay.db.exec(sql"DELETE FROM message WHERE created <= datetime('now', ?)", offstring) + +proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = + let orow = relay.db.getRow(sql""" + SELECT id, src, data + FROM message + WHERE + dst = ? + ORDER BY + created ASC, + id ASC + LIMIT 1""", dst.toDB) + if orow.isSome: + let row = orow.get() + result = some(RelayMessage( + kind: Data, + data_src: PublicKey.fromDB(row[1].s), + data_val: row[2].b.string, + )) + relay.db.exec(sql"DELETE FROM message WHERE id=?", row[0].i) + #------------------------------------------------------------------- # relay command handling #------------------------------------------------------------------- @@ -228,15 +293,26 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay except: conn.sendError("Invalid signature", cmd.kind, Generic) return + # successful connection conn.pubkey = cmd.iam_pubkey relay.clients[conn.pubkey] = conn conn.challenge = "" # disable authentication + relay.rememberPubkey(conn.pubkey) info &"[{conn.pubkey}] connected" conn.sendOkay cmd.kind + + # send all queued messages + relay.delExpiredMessages() + while true: + let nexto = relay.nextMessage(conn.pubkey) + if nexto.isSome: + conn.sendMessage(nexto.get()) + else: + break of PublishNote: - if cmd.pub_topic.len > MAX_TOPIC_SIZE: + if cmd.pub_topic.len > RELAY_MAX_TOPIC_SIZE: conn.sendError("Topic too long", cmd.kind, TooLarge) - elif cmd.pub_data.len > MAX_NOTE_SIZE: + elif cmd.pub_data.len > RELAY_MAX_NOTE_SIZE: conn.sendError("Data too long", cmd.kind, TooLarge) else: let opubkey = relay.getNoteSub(cmd.pub_topic) @@ -247,18 +323,19 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay other_conn.sendMessage(RelayMessage( kind: Note, note_data: cmd.pub_data, + note_topic: cmd.pub_topic, )) relay.delNoteSub(cmd.pub_topic) else: # no one is waiting relay.db.exec(sql"INSERT INTO note (topic, data) VALUES (?, ?)", - cmd.pub_topic, - cmd.pub_data, + cmd.pub_topic.DbBlob, + cmd.pub_data.DbBlob, ) conn.sendOkay cmd.kind of FetchNote: - if cmd.fetch_topic.len > MAX_TOPIC_SIZE: - conn.sendError("Topic too large", cmd.kind, TooLarge) + if cmd.fetch_topic.len > RELAY_MAX_TOPIC_SIZE: + conn.sendError("Topic too long", cmd.kind, TooLarge) else: let odata = relay.popNote(cmd.fetch_topic) if odata.isSome(): @@ -266,11 +343,31 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay conn.sendMessage(RelayMessage( kind: Note, note_data: odata.get(), + note_topic: cmd.fetch_topic, )) else: # the note isn't here yet relay.addNoteSub(cmd.fetch_topic, conn.pubkey) - + of SendData: + if cmd.data.len > RELAY_MAX_MESSAGE_SIZE: + conn.sendError("Data too long", cmd.kind, TooLarge) + else: + if relay.clients.hasKey(cmd.dst): + # someone is waiting + var other_conn = relay.clients[cmd.dst] + other_conn.sendMessage(RelayMessage( + kind: Data, + data_src: conn.pubkey, + data_val: cmd.data, + )) + else: + # no one is waiting + if relay.isKnown(cmd.dst): + relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?,?,?)", + conn.pubkey.toDB, cmd.dst.toDB, cmd.data.DbBlob, + ) + else: + discard "silently drop the message" #------------------------------------------------------------------- # Utilities #------------------------------------------------------------------- diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 8602af4..fe3d2ea 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -173,7 +173,7 @@ suite "PublishNote": var alice = relay.authenticatedConn() relay.handleCommand(alice, RelayCommand( kind: PublishNote, - pub_topic: "h".repeat(MAX_TOPIC_SIZE + 1), + pub_topic: "h".repeat(RELAY_MAX_TOPIC_SIZE + 1), pub_data: "foo", )) block: @@ -183,7 +183,7 @@ suite "PublishNote": relay.handleCommand(alice, RelayCommand( kind: FetchNote, - fetch_topic: "a".repeat(MAX_TOPIC_SIZE + 1), + fetch_topic: "a".repeat(RELAY_MAX_TOPIC_SIZE + 1), )) block: let err = alice.pop(Error) @@ -196,7 +196,7 @@ suite "PublishNote": relay.handleCommand(alice, RelayCommand( kind: PublishNote, pub_topic: "topic", - pub_data: "a".repeat(MAX_NOTE_SIZE + 1), + pub_data: "a".repeat(RELAY_MAX_NOTE_SIZE + 1), )) let err = alice.pop(Error) check err.err_code == TooLarge @@ -270,3 +270,121 @@ suite "PublishNote": )) let data = bob2.pop(Note) check data.note_data == "bar" + check data.note_topic == "foo" + + test "topic null byte": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "a\x00b", + pub_data: "c\x00d", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "a\x00b", + )) + let data = bob.pop(Note) + check data.note_data == "c\x00d" + check data.note_topic == "a\x00b" + +suite "data": + + test "basic": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: bob.pk, + data: "hel\x00lo", + )) + + let data = bob.pop(Data) + check data.data_src == alice.pk + check data.data_val == "hel\x00lo" + + test "store and forward": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob1 = relay.authenticatedConn() + relay.disconnect(bob1) + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: bob1.pk, + data: "hel\x00lo", + )) + + var bob2 = relay.authenticatedConn(bob1.keys) + let data = bob2.pop(Data) + check data.data_src == alice.pk + check data.data_val == "hel\x00lo" + + test "max data size": + let relay = testRelay() + var alice = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: alice.pk, + data: "a".repeat(RELAY_MAX_MESSAGE_SIZE + 1), + )) + let err = alice.pop(Error) + check err.err_code == TooLarge + check err.err_cmd == SendData + + test "drop unknown key": + let relay = testRelay() + var alice = relay.authenticatedConn() + let bobkeys = genkeys() + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: bobkeys.pk, + data: "a", + )) + check alice.msgCount == 0 + + var bob = relay.authenticatedConn(bobkeys) + check bob.msgCount == 0 # "Should not have stored the message" + + test "drop forgotten key": + let relay = testRelay() + var bob = relay.authenticatedConn() + relay.disconnect(bob) + + skewTime(RELAY_PUBKEY_MEMORY_SECONDS + 1) + var alice = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: bob.pk, + data: "a", + )) + check alice.msgCount == 0 + + var bob2 = relay.authenticatedConn(bob.keys) + check bob2.msgCount == 0 # "Should not have stored the message" + + test "expiration": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + relay.disconnect(bob) + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: bob.pk, + data: "hello", + )) + + skewTime(RELAY_MESSAGE_DURATION + 1) + var bob2 = relay.authenticatedConn(bob.keys) + check bob2.msgCount == 0 From 80de2037c222998c7a65b201a3324b7d303f6164 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Mon, 27 Oct 2025 22:42:01 -0400 Subject: [PATCH 04/46] Multiple recipients --- src/objs.nim | 11 ++++-- src/proto2.nim | 96 +++++++++++++++++++++++++++++++++++------------ tests/tproto2.nim | 49 +++++++++++++++++++++--- 3 files changed, 122 insertions(+), 34 deletions(-) diff --git a/src/objs.nim b/src/objs.nim index 11f9e1b..61844bd 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -60,7 +60,7 @@ type of FetchNote: fetch_topic*: string of SendData: - dst*: PublicKey + dst*: seq[PublicKey] data*: string const @@ -86,7 +86,7 @@ proc abbr*(s: string, size = 6): string = proc abbr*(a: PublicKey): string = abbr($a) proc `$`*(msg: RelayMessage): string = - result.add "(" & $msg.kind & " " + result.add $msg.kind & "(" case msg.kind of Who: result.add "challenge=" & b64encode(msg.who_challenge).abbr @@ -101,7 +101,7 @@ proc `$`*(msg: RelayMessage): string = result.add ")" proc `$`*(cmd: RelayCommand): string = - result.add "(" & $cmd.kind & " " + result.add $cmd.kind & "(" case cmd.kind of Iam: result.add &"{cmd.iam_pubkey.abbr} sig={cmd.iam_signature.b64encode.abbr}" @@ -110,5 +110,8 @@ proc `$`*(cmd: RelayCommand): string = of FetchNote: result.add &"'{cmd.fetch_topic}'" of SendData: - result.add &"dst={cmd.dst.abbr} data={cmd.data.b64encode.abbr} ({cmd.data.len})" + result.add &"data={cmd.data.b64encode.abbr} ({cmd.data.len}) dst=[" + for dst in cmd.dst: + result.add dst.abbr & " " + result.add "]" result.add ")" \ No newline at end of file diff --git a/src/proto2.nim b/src/proto2.nim index ccab791..a659adb 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -25,7 +25,7 @@ type sk: SecretKey Relay*[T] = object - db: DbConn + db*: DbConn clients: TableRef[PublicKey, RelayConnection[T]] RelayConnection*[T] = ref object @@ -94,21 +94,61 @@ proc updateSchema*(db: DbConn) = info "Already applied patches: ", applied.join(",") db.patch(applied, "initial"): + # note db.exec(sql"""CREATE TABLE note ( topic TEXT PRIMARY KEY, created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, data BLOB DEFAULT '' )""") db.exec(sql"CREATE INDEX note_created ON note(created)") + + # message db.exec(sql"""CREATE TABLE message ( id INTEGER PRIMARY KEY AUTOINCREMENT, created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, src TEXT NOT NULL, - dst TEXT NOT NULL, data BLOB NOT NULL )""") db.exec(sql"CREATE INDEX message_created ON message(created)") - db.exec(sql"CREATE INDEX message_dst ON message(dst)") + + # message_route + db.exec(sql"""CREATE TABLE message_route ( + id INTEGER PRIMARY KEY, + message_id INTEGER REFERENCES message(id) ON DELETE CASCADE, + dst TEXT NOT NULL + )""") + db.exec(sql"CREATE INDEX message_route_dst ON message_route(dst)") + + # message_dst + db.exec(sql"""CREATE VIEW message_dst AS + SELECT + m.id, + m.created, + m.src, + r.dst, + m.data, + r.id AS route_id + FROM + message_route AS r + JOIN message AS m + ON m.id = r.message_id + """) + + # auto-delete message orphans + db.exec(sql""" + CREATE TRIGGER IF NOT EXISTS delete_orphaned_message + AFTER DELETE ON message_route + FOR EACH ROW + BEGIN + DELETE FROM message + WHERE id = OLD.message_id + AND NOT EXISTS ( + SELECT 1 FROM message_route WHERE message_id = OLD.message_id + ); + END; + """) + + # known_pubkey db.exec(sql"""CREATE TABLE known_pubkey ( pubkey TEXT PRIMARY KEY, last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP @@ -121,6 +161,7 @@ proc updateSchema*(db: DbConn) = pubkey TEXT NOT NULL )""") db.exec(sql"CREATE INDEX note_sub_pubkey ON note_sub(pubkey)") + db.exec(sql"PRAGMA foreign_keys = ON") #------------------------------------------------------------------- # Relay code @@ -179,6 +220,7 @@ proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = proc disconnect*[T](relay: Relay[T], conn: RelayConnection[T]) = relay.db.exec(sql"DELETE FROM note_sub WHERE pubkey=?", conn.pubkey.toDB) relay.clients.del(conn.pubkey) + info &"[{conn.pubkey.abbr}] disconnected" #------------------------------------------------------------------- # pub/sub notes @@ -262,8 +304,8 @@ proc delExpiredMessages(relay: Relay) = proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = let orow = relay.db.getRow(sql""" - SELECT id, src, data - FROM message + SELECT src, data, route_id + FROM message_dst WHERE dst = ? ORDER BY @@ -274,10 +316,10 @@ proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = let row = orow.get() result = some(RelayMessage( kind: Data, - data_src: PublicKey.fromDB(row[1].s), - data_val: row[2].b.string, + data_src: PublicKey.fromDB(row[0].s), + data_val: row[1].b.string, )) - relay.db.exec(sql"DELETE FROM message WHERE id=?", row[0].i) + relay.db.exec(sql"DELETE FROM message_route WHERE id=?", row[2].i) #------------------------------------------------------------------- # relay command handling @@ -298,7 +340,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay relay.clients[conn.pubkey] = conn conn.challenge = "" # disable authentication relay.rememberPubkey(conn.pubkey) - info &"[{conn.pubkey}] connected" + info &"[{conn.pubkey.abbr}] connected" conn.sendOkay cmd.kind # send all queued messages @@ -352,22 +394,28 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay if cmd.data.len > RELAY_MAX_MESSAGE_SIZE: conn.sendError("Data too long", cmd.kind, TooLarge) else: - if relay.clients.hasKey(cmd.dst): - # someone is waiting - var other_conn = relay.clients[cmd.dst] - other_conn.sendMessage(RelayMessage( - kind: Data, - data_src: conn.pubkey, - data_val: cmd.data, - )) - else: - # no one is waiting - if relay.isKnown(cmd.dst): - relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?,?,?)", - conn.pubkey.toDB, cmd.dst.toDB, cmd.data.DbBlob, - ) + var message_id: Option[int64] + for dst in cmd.dst: + if relay.clients.hasKey(dst): + # dst is online + var other_conn = relay.clients[dst] + other_conn.sendMessage(RelayMessage( + kind: Data, + data_src: conn.pubkey, + data_val: cmd.data, + )) else: - discard "silently drop the message" + # dst is offline + if relay.isKnown(dst): + if message_id.isNone: + # first one of the recipients that's offline + let rowid = relay.db.insertID(sql"INSERT INTO message (src, data) VALUES (?, ?)", + conn.pubkey.toDB, cmd.data.DbBlob) + message_id = some(rowid) + relay.db.exec(sql"INSERT INTO message_route (message_id, dst) VALUES (?, ?)", + message_id.get(), dst.toDB) + else: + discard "silently drop the message" #------------------------------------------------------------------- # Utilities #------------------------------------------------------------------- diff --git a/tests/tproto2.nim b/tests/tproto2.nim index fe3d2ea..5b145da 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -1,5 +1,6 @@ import std/deques import std/logging +import std/options import std/os import std/strutils import std/unittest @@ -302,7 +303,7 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: bob.pk, + dst: @[bob.pk], data: "hel\x00lo", )) @@ -318,7 +319,7 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: bob1.pk, + dst: @[bob1.pk], data: "hel\x00lo", )) @@ -333,7 +334,7 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: alice.pk, + dst: @[alice.pk], data: "a".repeat(RELAY_MAX_MESSAGE_SIZE + 1), )) let err = alice.pop(Error) @@ -347,7 +348,7 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: bobkeys.pk, + dst: @[bobkeys.pk], data: "a", )) check alice.msgCount == 0 @@ -365,7 +366,7 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: bob.pk, + dst: @[bob.pk], data: "a", )) check alice.msgCount == 0 @@ -381,10 +382,46 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: bob.pk, + dst: @[bob.pk], data: "hello", )) skewTime(RELAY_MESSAGE_DURATION + 1) var bob2 = relay.authenticatedConn(bob.keys) check bob2.msgCount == 0 + + test "multiple dst": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + relay.disconnect(bob) + + var carl = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: @[bob.pk], + data: "first", + )) + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: @[bob.pk, carl.pk], + data: "second", + )) + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: @[carl.pk, bob.pk], + data: "third", + )) + relay.handleCommand(alice, RelayCommand( + kind: SendData, + dst: @[carl.pk], + data: "fourth", + )) + var bob2 = relay.authenticatedConn(bob.keys) + check bob2.pop(Data).data_val == "first" + check bob2.pop(Data).data_val == "second" + check bob2.pop(Data).data_val == "third" + check carl.pop(Data).data_val == "second" + check carl.pop(Data).data_val == "third" + check carl.pop(Data).data_val == "fourth" + check relay.db.getRow(sql"SELECT id FROM message").isNone() From 6826bdb46a24bbfb41fd6ada669e69f8fcaa4afb Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 10:14:17 -0400 Subject: [PATCH 05/46] Back to single recipient messages --- README.md | 11 +-- src/objs.nim | 198 ++++++++++++++++++++++++++++++++++++++++++++-- src/proto2.nim | 82 +++++-------------- tests/tproto2.nim | 60 +++----------- tests/tserde2.nim | 34 ++++++++ 5 files changed, 262 insertions(+), 123 deletions(-) create mode 100644 tests/tserde2.nim diff --git a/README.md b/README.md index d355d0e..f59a49a 100644 --- a/README.md +++ b/README.md @@ -90,14 +90,14 @@ Clients send the following commands: | `Iam` | In response to a `Who` event, proves that this client has the private key | | `PublishNote` | Send a few bytes to another client addressed by topic (good for key exchange) | | `FetchNote` | Request a note addressed by topic | -| `SendData` | Store/forward bytes to another client, addressed by relay-authenticated public key | +| `SendData` | Store/forward bytes to other clients, addressed by relay-authenticated public keys | ### Server Events The relay server sends the following events: -| Event | Description | -|-----------------|-------------| +| Event | Description | +|-----------------|------------------------------------| | `Okay` | Sent when certain commands succeed | | `Error` | Sent when commands fail | | `Who` | Challenge for authenticating a client's public/private keys | @@ -156,7 +156,7 @@ After authenticating, clients may send data to be stored and forwarded to client Here's how it works: -1. Alice sends `SendData(dst=BOBPK, data=hello)` +1. Alice sends `SendData(dst=[BOBPK], data=hello)` 2. Server sends to Bob `Data(src=ALICEPK, data=hello)` ``` @@ -164,9 +164,10 @@ Alice Relay Bob │ │ │ ├───Authenticated─┼─Authenticated───┤ │ │ │ - │SendData(Bob) │ │ + │ SendData(Bob) │ │ ├────────────────►│ Data(Alice) │ │ ├────────────────►│ │ │ │ ``` +Note that when multiple recipients are specified in a `SendData` command, all recipients will receive one copy of the message (assuming the message doesn't expire or get discarded instead). diff --git a/src/objs.nim b/src/objs.nim index 61844bd..47cbfac 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -10,6 +10,7 @@ import std/strformat import std/base64 import std/hashes +import std/strutils type PublicKey* = distinct string @@ -60,8 +61,8 @@ type of FetchNote: fetch_topic*: string of SendData: - dst*: seq[PublicKey] - data*: string + send_dst*: PublicKey + send_val*: string const RELAY_MAX_TOPIC_SIZE* = 512 @@ -95,11 +96,27 @@ proc `$`*(msg: RelayMessage): string = of Error: result.add &"cmd={msg.err_cmd} code={msg.err_code} msg={msg.err_message}" of Note: - result.add &"{msg.note_data.b64encode.abbr} ({msg.note_data.len})" + result.add &"'{msg.note_topic}' {msg.note_data.b64encode.abbr} ({msg.note_data.len})" of Data: result.add &"src={msg.data_src.abbr} data={msg.data_val.b64encode.abbr} ({msg.data_val.len})" result.add ")" +proc `==`*(a, b: RelayMessage): bool = + if a.kind != b.kind: + return false + else: + case a.kind + of Who: + return a.who_challenge == b.who_challenge + of Okay: + return a.ok_cmd == b.ok_cmd + of Error: + return a.err_cmd == b.err_cmd and a.err_code == b.err_code and a.err_message == b.err_message + of Note: + return a.note_data == b.note_data and a.note_topic == b.note_topic + of Data: + return a.data_src == b.data_src and a.data_val == b.data_val + proc `$`*(cmd: RelayCommand): string = result.add $cmd.kind & "(" case cmd.kind @@ -110,8 +127,173 @@ proc `$`*(cmd: RelayCommand): string = of FetchNote: result.add &"'{cmd.fetch_topic}'" of SendData: - result.add &"data={cmd.data.b64encode.abbr} ({cmd.data.len}) dst=[" - for dst in cmd.dst: - result.add dst.abbr & " " - result.add "]" - result.add ")" \ No newline at end of file + result.add &"{cmd.send_dst.abbr} data={cmd.send_val.b64encode.abbr} ({cmd.send_val.len})" + result.add ")" + +proc `==`*(a, b: RelayCommand): bool = + if a.kind != b.kind: + return false + else: + case a.kind + of Iam: + return a.iam_pubkey == b.iam_pubkey and a.iam_signature == b.iam_signature + of PublishNote: + return a.pub_topic == b.pub_topic and a.pub_data == b.pub_data + of FetchNote: + return a.fetch_topic == b.fetch_topic + of SendData: + return a.send_dst == b.send_dst and a.send_val == b.send_val + +#-------------------------------------------------------------- +# serialization +# +# TODO: consider if this should belong in a different file +#-------------------------------------------------------------- + +proc nsencode*(x: string): string = + ## Encode a string as a netstring + $len(x) & ":" & x & "," + +proc nsdecode*(x: string, start: var int = 0): string = + ## Read the netstring from x starting at index `start` + ## start will be moved to the next netstring location + if x.len == 0: + raise ValueError.newException("Empty string is invalid netstring") + var cursor = start + # get length prefix + var expectedLength = 0 + block: + var buf = "" + while true: + let ch = x[cursor] + cursor.inc() + case ch + of '0'..'9': + buf.add(ch) + of ':': + expectedLength = buf.parseInt() + break + else: + raise ValueError.newException("Invalid length character: " & ch & " at position " & $cursor) + + # check for terminal and length + let terminalIdx = cursor + expectedLength + if terminalIdx >= x.len: + raise ValueError.newException("Netstring incomplete") + + let terminalCh = x[terminalIdx] + if terminalCh notin {',','\n'}: + raise ValueError.newException("Invalid terminal character: " & terminalCh) + + result = x[cursor..(cursor + expectedLength - 1)] + start = terminalIdx + 1 + + +proc serialize*(kind: MessageKind): char = + case kind + of Who: '?' + of Okay: '+' + of Error: '-' + of Note: 'n' + of Data: 'd' + +proc deserialize*(kind: typedesc[MessageKind], val: char): MessageKind = + case val + of '?': Who + of '+': Okay + of '-': Error + of 'n': Note + of 'd': Data + else: raise ValueError.newException("Unknown MessageKind: " & val) + +proc serialize*(kind: CommandKind): char = + case kind: + of Iam: 'i' + of PublishNote: 'p' + of FetchNote: 'f' + of SendData: 's' + +proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = + case val: + of 'i': Iam + of 'p': PublishNote + of 'f': FetchNote + of 's': SendData + else: raise ValueError.newException("Unknown CommandKind: " & val) + +proc serialize*(err: ErrorCode): char = + case err + of Generic: '0' + of TooLarge: '1' + +proc deserialize*(typ: typedesc[ErrorCode], ch: char): ErrorCode = + case ch + of '0': Generic + of '1': TooLarge + else: raise ValueError.newException("Unknown ErrorCode: " & ch) + +proc serialize*(msg: RelayMessage): string = + result &= msg.kind.serialize() + case msg.kind + of Who: + result &= msg.who_challenge + of Okay: + result &= msg.ok_cmd.serialize() + of Error: + result &= msg.err_cmd.serialize() + result &= msg.err_code.serialize() + result &= msg.err_message + of Note: + result &= msg.note_topic.nsencode + result &= msg.note_data.nsencode + of Data: + result &= msg.data_src.string.nsencode + result &= msg.data_val.nsencode + +proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = + if s.len == 0: + raise ValueError.newException("Empty RelayMessage") + let kind = MessageKind.deserialize(s[0]) + case kind + of Who: + return RelayMessage(kind: Who, who_challenge: s[1..^1]) + of Okay: + return RelayMessage(kind: Okay, ok_cmd: CommandKind.deserialize(s[1])) + of Error: + return RelayMessage( + kind: Error, + err_cmd: CommandKind.deserialize(s[1]), + err_code: ErrorCode.deserialize(s[2]), + err_message: s[3..^1] + ) + of Note: + var idx = 1 + let note_topic = s.nsdecode(idx) + let note_data = s.nsdecode(idx) + return RelayMessage( + kind: Note, + note_topic: note_topic, + note_data: note_data, + ) + of Data: + var idx = 1 + let data_src = s.nsdecode(idx).PublicKey + let data_val = s.nsdecode(idx) + return RelayMessage( + kind: Data, + data_src: data_src, + data_val: data_val, + ) + +proc serialize*(cmd: RelayCommand): string = + result &= cmd.kind.serialize + +proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = + if s.len == 0: + raise ValueError.newException("Empty RelayCommand") + let kind = CommandKind.deserialize(s[0]) + case kind + of Iam: discard + of PublishNote: discard + of FetchNote: discard + of SendData: discard diff --git a/src/proto2.nim b/src/proto2.nim index a659adb..c16c434 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -107,46 +107,11 @@ proc updateSchema*(db: DbConn) = id INTEGER PRIMARY KEY AUTOINCREMENT, created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, src TEXT NOT NULL, + dst TEXT NOT NULL, data BLOB NOT NULL )""") db.exec(sql"CREATE INDEX message_created ON message(created)") - - # message_route - db.exec(sql"""CREATE TABLE message_route ( - id INTEGER PRIMARY KEY, - message_id INTEGER REFERENCES message(id) ON DELETE CASCADE, - dst TEXT NOT NULL - )""") - db.exec(sql"CREATE INDEX message_route_dst ON message_route(dst)") - - # message_dst - db.exec(sql"""CREATE VIEW message_dst AS - SELECT - m.id, - m.created, - m.src, - r.dst, - m.data, - r.id AS route_id - FROM - message_route AS r - JOIN message AS m - ON m.id = r.message_id - """) - - # auto-delete message orphans - db.exec(sql""" - CREATE TRIGGER IF NOT EXISTS delete_orphaned_message - AFTER DELETE ON message_route - FOR EACH ROW - BEGIN - DELETE FROM message - WHERE id = OLD.message_id - AND NOT EXISTS ( - SELECT 1 FROM message_route WHERE message_id = OLD.message_id - ); - END; - """) + db.exec(sql"CREATE INDEX message_dst ON message(dst)") # known_pubkey db.exec(sql"""CREATE TABLE known_pubkey ( @@ -304,8 +269,8 @@ proc delExpiredMessages(relay: Relay) = proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = let orow = relay.db.getRow(sql""" - SELECT src, data, route_id - FROM message_dst + SELECT src, data, id + FROM message WHERE dst = ? ORDER BY @@ -319,7 +284,7 @@ proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = data_src: PublicKey.fromDB(row[0].s), data_val: row[1].b.string, )) - relay.db.exec(sql"DELETE FROM message_route WHERE id=?", row[2].i) + relay.db.exec(sql"DELETE FROM message WHERE id=?", row[2].i) #------------------------------------------------------------------- # relay command handling @@ -391,31 +356,24 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay # the note isn't here yet relay.addNoteSub(cmd.fetch_topic, conn.pubkey) of SendData: - if cmd.data.len > RELAY_MAX_MESSAGE_SIZE: + if cmd.send_val.len > RELAY_MAX_MESSAGE_SIZE: conn.sendError("Data too long", cmd.kind, TooLarge) else: - var message_id: Option[int64] - for dst in cmd.dst: - if relay.clients.hasKey(dst): - # dst is online - var other_conn = relay.clients[dst] - other_conn.sendMessage(RelayMessage( - kind: Data, - data_src: conn.pubkey, - data_val: cmd.data, - )) + if relay.clients.hasKey(cmd.send_dst): + # dst is online + var other_conn = relay.clients[cmd.send_dst] + other_conn.sendMessage(RelayMessage( + kind: Data, + data_src: conn.pubkey, + data_val: cmd.send_val, + )) + else: + # dst is offline + if relay.isKnown(cmd.send_dst): + relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?, ?, ?)", + conn.pubkey.toDB, cmd.send_dst.toDB, cmd.send_val.DbBlob) else: - # dst is offline - if relay.isKnown(dst): - if message_id.isNone: - # first one of the recipients that's offline - let rowid = relay.db.insertID(sql"INSERT INTO message (src, data) VALUES (?, ?)", - conn.pubkey.toDB, cmd.data.DbBlob) - message_id = some(rowid) - relay.db.exec(sql"INSERT INTO message_route (message_id, dst) VALUES (?, ?)", - message_id.get(), dst.toDB) - else: - discard "silently drop the message" + discard "silently drop the message" #------------------------------------------------------------------- # Utilities #------------------------------------------------------------------- diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 5b145da..c46498c 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -303,8 +303,8 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: @[bob.pk], - data: "hel\x00lo", + send_dst: bob.pk, + send_val: "hel\x00lo", )) let data = bob.pop(Data) @@ -319,8 +319,8 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: @[bob1.pk], - data: "hel\x00lo", + send_dst: bob1.pk, + send_val: "hel\x00lo", )) var bob2 = relay.authenticatedConn(bob1.keys) @@ -334,8 +334,8 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: @[alice.pk], - data: "a".repeat(RELAY_MAX_MESSAGE_SIZE + 1), + send_dst: alice.pk, + send_val: "a".repeat(RELAY_MAX_MESSAGE_SIZE + 1), )) let err = alice.pop(Error) check err.err_code == TooLarge @@ -348,8 +348,8 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: @[bobkeys.pk], - data: "a", + send_dst: bobkeys.pk, + send_val: "a", )) check alice.msgCount == 0 @@ -366,8 +366,8 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: @[bob.pk], - data: "a", + send_dst: bob.pk, + send_val: "a", )) check alice.msgCount == 0 @@ -382,46 +382,10 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - dst: @[bob.pk], - data: "hello", + send_dst: bob.pk, + send_val: "hello", )) skewTime(RELAY_MESSAGE_DURATION + 1) var bob2 = relay.authenticatedConn(bob.keys) check bob2.msgCount == 0 - - test "multiple dst": - let relay = testRelay() - var alice = relay.authenticatedConn() - var bob = relay.authenticatedConn() - relay.disconnect(bob) - - var carl = relay.authenticatedConn() - relay.handleCommand(alice, RelayCommand( - kind: SendData, - dst: @[bob.pk], - data: "first", - )) - relay.handleCommand(alice, RelayCommand( - kind: SendData, - dst: @[bob.pk, carl.pk], - data: "second", - )) - relay.handleCommand(alice, RelayCommand( - kind: SendData, - dst: @[carl.pk, bob.pk], - data: "third", - )) - relay.handleCommand(alice, RelayCommand( - kind: SendData, - dst: @[carl.pk], - data: "fourth", - )) - var bob2 = relay.authenticatedConn(bob.keys) - check bob2.pop(Data).data_val == "first" - check bob2.pop(Data).data_val == "second" - check bob2.pop(Data).data_val == "third" - check carl.pop(Data).data_val == "second" - check carl.pop(Data).data_val == "third" - check carl.pop(Data).data_val == "fourth" - check relay.db.getRow(sql"SELECT id FROM message").isNone() diff --git a/tests/tserde2.nim b/tests/tserde2.nim new file mode 100644 index 0000000..18c4565 --- /dev/null +++ b/tests/tserde2.nim @@ -0,0 +1,34 @@ +import std/unittest + +import proto2 + +test "MessageKind": + for kind in low(MessageKind)..high(MessageKind): + check MessageKind.deserialize(kind.serialize()) == kind + +test "CommandKind": + for kind in low(CommandKind)..high(CommandKind): + check CommandKind.deserialize(kind.serialize()) == kind + +test "RelayMessage": + for kind in low(MessageKind)..high(MessageKind): + let example = case kind + of Who: RelayMessage(kind: Who, who_challenge: "test") + of Okay: RelayMessage(kind: Okay, ok_cmd: SendData) + of Error: RelayMessage(kind: Error, err_cmd: SendData, err_code: TooLarge, err_message: "foo") + of Note: RelayMessage(kind: Note, note_topic: "something", note_data: "data") + of Data: RelayMessage(kind: Data, data_src: "hey".PublicKey, data_val: "foo") + let serialized = example.serialize() + checkpoint "serialized: " & serialized + check RelayMessage.deserialize(serialized) == example + +test "RelayCommand": + for kind in low(CommandKind)..high(CommandKind): + let example = case kind + of Iam: RelayCommand(kind: Iam, iam_pubkey: "hey".PublicKey, iam_signature: "foo") + of PublishNote: RelayCommand(kind: PublishNote, pub_topic: "topic", pub_data: "data") + of FetchNote: RelayCommand(kind: FetchNote, fetch_topic: "topic") + of SendData: RelayCommand(kind: SendData, dst: @["one".PublicKey, "two".PublicKey], data: "data") + let serialized = example.serialize() + checkpoint "serialized: " & serialized + check RelayCommand.deserialize(serialized) == example \ No newline at end of file From fe21a759875c0497fb01e615f70566ee4771c048 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 13:17:52 -0400 Subject: [PATCH 06/46] Chunks work --- src/objs.nim | 96 ++++++++++++++++++++---- src/proto2.nim | 127 +++++++++++++++++++++++++++---- tests/tproto2.nim | 186 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 379 insertions(+), 30 deletions(-) diff --git a/src/objs.nim b/src/objs.nim index 47cbfac..65b0951 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -7,9 +7,10 @@ ## This file should be kept free of dependencies other than the stdlib ## as it's meant to be referenced by outside libraries. -import std/strformat import std/base64 import std/hashes +import std/options +import std/strformat import std/strutils type @@ -22,6 +23,7 @@ type Error Note Data + Chunk ErrorCode* = enum Generic = 0 @@ -43,12 +45,18 @@ type of Data: data_src*: PublicKey data_val*: string + of Chunk: + chunk_src*: PublicKey + chunk_key*: string + chunk_val*: Option[string] CommandKind* = enum Iam PublishNote FetchNote SendData + StoreChunk + GetChunks RelayCommand* = object case kind*: CommandKind @@ -63,20 +71,36 @@ type of SendData: send_dst*: PublicKey send_val*: string + of StoreChunk: + chunk_dst*: seq[PublicKey] + chunk_key*: string + chunk_val*: string + of GetChunks: + chunk_src*: PublicKey + chunk_keys*: seq[string] const RELAY_MAX_TOPIC_SIZE* = 512 RELAY_MAX_NOTE_SIZE* = 4096 RELAY_NOTE_DURATION* = 5 * 24 * 60 * 60 - RELAY_MAX_MESSAGE_SIZE* = 100_000 + RELAY_MAX_MESSAGE_SIZE* = 4096 + RELAY_MAX_CHUNK_KEY_SIZE* = 4096 + RELAY_MAX_CHUNK_SIZE* = 65536 + RELAY_MAX_CHUNK_DSTS* = 32 RELAY_MESSAGE_DURATION* = 30 * 24 * 60 * 60 RELAY_PUBKEY_MEMORY_SECONDS* = 60 * 24 * 60 * 60 -template b64encode(x: string): string = base64.encode(x) +const + nicestart = 'a' # '!' + niceend = 'z' # '~' + nicesize = ord(niceend) - ord(nicestart) -proc `$`*(k: PublicKey): string = b64encode(k.string) -proc hash*(p: PublicKey): Hash {.borrow.} -proc `==`*(a,b: PublicKey): bool {.borrow.} +proc nice*(s: string): string = + for c in s: + case c + of {'0'..'9', 'a'..'z', 'A'..'Z', ' '}: result.add c + else: + result.add chr(ord(c) mod nicesize + ord(nicestart)) proc abbr*(s: string, size = 6): string = if s.len > size: @@ -84,21 +108,36 @@ proc abbr*(s: string, size = 6): string = else: result.add(s) -proc abbr*(a: PublicKey): string = abbr($a) +proc nicelong*(s: string): string = + result = $s.len & ":" & s.nice.abbr & "," + +proc nicelong*(o: Option[string]): string = + if o.isNone: + result = "none" + else: + result = o.get().nicelong() + +proc nice*(k: PublicKey): string = nice(k.string) +proc hash*(p: PublicKey): Hash {.borrow.} +proc `==`*(a,b: PublicKey): bool {.borrow.} + +proc abbr*(a: PublicKey): string = abbr(a.nice) proc `$`*(msg: RelayMessage): string = result.add $msg.kind & "(" case msg.kind of Who: - result.add "challenge=" & b64encode(msg.who_challenge).abbr + result.add "challenge=" & msg.who_challenge.nicelong of Okay: result.add &"cmd={msg.ok_cmd}" of Error: - result.add &"cmd={msg.err_cmd} code={msg.err_code} msg={msg.err_message}" + result.add &"cmd={msg.err_cmd} code={msg.err_code} msg={msg.err_message.nice}" of Note: - result.add &"'{msg.note_topic}' {msg.note_data.b64encode.abbr} ({msg.note_data.len})" + result.add &"'{msg.note_topic.nice}' val={msg.note_data.nicelong}" of Data: - result.add &"src={msg.data_src.abbr} data={msg.data_val.b64encode.abbr} ({msg.data_val.len})" + result.add &"{msg.data_src.nice.abbr} val={msg.data_val.nicelong}" + of Chunk: + result.add &"{msg.chunk_src.nice.abbr} {msg.chunk_key.nice.abbr}={msg.chunk_val.nicelong}" result.add ")" proc `==`*(a, b: RelayMessage): bool = @@ -116,18 +155,30 @@ proc `==`*(a, b: RelayMessage): bool = return a.note_data == b.note_data and a.note_topic == b.note_topic of Data: return a.data_src == b.data_src and a.data_val == b.data_val + of Chunk: + return a.chunk_src == b.chunk_src and a.chunk_key == b.chunk_key and a.chunk_val == b.chunk_val proc `$`*(cmd: RelayCommand): string = result.add $cmd.kind & "(" case cmd.kind of Iam: - result.add &"{cmd.iam_pubkey.abbr} sig={cmd.iam_signature.b64encode.abbr}" + result.add &"{cmd.iam_pubkey.nice.abbr} sig={cmd.iam_signature.nicelong}" of PublishNote: - result.add &"'{cmd.pub_topic}' data={cmd.pub_data.b64encode}" + result.add &"'{cmd.pub_topic.nice.abbr}' val={cmd.pub_data.nicelong}" of FetchNote: - result.add &"'{cmd.fetch_topic}'" + result.add &"'{cmd.fetch_topic.nice.abbr}'" of SendData: - result.add &"{cmd.send_dst.abbr} data={cmd.send_val.b64encode.abbr} ({cmd.send_val.len})" + result.add &"{cmd.send_dst.nice.abbr} val={cmd.send_val.nicelong}" + of StoreChunk: + result.add &"{cmd.chunk_key.nice.abbr}={cmd.chunk_val.nicelong} dst=[" + for dst in cmd.chunk_dst: + result.add dst.nice.abbr & ", " + result.add "]" + of GetChunks: + result.add &"{cmd.chunk_src.nice.abbr} keys=[" + for key in cmd.chunk_keys: + result.add key.nice.abbr & ", " + result.add "]" result.add ")" proc `==`*(a, b: RelayCommand): bool = @@ -143,6 +194,10 @@ proc `==`*(a, b: RelayCommand): bool = return a.fetch_topic == b.fetch_topic of SendData: return a.send_dst == b.send_dst and a.send_val == b.send_val + of StoreChunk: + return a.chunk_dst == b.chunk_dst and a.chunk_key == b.chunk_key and a.chunk_val == b.chunk_val + of GetChunks: + return a.chunk_src == b.chunk_src and a.chunk_keys == b.chunk_keys #-------------------------------------------------------------- # serialization @@ -196,6 +251,7 @@ proc serialize*(kind: MessageKind): char = of Error: '-' of Note: 'n' of Data: 'd' + of Chunk: 'k' proc deserialize*(kind: typedesc[MessageKind], val: char): MessageKind = case val @@ -204,6 +260,7 @@ proc deserialize*(kind: typedesc[MessageKind], val: char): MessageKind = of '-': Error of 'n': Note of 'd': Data + of 'k': Chunk else: raise ValueError.newException("Unknown MessageKind: " & val) proc serialize*(kind: CommandKind): char = @@ -212,6 +269,8 @@ proc serialize*(kind: CommandKind): char = of PublishNote: 'p' of FetchNote: 'f' of SendData: 's' + of StoreChunk: 'c' + of GetChunks: 'g' proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = case val: @@ -219,6 +278,8 @@ proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = of 'p': PublishNote of 'f': FetchNote of 's': SendData + of 'c': StoreChunk + of 'g': GetChunks else: raise ValueError.newException("Unknown CommandKind: " & val) proc serialize*(err: ErrorCode): char = @@ -249,6 +310,9 @@ proc serialize*(msg: RelayMessage): string = of Data: result &= msg.data_src.string.nsencode result &= msg.data_val.nsencode + of Chunk: + discard + proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = if s.len == 0: @@ -284,6 +348,8 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = data_src: data_src, data_val: data_val, ) + of Chunk: + discard proc serialize*(cmd: RelayCommand): string = result &= cmd.kind.serialize diff --git a/src/proto2.nim b/src/proto2.nim index c16c434..2a31bbd 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -32,6 +32,7 @@ type sender*: T pubkey*: PublicKey ## The authenticated pubkey challenge: string + relay*: Relay[T] when TESTMODE: var TIME_SKEW = 0 @@ -54,14 +55,11 @@ func strval*(dbval: sqlite.DbValue): string = else: raise ValueError.newException("Can't get string from " & $dbval.kind) -proc toDB*(p: PublicKey): string = - base64.encode(p.string) - -proc fromDB*(t: typedesc[PublicKey], v: string): PublicKey = - base64.decode(v).PublicKey +proc dbValue*(p: PublicKey): DbValue = + dbValue(p.string.DbBlob) proc fromDB*(t: typedesc[PublicKey], v: DbBlob): PublicKey = - base64.decode(v.string).PublicKey + v.string.PublicKey template patch(db: untyped, applied: seq[string], name: string, body: untyped): untyped = block: @@ -80,6 +78,8 @@ template patch(db: untyped, applied: seq[string], name: string, body: untyped): debug name, " - applied" proc updateSchema*(db: DbConn) = + db.exec(sql"PRAGMA foreign_keys = ON") + ## Upgrade the schema db.exec(sql"""CREATE TABLE IF NOT EXISTS _schema_patches ( id INTEGER PRIMARY KEY, @@ -113,6 +113,23 @@ proc updateSchema*(db: DbConn) = db.exec(sql"CREATE INDEX message_created ON message(created)") db.exec(sql"CREATE INDEX message_dst ON message(dst)") + # chunks + db.exec(sql"""CREATE TABLE chunk ( + src TEXT NOT NULL, + key TEXT NOT NULL, + last_used TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + val BLOB NOT NULL, + PRIMARY KEY (src, key) + )""") + db.exec(sql"CREATE INDEX chunk_last_used ON chunk(last_used)") + db.exec(sql"""CREATE TABLE chunk_dst ( + src TEXT NOT NULL, + key TEXT NOT NULL, + dst TEXT NOT NULL, + PRIMARY KEY (src, key, dst), + FOREIGN KEY (src, key) REFERENCES chunk(src, key) ON DELETE CASCADE + )""") + # known_pubkey db.exec(sql"""CREATE TABLE known_pubkey ( pubkey TEXT PRIMARY KEY, @@ -126,7 +143,7 @@ proc updateSchema*(db: DbConn) = pubkey TEXT NOT NULL )""") db.exec(sql"CREATE INDEX note_sub_pubkey ON note_sub(pubkey)") - db.exec(sql"PRAGMA foreign_keys = ON") + #------------------------------------------------------------------- # Relay code @@ -181,9 +198,10 @@ proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = kind: Who, who_challenge: result.challenge, )) + result.relay = relay proc disconnect*[T](relay: Relay[T], conn: RelayConnection[T]) = - relay.db.exec(sql"DELETE FROM note_sub WHERE pubkey=?", conn.pubkey.toDB) + relay.db.exec(sql"DELETE FROM note_sub WHERE pubkey=?", conn.pubkey) relay.clients.del(conn.pubkey) info &"[{conn.pubkey.abbr}] disconnected" @@ -202,7 +220,7 @@ proc delExpiredNotes(relay: Relay) = proc addNoteSub(relay: Relay, topic: string, pubkey: PublicKey) = ## Record that a pubkey is subscribed to a topic try: - relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic.DbBlob, pubkey.toDB) + relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic.DbBlob, pubkey) info &"[{pubkey.abbr}] sub {topic}" except: raise ValueError.newException("Topic already subscribed") @@ -212,7 +230,7 @@ proc getNoteSub(relay: Relay, topic: string): Option[PublicKey] = relay.delExpiredNotes() let orow = relay.db.getRow(sql"SELECT pubkey FROM note_sub WHERE topic = ?", topic.DbBlob) if orow.isSome: - return some(PublicKey.fromDB(orow.get()[0].s)) + return some(PublicKey.fromDB(orow.get()[0].b)) proc popNote(relay: Relay, topic: string): Option[string] = let db = relay.db @@ -250,13 +268,17 @@ proc forgetOldPubkeys(relay: Relay) = relay.db.exec(sql"DELETE FROM known_pubkey WHERE last_seen <= datetime('now', ?)", offstring) proc rememberPubkey(relay: Relay, pubkey: PublicKey) = + let offset = when TESTMODE: + $TIME_SKEW & " seconds" + else: + "0 seconds" relay.db.exec(sql""" INSERT OR REPLACE INTO known_pubkey (pubkey, last_seen) - VALUES (?, CURRENT_TIMESTAMP)""", pubkey.toDB) + VALUES (?, datetime('now', ?))""", pubkey, offset) proc isKnown(relay: Relay, pubkey: PublicKey): bool = relay.forgetOldPubkeys() - let orow = relay.db.getRow(sql"SELECT last_seen FROM known_pubkey WHERE pubkey = ?", pubkey.toDB) + let orow = relay.db.getRow(sql"SELECT last_seen FROM known_pubkey WHERE pubkey = ?", pubkey) return orow.isSome() proc delExpiredMessages(relay: Relay) = @@ -276,16 +298,26 @@ proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = ORDER BY created ASC, id ASC - LIMIT 1""", dst.toDB) + LIMIT 1""", dst) if orow.isSome: let row = orow.get() result = some(RelayMessage( kind: Data, - data_src: PublicKey.fromDB(row[0].s), + data_src: PublicKey.fromDB(row[0].b), data_val: row[1].b.string, )) relay.db.exec(sql"DELETE FROM message WHERE id=?", row[2].i) +proc delExpiredChunks(relay: Relay) = + let offset = when TESTMODE: + -RELAY_MESSAGE_DURATION + TIME_SKEW + else: + -RELAY_MESSAGE_DURATION + let offstring = &"{offset} seconds" + echo "FRANK delExpiredChunks ", offstring + relay.db.exec(sql"DELETE FROM chunk WHERE last_used <= datetime('now', ?)", offstring) + + #------------------------------------------------------------------- # relay command handling #------------------------------------------------------------------- @@ -371,9 +403,74 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay # dst is offline if relay.isKnown(cmd.send_dst): relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?, ?, ?)", - conn.pubkey.toDB, cmd.send_dst.toDB, cmd.send_val.DbBlob) + conn.pubkey, cmd.send_dst, cmd.send_val.DbBlob) else: discard "silently drop the message" + of StoreChunk: + if cmd.chunk_key.len > RELAY_MAX_CHUNK_KEY_SIZE: + conn.sendError("Key too long", cmd.kind, TooLarge) + elif cmd.chunk_val.len > RELAY_MAX_CHUNK_SIZE: + conn.sendError("Value too long", cmd.kind, TooLarge) + elif cmd.chunk_dst.len > RELAY_MAX_CHUNK_DSTS: + conn.sendError("Too many recipients", cmd.kind, TooLarge) + else: + relay.db.exec(sql"BEGIN") + try: + relay.db.exec(sql"DELETE FROM chunk_dst WHERE src=? AND key=?", conn.pubkey, cmd.chunk_key.DbBlob) + let offset = when TESTMODE: + $TIME_SKEW & " seconds" + else: + "0 seconds" + echo "FRANK offset: ", offset + relay.db.exec(sql""" + INSERT OR REPLACE INTO chunk (last_used, src, key, val) + VALUES (datetime('now', ?), ?, ?, ?) + """, offset, conn.pubkey, cmd.chunk_key.DbBlob, cmd.chunk_val.DbBlob) + var dsts: seq[PublicKey] + dsts.add(cmd.chunk_dst) + if conn.pubkey notin dsts: + dsts.add(conn.pubkey) + for dst in dsts: + relay.db.exec(sql"INSERT INTO chunk_dst (src, key, dst) VALUES (?, ?, ?)", + conn.pubkey, cmd.chunk_key.DbBlob, dst) + relay.db.exec(sql"COMMIT") + except: + relay.db.exec(sql"ROLLBACK") + of GetChunks: + for key in cmd.chunk_keys: + if key.len > RELAY_MAX_CHUNK_KEY_SIZE: + conn.sendError("Key too long", cmd.kind, TooLarge) + return + relay.delExpiredChunks() + for key in cmd.chunk_keys: + let orow = relay.db.getRow(sql""" + SELECT + c.val + FROM + chunk_dst AS d + JOIN chunk AS c + ON d.src = c.src + AND d.key = c.key + WHERE + d.src = ? + AND d.key = ? + AND d.dst = ? + """, cmd.chunk_src, key.DbBlob, conn.pubkey) + if orow.isSome: + let row = orow.get() + conn.sendMessage(RelayMessage( + kind: Chunk, + chunk_src: cmd.chunk_src, + chunk_key: key, + chunk_val: some(row[0].b.string), + )) + else: + conn.sendMessage(RelayMessage( + kind: Chunk, + chunk_src: cmd.chunk_src, + chunk_key: key, + chunk_val: none[string](), + )) #------------------------------------------------------------------- # Utilities #------------------------------------------------------------------- diff --git a/tests/tproto2.nim b/tests/tproto2.nim index c46498c..5860f30 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -389,3 +389,189 @@ suite "data": skewTime(RELAY_MESSAGE_DURATION + 1) var bob2 = relay.authenticatedConn(bob.keys) check bob2.msgCount == 0 + + +proc storeChunk(conn: var RelayConnection[TestClient], key: string, val: string, dst = newSeq[PublicKey]()) = + conn.relay.handleCommand(conn, RelayCommand( + kind: StoreChunk, + chunk_dst: dst, + chunk_key: key, + chunk_val: val, + )) + +proc getChunk(conn: var RelayConnection[TestClient], src: var RelayConnection[TestClient], key: string): Option[string] = + conn.relay.handleCommand(conn, RelayCommand( + kind: GetChunks, + chunk_src: src.pk, + chunk_keys: @[key], + )) + let chunk = conn.pop(Chunk) + return chunk.chunk_val + +suite "store": + + test "basic": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: @[bob.pk], + chunk_key: "key1", + chunk_val: "\x00data1", + )) + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: @[bob.pk], + chunk_key: "key2", + chunk_val: "data2", + )) + check bob.msgCount == 0 + + relay.handleCommand(bob, RelayCommand( + kind: GetChunks, + chunk_src: alice.pk, + chunk_keys: @["key1", "key2"], + )) + block: + let chunk = bob.pop(Chunk) + check chunk.chunk_src == alice.pk + check chunk.chunk_key == "key1" + check chunk.chunk_val.get() == "\x00data1" + block: + let chunk = bob.pop(Chunk) + check chunk.chunk_src == alice.pk + check chunk.chunk_key == "key2" + check chunk.chunk_val.get() == "data2" + + relay.handleCommand(alice, RelayCommand( + kind: GetChunks, + chunk_src: alice.pk, + chunk_keys: @["key2"], + )) + block: + let chunk = alice.pop(Chunk) + check chunk.chunk_src == alice.pk + check chunk.chunk_key == "key2" + check chunk.chunk_val.get() == "data2" + + test "overwrite": + let relay = testRelay() + var alice = relay.authenticatedConn() + alice.storeChunk("key", "first") + alice.storeChunk("key", "second") + check alice.getChunk(alice, "key").get() == "second" + + test "multiple dst": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + var carl = relay.authenticatedConn() + alice.storeChunk("key", "val", @[bob.pk, carl.pk]) + check bob.getChunk(alice, "key").get() == "val" + check carl.getChunk(alice, "key").get() == "val" + + test "dne": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: GetChunks, + chunk_src: alice.pk, + chunk_keys: @["dne"], + )) + block: + let chunk = alice.pop(Chunk) + check chunk.chunk_src == alice.pk + check chunk.chunk_key == "dne" + check chunk.chunk_val.isNone() + + test "only dst allowed": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + alice.storeChunk("key", "first") + check bob.getChunk(alice, "key").isNone() + + test "expiration": + let relay = testRelay() + var alice = relay.authenticatedConn() + alice.storeChunk("key", "foo") + skewTime(RELAY_MESSAGE_DURATION + 1) + check alice.getChunk(alice, "key").isNone() + + test "expiration update": + let relay = testRelay() + var alice = relay.authenticatedConn() + alice.storeChunk("key", "foo") + skewTime(RELAY_MESSAGE_DURATION - 1) + alice.storeChunk("key", "foo") + skewTime(3) + check alice.getChunk(alice, "key").get() == "foo" + + test "remove dst": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + var sam = relay.authenticatedConn() + alice.storeChunk("key", "first", @[bob.pk, sam.pk]) + check bob.getChunk(alice, "key").get() == "first" + check sam.getChunk(alice, "key").get() == "first" + alice.storeChunk("key", "first", @[bob.pk]) + check bob.getChunk(alice, "key").get() == "first" + check sam.getChunk(alice, "key").isNone() + + test "max key len": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: @[], + chunk_key: "a".repeat(RELAY_MAX_CHUNK_KEY_SIZE + 1), + chunk_val: "data1", + )) + let err = alice.pop(Error) + check err.err_cmd == StoreChunk + check err.err_code == TooLarge + + test "max val len": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: @[], + chunk_key: "a", + chunk_val: "a".repeat(RELAY_MAX_CHUNK_SIZE + 1), + )) + let err = alice.pop(Error) + check err.err_cmd == StoreChunk + check err.err_code == TooLarge + + test "max key len get": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: GetChunks, + chunk_src: alice.pk, + chunk_keys: @["a".repeat(RELAY_MAX_CHUNK_KEY_SIZE + 1)], + )) + let err = alice.pop(Error) + check err.err_cmd == GetChunks + check err.err_code == TooLarge + + test "max dst.len": + let relay = testRelay() + var alice = relay.authenticatedConn() + var dsts: seq[PublicKey] + for i in 0..(RELAY_MAX_CHUNK_DSTS+1): + var conn = relay.authenticatedConn() + dsts.add(conn.pk) + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: dsts, + chunk_key: "a", + chunk_val: "b", + )) + let err = alice.pop(Error) + check err.err_cmd == StoreChunk + check err.err_code == TooLarge From f924394da53992cb480a6d1fcd63d34da668cb82 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 13:23:04 -0400 Subject: [PATCH 07/46] Remove known-pubkey concept --- src/proto2.nim | 52 +++++++++-------------------------------------- tests/tproto2.nim | 35 +------------------------------ 2 files changed, 11 insertions(+), 76 deletions(-) diff --git a/src/proto2.nim b/src/proto2.nim index 2a31bbd..0dee190 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -70,7 +70,7 @@ template patch(db: untyped, applied: seq[string], name: string, body: untyped): body db.exec(sql"INSERT INTO _schema_patches (name) VALUES (?)", name) db.exec(sql"COMMIT") - except: + except CatchableError: error name, " - error applying patch: " & getCurrentExceptionMsg() db.exec(sql"ROLLBACK") raise @@ -129,13 +129,6 @@ proc updateSchema*(db: DbConn) = PRIMARY KEY (src, key, dst), FOREIGN KEY (src, key) REFERENCES chunk(src, key) ON DELETE CASCADE )""") - - # known_pubkey - db.exec(sql"""CREATE TABLE known_pubkey ( - pubkey TEXT PRIMARY KEY, - last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP - )""") - db.exec(sql"CREATE INDEX known_pubkey_last_seen ON known_pubkey(last_seen)") #----------- in-memory stuff db.exec(sql"""CREATE TEMPORARY TABLE note_sub ( @@ -222,7 +215,7 @@ proc addNoteSub(relay: Relay, topic: string, pubkey: PublicKey) = try: relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic.DbBlob, pubkey) info &"[{pubkey.abbr}] sub {topic}" - except: + except CatchableError: raise ValueError.newException("Topic already subscribed") proc getNoteSub(relay: Relay, topic: string): Option[PublicKey] = @@ -246,7 +239,7 @@ proc popNote(relay: Relay, topic: string): Option[string] = else: debug &"[note] dne {topic}" db.exec(sql"COMMIT") - except: + except CatchableError: warn &"[note] error " & getCurrentExceptionMsg() db.exec(sql"ROLLBACK") @@ -259,28 +252,6 @@ proc delNoteSub(relay: Relay, topic: string) = # send/receive data #------------------------------------------------------------------- -proc forgetOldPubkeys(relay: Relay) = - let offset = when TESTMODE: - -RELAY_PUBKEY_MEMORY_SECONDS + TIME_SKEW - else: - -RELAY_PUBKEY_MEMORY_SECONDS - let offstring = &"{offset} seconds" - relay.db.exec(sql"DELETE FROM known_pubkey WHERE last_seen <= datetime('now', ?)", offstring) - -proc rememberPubkey(relay: Relay, pubkey: PublicKey) = - let offset = when TESTMODE: - $TIME_SKEW & " seconds" - else: - "0 seconds" - relay.db.exec(sql""" - INSERT OR REPLACE INTO known_pubkey (pubkey, last_seen) - VALUES (?, datetime('now', ?))""", pubkey, offset) - -proc isKnown(relay: Relay, pubkey: PublicKey): bool = - relay.forgetOldPubkeys() - let orow = relay.db.getRow(sql"SELECT last_seen FROM known_pubkey WHERE pubkey = ?", pubkey) - return orow.isSome() - proc delExpiredMessages(relay: Relay) = let offset = when TESTMODE: -RELAY_MESSAGE_DURATION + TIME_SKEW @@ -314,7 +285,6 @@ proc delExpiredChunks(relay: Relay) = else: -RELAY_MESSAGE_DURATION let offstring = &"{offset} seconds" - echo "FRANK delExpiredChunks ", offstring relay.db.exec(sql"DELETE FROM chunk WHERE last_used <= datetime('now', ?)", offstring) @@ -329,14 +299,16 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay of Iam: try: crypto_sign_verify_detached(cmd.iam_pubkey.string, conn.challenge, cmd.iam_signature) - except: + except SodiumError: conn.sendError("Invalid signature", cmd.kind, Generic) return + except CatchableError: + conn.sendError("Error validating signature", cmd.kind, Generic) + return # successful connection conn.pubkey = cmd.iam_pubkey relay.clients[conn.pubkey] = conn conn.challenge = "" # disable authentication - relay.rememberPubkey(conn.pubkey) info &"[{conn.pubkey.abbr}] connected" conn.sendOkay cmd.kind @@ -401,11 +373,8 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay )) else: # dst is offline - if relay.isKnown(cmd.send_dst): - relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?, ?, ?)", - conn.pubkey, cmd.send_dst, cmd.send_val.DbBlob) - else: - discard "silently drop the message" + relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?, ?, ?)", + conn.pubkey, cmd.send_dst, cmd.send_val.DbBlob) of StoreChunk: if cmd.chunk_key.len > RELAY_MAX_CHUNK_KEY_SIZE: conn.sendError("Key too long", cmd.kind, TooLarge) @@ -421,7 +390,6 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay $TIME_SKEW & " seconds" else: "0 seconds" - echo "FRANK offset: ", offset relay.db.exec(sql""" INSERT OR REPLACE INTO chunk (last_used, src, key, val) VALUES (datetime('now', ?), ?, ?, ?) @@ -434,7 +402,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay relay.db.exec(sql"INSERT INTO chunk_dst (src, key, dst) VALUES (?, ?, ?)", conn.pubkey, cmd.chunk_key.DbBlob, dst) relay.db.exec(sql"COMMIT") - except: + except CatchableError: relay.db.exec(sql"ROLLBACK") of GetChunks: for key in cmd.chunk_keys: diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 5860f30..1032ec4 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -93,7 +93,7 @@ suite "Auth": var alice = relay.initAuth(aclient) let who = alice.pop(Who) check who.who_challenge != "" - echo $who + checkpoint $who checkpoint "iam" let signature = aclient.sk.sign(who.who_challenge) @@ -341,39 +341,6 @@ suite "data": check err.err_code == TooLarge check err.err_cmd == SendData - test "drop unknown key": - let relay = testRelay() - var alice = relay.authenticatedConn() - let bobkeys = genkeys() - - relay.handleCommand(alice, RelayCommand( - kind: SendData, - send_dst: bobkeys.pk, - send_val: "a", - )) - check alice.msgCount == 0 - - var bob = relay.authenticatedConn(bobkeys) - check bob.msgCount == 0 # "Should not have stored the message" - - test "drop forgotten key": - let relay = testRelay() - var bob = relay.authenticatedConn() - relay.disconnect(bob) - - skewTime(RELAY_PUBKEY_MEMORY_SECONDS + 1) - var alice = relay.authenticatedConn() - - relay.handleCommand(alice, RelayCommand( - kind: SendData, - send_dst: bob.pk, - send_val: "a", - )) - check alice.msgCount == 0 - - var bob2 = relay.authenticatedConn(bob.keys) - check bob2.msgCount == 0 # "Should not have stored the message" - test "expiration": let relay = testRelay() var alice = relay.authenticatedConn() From 2a2cb4bcd9e3860a05168c4580106a9ec8bea356 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 14:34:28 -0400 Subject: [PATCH 08/46] Finish serialization/deserialization --- src/objs.nim | 104 +++++++++++++++++++++++++++++++++++++++++++--- tests/tserde2.nim | 24 ++++++++++- 2 files changed, 120 insertions(+), 8 deletions(-) diff --git a/src/objs.nim b/src/objs.nim index 65b0951..4ab947c 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -10,6 +10,7 @@ import std/base64 import std/hashes import std/options +import std/sequtils import std/strformat import std/strutils @@ -118,6 +119,7 @@ proc nicelong*(o: Option[string]): string = result = o.get().nicelong() proc nice*(k: PublicKey): string = nice(k.string) +proc `$`*(k: PublicKey): string = k.nice() proc hash*(p: PublicKey): Hash {.borrow.} proc `==`*(a,b: PublicKey): bool {.borrow.} @@ -293,6 +295,24 @@ proc deserialize*(typ: typedesc[ErrorCode], ch: char): ErrorCode = of '1': TooLarge else: raise ValueError.newException("Unknown ErrorCode: " & ch) +proc serialize*(keys: seq[PublicKey]): string = + for key in keys: + result &= nsencode(key.string) + +proc deserializePubKeys*(val: string): seq[PublicKey] = + var idx = 0 + while idx < val.len: + result.add(val.nsdecode(idx).PublicKey) + +proc serialize*(s: seq[string]): string = + for item in s: + result &= nsencode(item) + +proc deserialize*(typ: typedesc[seq[string]], val: string): seq[string] = + var idx = 0 + while idx < val.len: + result.add(val.nsdecode(idx)) + proc serialize*(msg: RelayMessage): string = result &= msg.kind.serialize() case msg.kind @@ -311,7 +331,10 @@ proc serialize*(msg: RelayMessage): string = result &= msg.data_src.string.nsencode result &= msg.data_val.nsencode of Chunk: - discard + result &= msg.chunk_src.string.nsencode + result &= msg.chunk_key.nsencode + if msg.chunk_val.isSome: + result &= msg.chunk_val.get().nsencode proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = @@ -349,17 +372,86 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = data_val: data_val, ) of Chunk: - discard + var idx = 1 + let chunk_src = s.nsdecode(idx).PublicKey + let chunk_key = s.nsdecode(idx) + let chunk_val = if idx >= s.len: + none[string]() + else: + some(s.nsdecode(idx)) + return RelayMessage( + kind: Chunk, + chunk_src: chunk_src, + chunk_key: chunk_key, + chunk_val: chunk_val, + ) proc serialize*(cmd: RelayCommand): string = result &= cmd.kind.serialize + case cmd.kind + of Iam: + result &= cmd.iam_pubkey.string.nsencode + result &= cmd.iam_signature.nsencode + of PublishNote: + result &= cmd.pub_topic.nsencode + result &= cmd.pub_data.nsencode + of FetchNote: + result &= cmd.fetch_topic.nsencode + of SendData: + result &= cmd.send_dst.string.nsencode + result &= cmd.send_val.nsencode + of StoreChunk: + result &= nsencode(cmd.chunk_dst.serialize()) + result &= cmd.chunk_key.nsencode + result &= cmd.chunk_val.nsencode + of GetChunks: + result &= cmd.chunk_src.string.nsencode + result &= nsencode(cmd.chunk_keys.serialize()) proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = if s.len == 0: raise ValueError.newException("Empty RelayCommand") let kind = CommandKind.deserialize(s[0]) case kind - of Iam: discard - of PublishNote: discard - of FetchNote: discard - of SendData: discard + of Iam: + var idx = 1 + return RelayCommand( + kind: Iam, + iam_pubkey: s.nsdecode(idx).PublicKey, + iam_signature: s.nsdecode(idx), + ) + of PublishNote: + var idx = 1 + return RelayCommand( + kind: PublishNote, + pub_topic: s.nsdecode(idx), + pub_data: s.nsdecode(idx), + ) + of FetchNote: + var idx = 1 + return RelayCommand( + kind: FetchNote, + fetch_topic: s.nsdecode(idx), + ) + of SendData: + var idx = 1 + return RelayCommand( + kind: SendData, + send_dst: s.nsdecode(idx).PublicKey, + send_val: s.nsdecode(idx), + ) + of StoreChunk: + var idx = 1 + return RelayCommand( + kind: StoreChunk, + chunk_dst: deserializePubKeys(s.nsdecode(idx)), + chunk_key: s.nsdecode(idx), + chunk_val: s.nsdecode(idx), + ) + of GetChunks: + var idx = 1 + return RelayCommand( + kind: GetChunks, + chunk_src: s.nsdecode(idx).PublicKey, + chunk_keys: deserialize(seq[string], s.nsdecode(idx)), + ) diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 18c4565..40248c6 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -1,4 +1,5 @@ import std/unittest +import std/options import proto2 @@ -18,6 +19,7 @@ test "RelayMessage": of Error: RelayMessage(kind: Error, err_cmd: SendData, err_code: TooLarge, err_message: "foo") of Note: RelayMessage(kind: Note, note_topic: "something", note_data: "data") of Data: RelayMessage(kind: Data, data_src: "hey".PublicKey, data_val: "foo") + of Chunk: RelayMessage(kind: Chunk, chunk_src: "hey".PublicKey, chunk_key: "key", chunk_val: some("theval")) let serialized = example.serialize() checkpoint "serialized: " & serialized check RelayMessage.deserialize(serialized) == example @@ -28,7 +30,25 @@ test "RelayCommand": of Iam: RelayCommand(kind: Iam, iam_pubkey: "hey".PublicKey, iam_signature: "foo") of PublishNote: RelayCommand(kind: PublishNote, pub_topic: "topic", pub_data: "data") of FetchNote: RelayCommand(kind: FetchNote, fetch_topic: "topic") - of SendData: RelayCommand(kind: SendData, dst: @["one".PublicKey, "two".PublicKey], data: "data") + of SendData: RelayCommand(kind: SendData, send_dst: "one".PublicKey, send_val: "data") + of StoreChunk: RelayCommand( + kind: StoreChunk, + chunk_dst: @["one".PublicKey], + chunk_key: "theky", + chunk_val: "someval" + ) + of GetChunks: RelayCommand(kind: GetChunks, chunk_src: "hey".PublicKey, chunk_keys: @["foo", "bar"]) let serialized = example.serialize() checkpoint "serialized: " & serialized - check RelayCommand.deserialize(serialized) == example \ No newline at end of file + check RelayCommand.deserialize(serialized) == example + +test "Chunk with none": + let chunk = RelayMessage(kind: Chunk, + chunk_src: "foo".PublicKey, + chunk_key: "key", + chunk_val: none[string](), + ) + checkpoint $chunk + let serialized = chunk.serialize() + checkpoint "serialized: " & serialized + check RelayMessage.deserialize(serialized) == chunk From ef760939f9890ebe91c9260c5bd054623031b5db Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 16:36:28 -0400 Subject: [PATCH 09/46] Remove v1 --- .github/workflows/main.yml | 53 +- README.md | 104 +-- bucketsrelay.nimble | 34 +- changes/new-Move-to-store-20251028-153340.md | 1 + config.nims | 2 +- docker/Dockerfile | 22 + docker/config.nims | 13 +- docker/multiuser.Dockerfile | 19 - docker/singleuser.Dockerfile | 18 - nim.cfg | 1 + pkger/deps.json | 13 +- src/bclient.nim | 201 ---- src/brelay.nim | 178 ---- src/bucketsrelay/asyncstdin.nim | 84 -- src/bucketsrelay/client.nim | 246 ----- src/bucketsrelay/common.nim | 59 -- src/bucketsrelay/dbschema.nim | 41 - src/bucketsrelay/httpreq.nim | 88 -- src/bucketsrelay/jwtrsaonly.nim | 253 ----- src/bucketsrelay/licenses.nim | 80 -- src/bucketsrelay/mailer.nim | 56 -- src/bucketsrelay/netstring.nim | 116 --- src/bucketsrelay/proto.nim | 403 -------- src/bucketsrelay/server.nim | 922 ------------------- src/bucketsrelay/stringproto.nim | 108 --- src/objs.nim | 35 +- src/partials/index.mustache | 384 -------- src/server2.nim | 71 ++ src/templates/index.nimja | 0 tests/all.sh | 13 + tests/tbrelay.nim | 75 -- tests/tclient.nim | 173 ---- tests/tlicenses.nim | 89 -- tests/tnetstring.nim | 105 +-- tests/tproto.nim | 271 ------ tests/tserver.nim | 207 ----- tests/tstringproto.nim | 57 -- tests/util.nim | 28 - 38 files changed, 246 insertions(+), 4377 deletions(-) create mode 100644 changes/new-Move-to-store-20251028-153340.md create mode 100644 docker/Dockerfile delete mode 100644 docker/multiuser.Dockerfile delete mode 100644 docker/singleuser.Dockerfile delete mode 100644 src/bclient.nim delete mode 100644 src/brelay.nim delete mode 100644 src/bucketsrelay/asyncstdin.nim delete mode 100644 src/bucketsrelay/client.nim delete mode 100644 src/bucketsrelay/common.nim delete mode 100644 src/bucketsrelay/dbschema.nim delete mode 100644 src/bucketsrelay/httpreq.nim delete mode 100644 src/bucketsrelay/jwtrsaonly.nim delete mode 100644 src/bucketsrelay/licenses.nim delete mode 100644 src/bucketsrelay/mailer.nim delete mode 100644 src/bucketsrelay/netstring.nim delete mode 100644 src/bucketsrelay/proto.nim delete mode 100644 src/bucketsrelay/server.nim delete mode 100644 src/bucketsrelay/stringproto.nim delete mode 100644 src/partials/index.mustache create mode 100644 src/server2.nim create mode 100644 src/templates/index.nimja create mode 100755 tests/all.sh delete mode 100644 tests/tbrelay.nim delete mode 100644 tests/tclient.nim delete mode 100644 tests/tlicenses.nim delete mode 100644 tests/tproto.nim delete mode 100644 tests/tserver.nim delete mode 100644 tests/tstringproto.nim diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9ae502..29bf7ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,57 +21,18 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: version: ${{ matrix.version }} - - name: Update nimble - run: nimble install -y nimble - #---------------------------------------- - # multi user mode (default) - #---------------------------------------- - - name: Install - run: nimble install -y - - name: Build bins - run: nimble multiuserbins + - name: Install pkger + run: nimble install -y https://github.com/iffy/pkger/ + - name: Install deps + run: pkger fetch - name: Run tests run: | - export PATH="${PATH}:$(pwd)/bin" - nimble test - - name: Command-line tests - run: | - export PATH="${PATH}:$(pwd)/bin" - tests/func1.sh - #---------------------------------------- - # single user mode - #---------------------------------------- - - name: Install (single user mode) - run: nimble singleuserbins - - name: Run tests (single user mode) - run: | - export PATH="${PATH}:$(pwd)/bin" - nimble -d:relaysingleusermode test + export SHOW_LOGS=1 + tests/all.sh docker: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - run: docker build --file docker/singleuser.Dockerfile . - - run: docker build --file docker/multiuser.Dockerfile . - - binaries: - strategy: - matrix: - version: - - binary:2.2.4 - os: - - ubuntu-latest - - macos-latest - - windows-latest - steps: - - uses: actions/checkout@v1 - - uses: iffy/install-nim@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - version: ${{ matrix.version }} - - name: Update nimble - run: nimble install -y nimble - - name: Install deps + - run: docker build --file docker/Dockerfile . diff --git a/README.md b/README.md index f59a49a..8c39360 100644 --- a/README.md +++ b/README.md @@ -8,34 +8,30 @@ This repository contains the open source code for the [Buckets](https://www.budg You can use the publicly available relay at -## Quickstart +## Quickstart w/ Docker/Podman -If you want to run the relay on your own computer, do the following: +If you want to run the relay on with docker: -1. Install [Nim](https://nim-lang.org/) -2. Get the code: +1. Get the code: ``` git clone https://github.com/buckets/relay.git buckets-relay.git cd buckets-relay.git ``` -3. Install dependencies +2. Build the image ``` -nimble install https://github.com/iffy/pkger/ -pkger fetch +docker build -f docker/Dockerfile -t buckets/relay . ``` -4. Run the server: +3. Run it: -TODO: - -```sh -nim r src/brelay.nim server +``` +docker run -it --rm -p 8080:8080 buckets/relay ``` -This will launch the relay on the default port. Run with `--help` for more options. +Read `docker/Dockerfile` to get an idea of how to build it yourself if you'd like. ## Security @@ -43,43 +39,15 @@ This will launch the relay on the default port. Run with `--help` for more optio - This relay server can see all traffic, so clients should encrypt data intended for other clients. - Clients should also authenticate each other through the relay and not trust the authentication done by this server. -## Development - -TODO: - -To run the server locally: - -```sh -nimble run brelay server -``` - -## Deployment to fly.io - -If you'd like to run a relay server on [fly.io](https://fly.io/), sign up for the service then do one of the following. If you'd like to host somewhere else, you could use the Dockerfiles in [docker/](./docker/) as a starting point. - -TODO: - -### Single-user mode - -```sh -fly launch --dockerfile docker/singleuser.Dockerfile -``` - -### Multi-user mode - -```sh -fly launch --dockerfile docker/multiuser.Dockerfile -``` - ## Protocol -Relay clients communicate with the relay server using the following protocol. See [./src/bucketsrelay/proto.nim](./src/bucketsrelay/proto.nim) for more information, and [./src/bucketsrelay/stringproto.nim](./src/bucketsrelay/stringproto.nim) for encoding details. +Relay clients communicate with the relay server using the following protocol. See [./src/proto2.nim](./src/proto2.nim) for more information. -In summary, devices connect with websockets and exchange messages. Messages sent from client to server are called commands. Messages sent from server to client are called events. +In summary, devices connect with websockets and exchange messages. Messages sent from client to server are called *commands*. Messages sent from server to client are called *events*. ### Authentication -Clients authenticate with the server with a public/private key. A single person may have multiple public/private keys; typically one for each device. +Clients authenticate with the server using a public/private key. A single person may have multiple public/private keys; typically one for each device. ### Client Commands @@ -91,6 +59,9 @@ Clients send the following commands: | `PublishNote` | Send a few bytes to another client addressed by topic (good for key exchange) | | `FetchNote` | Request a note addressed by topic | | `SendData` | Store/forward bytes to other clients, addressed by relay-authenticated public keys | +| `StoreChunk` | Store bytes for other clients to fetch addressed by key and public key. | +| `GetChunk` | Request stored chunk | + ### Server Events @@ -103,10 +74,9 @@ The relay server sends the following events: | `Who` | Challenge for authenticating a client's public/private keys | | `Note` | Data payload of a note requested by `FetchNote` | | `Data` | Data payload from another client, addressed by relay-authenticated public key | +| `Chunk` | Data payload response to `GetChunk` request | -### Sequences and Usage - -#### Authentication +### Authentication Authentication happens like this: @@ -128,9 +98,17 @@ Client Relay │ │ ``` -#### Notes +### Data + +There are 3 ways clients can exchange data: + +1. Notes - public notes that are accessed by knowing the note *topic*. Notes are a good way to do key exchange. Notes expire after a short time. +2. Messages - ordered, stored-and-forwarded messages sent from one client to another client. These are automatically sent to a client upon connection, and deleted when sent. Messages expire after a while. +3. Chunks - clients store chunks with a string *key* and choose which clients (by their public key) are allowed to fetch uploaded chunks. Chunks may be overwritten. Chunks expire a while after their last update. -After authenticating, clients can send each other short notes, addressed by a string *topic*. Each note expires after a time and will only ever been sent to one client who. The `FetchNote` command may be sent before or after the note is published. It works like this: +All forms of exchanging data are unreliable. Build with that in mind. + +#### Notes 1. Alice sends `PublishNote(topic=apple, data=something)` 2. Bob sends `FetchNote(topic=apple)` @@ -150,13 +128,9 @@ Alice Relay Bob │ │ │ ``` -#### Data +#### Messages -After authenticating, clients may send data to be stored and forwarded to clients next time they connect. Stored data expires after a time. Messages sent to unknown public keys will be dropped without notice. In other words, the transport is unreliable by design. - -Here's how it works: - -1. Alice sends `SendData(dst=[BOBPK], data=hello)` +1. Alice sends `SendData(dst=BOBPK, data=hello)` 2. Server sends to Bob `Data(src=ALICEPK, data=hello)` ``` @@ -170,4 +144,22 @@ Alice Relay Bob │ │ │ ``` -Note that when multiple recipients are specified in a `SendData` command, all recipients will receive one copy of the message (assuming the message doesn't expire or get discarded instead). +#### Chunks + +1. Alice sends `StoreChunk(dst=[BOBPK], key=apple, val=seed)` +2. Bob sends `GetChunks(src=ALICEPK, keys=[apple])` +3. Server sends `Chunk(src=ALICEPK, key=apple, val=seed)` + +``` +Alice Relay Bob + │ │ │ + ├───────Authenticated─|─Authenticated──────┤ + │ │ │ + │ StoreChunk(apple) | │ + ├────────────────────►│ GetChunks([apple]) | + │ │◄───────────────────┤ + │ │ │ + │ │ Chunk(apple) │ + │ │───────────────────►│ + │ │ │ +``` diff --git a/bucketsrelay.nimble b/bucketsrelay.nimble index 8d22aa5..688d6cf 100644 --- a/bucketsrelay.nimble +++ b/bucketsrelay.nimble @@ -1,39 +1,7 @@ -# Package - version = "0.3.1" author = "Matt Haggard" description = "The relay service for the Buckets budgeting app" license = "MIT" srcDir = "src" -installExt = @["nim", "mustache", "png"] - - -# Dependencies -requires "nim >= 1.6.0" -requires "argparse == 4.0.1" -requires "libsodium == 0.6.0" -requires "mustache == 0.4.3" -requires "ndb == 0.19.9" -requires "https://github.com/status-im/nim-stew.git#d085e48e89062de307aab0d0629fba2f867cb49a" -requires "https://github.com/status-im/nim-serialization.git#9f56a0738c616061382928b9f60e1c5721622f51" -requires "https://github.com/status-im/nim-json-serialization.git#b068e1440d4cb2cf3ede6b3567eaaeecd6c8c96a" -requires "https://github.com/status-im/nim-chronos.git#ba143e029f35fd9b4cd3d89d007cc834d0d5ba3c" -requires "https://github.com/cheatfate/nimcrypto.git#a065c1741836462762d18d2fced1fedd46095b02" -requires "https://github.com/status-im/nim-websock.git#fea05cde8b123b38d1a0a8524b77efbc84daa848" -requires "https://github.com/yglukhov/bearssl_pkey_decoder.git#546f8d9bb887ae1d8a23f62155c583acb0358046" - - -# dependency locks -requires "https://github.com/status-im/nim-zlib.git#826e2fc013f55b4478802d4f2e39f187c50d520a" - -import std/os - -task singleuserbins, "Build single user brelay and bclient bins": - exec("mkdir -p bin") - exec("nimble c -d:relaysingleusermode -o:bin/brelay src/brelay.nim") - exec("nimble c -d:relaysingleusermode -o:bin/bclient src/bclient.nim") -task multiuserbins, "Build multi user brelay and bclient bins": - exec("mkdir -p bin") - exec("nimble c -o:bin/brelay src/brelay.nim") - exec("nimble c -o:bin/bclient src/bclient.nim") +requires "nim >= 2.2.4" diff --git a/changes/new-Move-to-store-20251028-153340.md b/changes/new-Move-to-store-20251028-153340.md new file mode 100644 index 0000000..77bf092 --- /dev/null +++ b/changes/new-Move-to-store-20251028-153340.md @@ -0,0 +1 @@ +Move to store and forward diff --git a/config.nims b/config.nims index e5ed4ac..5a61f19 100644 --- a/config.nims +++ b/config.nims @@ -1,6 +1,6 @@ # See LICENSE.md for licensing switch("gc", "orc") -switch("threads", "on") +switch("threads", "off") import os const ROOT = currentSourcePath.parentDir() diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..225e539 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,22 @@ +# -- Stage 1 -- # +FROM nimlang/nim:2.2.4-alpine-regular as builder +WORKDIR /app +RUN apk update && apk add libsodium-static libsodium musl-dev +RUN nimble refresh +RUN nimble install -y https://github.com/iffy/pkger/ +COPY pkger.json . +COPY ./pkger ./pkger +RUN pkger fetch +COPY . . +COPY docker/config.nims . +RUN nim c -d:release -o:brelay src/server2.nim +RUN strip brelay + +# -- Stage 2 -- # +FROM alpine:3.22.2 +WORKDIR /root/ +RUN apk update && apk add libressl4.1-libcrypto sqlite-dev +COPY --from=builder /app/brelay /usr/local/bin/ +RUN mkdir -p /data +EXPOSE 8080 +CMD ["/usr/local/bin/brelay", "--database", "/data/bucketsrelay.sqlite", "server", "--address", "0.0.0.0", "--port", "8080"] diff --git a/docker/config.nims b/docker/config.nims index dd44dfc..f8dd00a 100644 --- a/docker/config.nims +++ b/docker/config.nims @@ -1,8 +1,5 @@ -switch("gc", "orc") - -when defined(linux): - import os - switch("dynlibOverride", "libsodium") - switch("cincludes", "/usr/include") - switch("clibdir", "/usr/lib") - switch("passL", "-lsodium") +switch("threads", "off") +switch("dynlibOverride", "libsodium") +switch("cincludes", "/usr/include") +switch("clibdir", "/usr/lib") +switch("passL", "-lsodium") diff --git a/docker/multiuser.Dockerfile b/docker/multiuser.Dockerfile deleted file mode 100644 index de64ce0..0000000 --- a/docker/multiuser.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# -- Stage 1 -- # -FROM nimlang/nim:1.6.18-alpine@sha256:e54f241d4cc4c7e677641a535df6f5cae2e6fa527cb36f53a4c7bd77214b1b80 as builder -WORKDIR /app -RUN apk update && apk add libsodium-static libsodium musl-dev -RUN nimble refresh -RUN nimble install -y nimble -COPY bucketsrelay.nimble . -RUN nimble install -y --depsOnly --verbose -COPY . . -COPY docker/config.nims . -RUN nimble c -o:brelay -d:release src/brelay - -# -- Stage 2 -- # -FROM alpine:3.13.12@sha256:16fd981ddc557fd3b38209d15e7ee8e3e6d9d4d579655e8e47243e2c8525b503 -WORKDIR /root/ -RUN apk update && apk add libressl3.1-libcrypto sqlite-dev -COPY --from=builder /app/brelay /usr/local/bin/ -RUN mkdir -p /data -CMD ["/usr/local/bin/brelay", "--database", "/data/bucketsrelay.sqlite", "server", "--address", "0.0.0.0", "--port", "8080"] diff --git a/docker/singleuser.Dockerfile b/docker/singleuser.Dockerfile deleted file mode 100644 index 1e1aae0..0000000 --- a/docker/singleuser.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -- Stage 1 -- # -FROM nimlang/nim:1.6.18-alpine@sha256:e54f241d4cc4c7e677641a535df6f5cae2e6fa527cb36f53a4c7bd77214b1b80 as builder -WORKDIR /app -RUN apk update && apk add libsodium-static libsodium musl-dev -RUN nimble refresh -RUN nimble install -y nimble -COPY bucketsrelay.nimble . -RUN nimble install -y --depsOnly --verbose -COPY . . -COPY docker/config.nims . -RUN nimble c -o:brelay -d:relaysingleusermode -d:release src/brelay - -# -- Stage 2 -- # -FROM alpine:3.13.12@sha256:16fd981ddc557fd3b38209d15e7ee8e3e6d9d4d579655e8e47243e2c8525b503 -WORKDIR /root/ -RUN apk update && apk add libressl3.1-libcrypto sqlite-dev -COPY --from=builder /app/brelay /usr/local/bin/ -CMD ["/usr/local/bin/brelay", "server", "--address", "0.0.0.0", "--port", "8080"] diff --git a/nim.cfg b/nim.cfg index 518c89e..a8f6599 100644 --- a/nim.cfg +++ b/nim.cfg @@ -5,4 +5,5 @@ --path:"pkger/lazy/ws/src" --path:"pkger/lazy/nimja/src" --path:"pkger/lazy/db_connector/src" +--path:"pkger/lazy/argparse/src" ### PKGER END - DO NOT EDIT ABOVE ########### diff --git a/pkger/deps.json b/pkger/deps.json index ebd7bec..6fa80ad 100644 --- a/pkger/deps.json +++ b/pkger/deps.json @@ -31,10 +31,10 @@ "pkgname": "nimja", "parent": "", "src": { - "url": "https://github.com/enthus1ast/nimja", + "url": "https://github.com/iffy/nimja", "kind": "git" }, - "sha": "438fc1f6c654c69cd85c9a5d9cbe45a48f507f8c" + "sha": "3ce4c093e3699317b4cdf69111bca38e8e0f2b69" }, "db_connector": { "pkgname": "db_connector", @@ -44,6 +44,15 @@ "kind": "git" }, "sha": "74aef399e5c232f95c9fc5c987cebac846f09d62" + }, + "argparse": { + "pkgname": "argparse", + "parent": "", + "src": { + "url": "https://github.com/iffy/nim-argparse", + "kind": "git" + }, + "sha": "98c7c99bfbcaae750ac515a6fd603f85ed68668f" } } } \ No newline at end of file diff --git a/src/bclient.nim b/src/bclient.nim deleted file mode 100644 index db70a92..0000000 --- a/src/bclient.nim +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import std/logging -import std/strformat -import std/strutils -import std/base64 -import std/os - -import chronos except debug, info, warn, error - -import bucketsrelay/client -import bucketsrelay/proto -import bucketsrelay/asyncstdin - -type - SendHandler = ref object - data: string - sent: Future[void] - -proc newSendHandler(data: string): SendHandler = - new(result) - result.data = data - result.sent = newFuture[void]("newSendHandler") - -proc handleEvent(handler: SendHandler, ev: RelayEvent, remote: RelayClient) {.async.} = - try: - case ev.kind - of Connected: - await remote.sendData(ev.conn_pubkey, handler.data) - callSoon(proc(udata: pointer) = - handler.sent.complete()) - else: - discard - except CancelledError: - warn "SendHandler cancelled during event handling" - raise - -proc handleLifeEvent(handler: SendHandler, ev: ClientLifeEvent, remote: RelayClient) {.async.} = - discard - - -type - RecvHandler = ref object - buf: string - data: Future[string] - -proc newRecvHandler(): RecvHandler = - new(result) - result.data = newFuture[string]("newRecvHandler") - -proc handleEvent(handler: RecvHandler, ev: RelayEvent, remote: RelayClient) {.async.} = - try: - case ev.kind - of Data: - handler.buf.add(ev.data) - of Disconnected: - handler.data.complete(handler.buf) - else: - discard - except CancelledError: - warn "RecvHandler cancelled during event handling" - raise - -proc handleLifeEvent(handler: RecvHandler, ev: ClientLifeEvent, remote: RelayClient) {.async.} = - discard - -proc relaySend*(data: string, topubkey: PublicKey, relayurl: string, mykeys: KeyPair, username: string, password: string, verify = true): Future[void] {.async.} = - debug &"Sending {data.len} bytes to {topubkey} via {relayurl} ..." - var sh = newSendHandler(data) - var client = newRelayClient(mykeys, sh, username, password, verifyHostname = verify) - await client.connect(relayurl) - await client.connect(topubkey) - await sh.sent - await client.disconnect() - await client.done - -proc relayReceive*(frompubkey: PublicKey, relayurl: string, mykeys: KeyPair, username: string, password: string, verify = true): Future[string] {.async.} = - debug &"Receiving from {frompubkey} via {relayurl} ..." - var rh = newRecvHandler() - var client = newRelayClient(mykeys, rh, username, password, verifyHostname = verify) - await client.connect(relayurl) - await client.connect(frompubkey) - result = await rh.data - await client.disconnect() - await client.done - -type - ChatHandler = ref object - done: Future[void] - -proc handleEvent(handler: ChatHandler, ev: RelayEvent, remote: RelayClient) {.async.} = - case ev.kind - of Data: - stdout.write(ev.data) - stdout.flushFile() - of Disconnected: - handler.done.complete() - else: - discard - -proc handleLifeEvent(handler: ChatHandler, ev: ClientLifeEvent, remote: RelayClient) {.async.} = - discard - -proc chat(ch: ChatHandler, remote: RelayClient, remotePubkey: PublicKey) {.async.} = - let reader = asyncStdinReader() - while true: - let inp = reader.read(1) - await ch.done or inp - if ch.done.completed: - await inp.cancelAndWait() - break - else: - let data = inp.read() - await remote.sendData(remotePubkey, data) - -proc relayChat*(otherpubkey: PublicKey, relayurl: string, mykeys: KeyPair, username: string, password: string, verify = true): Future[string] {.async.} = - debug &"Attempting to chat with {otherpubkey} via {relayurl} ..." - var ch = ChatHandler() - ch.done = newFuture[void]() - var client = newRelayClient(mykeys, ch, username, password, verifyHostname = verify) - await client.connect(relayurl) - await client.connect(otherpubkey) - await ch.chat(client, otherpubkey) - -when isMainModule: - import argparse - var p = newParser: - command("genkeys"): - help("Generate a keypair") - option("-p", "--public", help="Filename to save public key to", default=some("relay.key.public")) - option("-s", "--secret", help="Filename to save secret key to", default=some("relay.key.secret")) - run: - var keys = genkeys() - writeFile(opts.public, keys.pk.string.encode & "\n") - echo "Wrote ", opts.public - writeFile(opts.secret, keys.sk.string.encode & "\n") - echo "Wrote ", opts.secret - echo "Public key:" - echo keys.pk.string.encode() - command("send"): - help("Send stdin through the relay") - option("-u", "--username", help="Relay username", env = "RELAY_USERNAME") - option("-p", "--password", help="Relay password", env = "RELAY_PASSWORD") - flag("-k", "--no-ssl-verify", help="Disable SSL verification") - option("--local-secret", help="Path to local secret key", default=some("relay.key.secret")) - option("--local-public", help="Path to local public key", default=some("relay.key.public")) - arg("url", help="Relay URL to connect to. Should end in /relay") - arg("public_key", help="Public key of remote client to connect to") - run: - newConsoleLogger(lvlAll, useStderr = true).addHandler() - let keys = ( - readFile(opts.local_public).decode().PublicKey, - readFile(opts.local_secret).decode().SecretKey, - ) - let pubkey = opts.public_key.decode().PublicKey - let data = stdin.readAll() - waitFor relaySend(data, pubkey, relayurl = opts.url, mykeys = keys, username = opts.username, password = opts.password, verify = not opts.no_ssl_verify) - command("receive"): - help("Receive data through the relay to stdout") - option("-u", "--username", help="Relay username", env = "RELAY_USERNAME") - option("-p", "--password", help="Relay password", env = "RELAY_PASSWORD") - flag("-k", "--no-ssl-verify", help="Disable SSL verification") - option("--local-secret", help="Path to local secret key", default=some("relay.key.secret")) - option("--local-public", help="Path to local public key", default=some("relay.key.public")) - arg("url", help="Relay URL to connect to. Should end in /relay") - arg("public_key", help="Public key of remote client to connect to") - run: - newConsoleLogger(lvlAll, useStderr = true).addHandler() - let keys = ( - readFile(opts.local_public).decode().PublicKey, - readFile(opts.local_secret).decode().SecretKey, - ) - let pubkey = opts.public_key.decode().PublicKey - echo waitFor relayReceive(pubkey, relayurl = opts.url, mykeys = keys, username = opts.username, password = opts.password, verify = not opts.no_ssl_verify) - command("chat"): - help("Open a symmetric chat stream with another client") - option("-u", "--username", help="Relay username", env = "RELAY_USERNAME") - option("-p", "--password", help="Relay password", env = "RELAY_PASSWORD") - flag("-k", "--no-ssl-verify", help="Disable SSL verification") - option("--local-secret", help="Path to local secret key", default=some("relay.key.secret")) - option("--local-public", help="Path to local public key", default=some("relay.key.public")) - flag("-v", "--verbose", help="Verbose logging") - arg("url", help="Relay URL to connect to. Should end in /relay") - arg("public_key", help="Public key of remote client to connect to") - run: - if opts.verbose: - newConsoleLogger(lvlAll, useStderr = true).addHandler() - let keys = ( - readFile(opts.local_public).decode().PublicKey, - readFile(opts.local_secret).decode().SecretKey, - ) - let pubkey = opts.public_key.decode().PublicKey - echo waitFor relayChat(pubkey, relayurl = opts.url, mykeys = keys, username = opts.username, password = opts.password, verify = not opts.no_ssl_verify) - try: - p.run() - except UsageError: - stderr.writeLine getCurrentExceptionMsg() - quit(1) diff --git a/src/brelay.nim b/src/brelay.nim deleted file mode 100644 index 192e70d..0000000 --- a/src/brelay.nim +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import std/logging -import std/strformat -import std/json - -import chronos - -import bucketsrelay/common -import bucketsrelay/server - -proc monitorMemory() {.async.} = - var - lastTotal = 0 - lastOccupied = 0 - lastFree = 0 - while true: - let - newTotal = getTotalMem() - newOccupied = getOccupiedMem() - newFree = getFreeMem() - diffTotal = newTotal - lastTotal - diffOccupied = newOccupied - lastOccupied - diffFree = newFree - lastFree - debug "--- Memory report ---" - debug &"Total memory: {newTotal:>10} <- {lastTotal:>10} diff {diffTotal:>10}" - debug &"Occupied memory: {newOccupied:>10} <- {lastOccupied:>10} diff {diffOccupied:>10}" - debug &"Free memory: {newFree:>10} <- {lastFree:>10} diff {diffFree:>10}" - lastTotal = newTotal - lastOccupied = newOccupied - lastFree = newFree - await sleepAsync(10.seconds) - -proc startRelaySingleUser*(username, password: string, port = 9001.Port, address = "127.0.0.1"): RelayServer {.singleuseronly.} = - ## Start the relay server on the given port. - result = newRelayServer(username, password) - let taddress = initTAddress(address, port.int) - info &"Starting Single-User Buckets Relay on {taddress} ..." - stderr.flushFile - result.start(taddress) - -proc getRelayServer(dbfilename: string): RelayServer {.multiuseronly.} = - newRelayServer(dbfilename, pubkey = AUTH_LICENSE_PUBKEY) - -proc startRelay*(dbfilename: string, port = 9001.Port, address = "127.0.0.1"): RelayServer {.multiuseronly.} = - ## Start the relay server on the given port. - result = getRelayServer(dbfilename) - let taddress = initTAddress(address, port.int) - info &"Starting Buckets Relay on {taddress} ..." - if result.pubkey == "": - info &"[config] License auth: DISABLED" - else: - info &"[config] License auth: on" - stderr.flushFile - result.start(taddress) - result.periodically_delete_old_stats() - -proc addverifieduser*(dbfilename, username, password: string) {.multiuseronly.} = - var rs = getRelayServer(dbfilename) - let userid = rs.register_user(username, password) - let token = rs.generate_email_verification_token(userid) - doAssert rs.use_email_verification_token(userid, token) == true - -proc blockuser*(dbfilename, email: string) {.multiuseronly.} = - var rs = getRelayServer(dbfilename) - let uid = rs.get_user_id(email) - rs.block_user(uid) - -proc unblockuser*(dbfilename, email: string) {.multiuseronly.} = - var rs = getRelayServer(dbfilename) - let uid = rs.get_user_id(email) - rs.unblock_user(uid) - -proc blocklicense*(dbfilename, email: string) {.multiuseronly.} = - var rs = getRelayServer(dbfilename) - let uid = rs.get_user_id(email) - rs.disable_most_recently_used_license(uid) - -proc stats(dbfilename: string, days = 30): JsonNode {.multiuseronly.} = - result = %* { - "days": days, - "users": [], - "ips": [], - } - var rs = newRelayServer(dbfilename, updateSchema = false, pubkey = AUTH_LICENSE_PUBKEY) - for row in rs.top_data_users(20, days = days): - result["users"].add(%* { - "sent": row.data.sent, - "recv": row.data.recv, - "user": row.user, - }) - for row in rs.top_data_ips(20, days = days): - result["ips"].add(%* { - "sent": row.data.sent, - "recv": row.data.recv, - "ip": row.ip, - }) - -proc showStats(dbfilename: string, days = 30): string {.multiuseronly.} = - ## Show some usage stats - return stats(dbfilename, days).pretty - -when defined(posix): - proc getpass(prompt: cstring) : cstring {.header: "", importc: "getpass".} -else: - proc getpass(prompt: cstring): cstring = - stdout.write(prompt) - stdout.flushFile() - stdin.readLine() - -when isMainModule: - import argparse - newConsoleLogger(lvlAll, useStderr = true).addHandler() - when multiusermode: - var p = newParser: - option("-d", "--database", help="User/stats database filename", default=some("bucketsrelay.sqlite")) - command("adduser"): - help("Add a user") - arg("email", help="Email address of user") - flag("--password-stdin", help="If given, read the password from stdin rather than from the terminal") - run: - var password = if opts.password_stdin: - stdout.write("Password? ") - stdout.flushFile - stdin.readLine() - else: - $getpass("Password? ".cstring) - addverifieduser(opts.parentOpts.database, opts.email, password) - echo "added user ", opts.email - command("blockuser"): - help("Block a user from using the relay") - arg("email", help="Email address of user to block") - run: - blockuser(opts.parentOpts.database, opts.email) - echo "User blocked" - command("unblockuser"): - help("Unblock a previously blocked user") - arg("email", help="Email address of user to block") - run: - unblockuser(opts.parentOpts.database, opts.email) - echo "User unblocked" - command("disablelicense"): - help("Disable a user's most recently-used license") - arg("email", help="Email address of user") - run: - blocklicense(opts.parentOpts.database, opts.email) - echo "License disabled" - command("stats"): - help("Show some statistics") - option("--days", help = "Show data for this number of days", default=some("30")) - run: - echo showStats(opts.parentOpts.database, days=opts.days.parseInt) - command("server"): - help("Start the relay server") - option("-p", "--port", help="Port to run server on", default=some("9001")) - option("-a", "--address", help="Address to run on", default=some("127.0.0.1")) - run: - var server = startRelay(opts.parentOpts.database, opts.port.parseInt.Port, opts.address) - runForever() - elif singleusermode: - var p = newParser: - command("server"): - help("Start a single user relay server. Set RELAY_USERNAME and RELAY_PASSWORD environment variables") - option("-p", "--port", help="Port to run server on", default=some("9001")) - option("-a", "--address", help="Address to run on", default=some("127.0.0.1")) - option("-u", "--username", help="Username", env = "RELAY_USERNAME") - option("-P", "--password", help="Password", env = "RELAY_PASSWORD") - run: - var server = startRelaySingleUser(opts.username, opts.password, opts.port.parseInt.Port, opts.address) - runForever() - try: - p.run() - except UsageError: - stderr.writeLine getCurrentExceptionMsg() - quit(1) diff --git a/src/bucketsrelay/asyncstdin.nim b/src/bucketsrelay/asyncstdin.nim deleted file mode 100644 index b909bbe..0000000 --- a/src/bucketsrelay/asyncstdin.nim +++ /dev/null @@ -1,84 +0,0 @@ -## Asynchronous reading from stdin -## -## The implementation may change. The important thing is that this works: -## -## var reader = asyncStdinReader() -## let res = waitFor reader.read(10) -import std/deques -import chronos - -const BUFSIZE = 4096.uint - -type - ReadResponse = uint - - AsyncStdinReader* = ref object - outQ: AsyncQueue[string] - inQ: AsyncQueue[uint] - closed: bool - thread: Thread[AsyncFD] - -var requestCh: Channel[uint] -requestCh.open(0) - -proc workerReadLoop(wfd: AsyncFD) {.thread.} = - ## Run this in a thread other than the main one - ## to get somewhat asynchronous IO - let transp = fromPipe(wfd) - var buf: array[BUFSIZE, char] - var closed = false - while not closed: - let req = requestCh.recv() - var toRead = req - var didRead: uint = 0 - while toRead > 0: - let toReadThisTime = min(BUFSIZE, toRead) - let n = stdin.readBuffer(addr buf, toReadThisTime) - if n == 0: - closed = true - break - didRead.inc(n) - toRead.dec(n) - discard waitFor transp.write(addr buf, n) - waitFor transp.closeWait() - -proc mainReadLoop(reader: AsyncStdinReader, transp: StreamTransport) {.async.} = - ## Run this companion loop of workerReadLoop in the main thread - while true: - let size = await reader.inQ.get() - var ret = "" - if not reader.closed: - var toRead = size - while toRead > 0 and not reader.closed: - var data: seq[byte] - try: - data = await transp.read(toRead.int) - except: - discard - if data.len == 0: - reader.closed = true - break - for c in data: - ret.add(chr(c)) - toRead.dec(data.len) - reader.outQ.putNoWait(ret) - -#--------------------------------------------------------------- -# Public API -#--------------------------------------------------------------- -proc asyncStdinReader*(): AsyncStdinReader = - new(result) - result.inQ = newAsyncQueue[uint]() - result.outQ = newAsyncQueue[string]() - let (rfd, wfd) = createAsyncPipe() - let readTransp = fromPipe(rfd) - result.thread.createThread(workerReadLoop, wfd) - asyncSpawn result.mainReadLoop(readTransp) - -proc read*(reader: AsyncStdinReader, size: uint): Future[string] {.async.} = - requestCh.send(size) - reader.inQ.putNoWait(size) - return await reader.outQ.get() - -template read*(reader: AsyncStdinReader, size: int): untyped = - reader.read(size.uint) diff --git a/src/bucketsrelay/client.nim b/src/bucketsrelay/client.nim deleted file mode 100644 index 586e161..0000000 --- a/src/bucketsrelay/client.nim +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. -import std/base64 -import std/logging -import std/options -import std/random -import std/strformat - -import chronos; export chronos -import chronicles except debug, info, warn, error -export activeChroniclesStream, Record, activeChroniclesScope -import stew/byteutils -import websock/websock - -import ./common -import ./netstring -import ./proto; export proto -import ./stringproto - -const HEARTBEAT_INTERVAL = 50.seconds -const HEARTBEAT_JITTER = 1000 - -type - ## RelayClient wraps a single websockets connection - ## to a relay server. It will call things on - ## `handler: T` to interact with your code. - RelayClient*[T] = ref object - keys: KeyPair - wsopt: Option[WSSession] - handler*: T - username: string - password: string - verifyHostname: bool - done*: Future[void] - tasks*: seq[Future[void]] - debugname*: string - - ClientLifeEventKind* = enum - ConnectedToServer - DisconnectedFromServer - - ClientLifeEvent* = ref object - case kind*: ClientLifeEventKind - of ConnectedToServer: - discard - of DisconnectedFromServer: - discard - - RelayErrLoginFailed* = RelayErr - RelayNotConnected* = RelayErr - -proc newRelayClient*[T](keys: KeyPair, handler: T, username, password: string, verifyHostname = true): RelayClient[T] = - new(result) - result.keys = keys - result.handler = handler - result.username = username - result.password = password - result.verifyHostname = verifyHostname - # result.done = newFuture[void]("newRelayClient done") - result.debugname = "RelayClient" & nextDebugName() - -proc logname*(client: RelayClient): string = - "(" & client.debugname & ") " - -proc `$`*(client: RelayClient): string = client.debugname - -proc ws*(client: RelayClient): WSSession = - if client.wsopt.isSome: - client.wsopt.get() - else: - raise RelayNotConnected.newException("Not connected") - -proc send(ws: WSSession, cmd: RelayCommand) {.async.} = - ## Send a RelayCommand to the server - await ws.send(nsencode(dumps(cmd)).toBytes, Opcode.Binary) - -proc keepAliveLoop(client: RelayClient) {.async.} = - ## Start a loop that periodically issues a ping to keep the - ## connection alive - try: - while true: - await sleepAsync(HEARTBEAT_INTERVAL + rand(HEARTBEAT_JITTER).milliseconds) - if client.wsopt.isSome: - let ws = client.wsopt.get() - await ws.ping() - else: - break - except CancelledError: - discard - except: - error client.logname, "unexpected error in ws keepAliveLoop" - -proc loop(client: RelayClient, authenticated: Future[void]): Future[void] {.async.} = - var decoder = newNetstringDecoder() - if client.wsopt.isSome(): - let ws = client.ws - while ws.readyState != ReadyState.Closed: - try: - let buff = try: - await ws.recvMsg() - except Exception as exc: - break - if buff.len <= 0: - break - let data = string.fromBytes(buff) - decoder.consume(data) - while decoder.hasMessage(): - let ev = loadsRelayEvent(decoder.nextMessage()) - case ev.kind - of Who: - await ws.send(RelayCommand( - kind: Iam, - iam_signature: sign(client.keys.sk, ev.who_challenge), - iam_pubkey: client.keys.pk, - )) - of Authenticated: - authenticated.complete() - else: - discard - try: - await client.handler.handleEvent(ev, client) - except: - debug client.logname, "Error handling event ", $ev, " ", getCurrentExceptionMsg() - raise - except CancelledError: - break - debug client.logname, "closing..." - client.wsopt = none[WSSession]() - await ws.close() - await client.handler.handleLifeEvent(ClientLifeEvent( - kind: DisconnectedFromServer, - ), client) - -proc authHeaderHook*(username, password: string): Hook = - ## Create a websock connection hook that adds Basic HTTP authentication - ## to the websocket. - new(result) - result.append = proc(ctx: Hook, headers: var HttpTable): Result[void, string] = - headers.add("Authorization", "Basic " & base64.encode(username & ":" & password)) - ok() - -proc addHeadersHook(addheaders: HttpTable): Hook = - new(result) - result.append = proc(ctx: Hook, headers: var HttpTable): Result[void, string] = - for key, val in addheaders.stringItems: - headers.add(key, val) - ok() - -proc connect*(client: RelayClient, url: string) {.async.} = - ## Connect and authenticate with a relay server. Returns - ## after authentication succeeds. - var uri = parseUri(url) - if uri.scheme == "http": - uri.scheme = "ws" - elif uri.scheme == "https": - uri.scheme = "wss" - let - hostname = uri.hostname - port = if uri.port == "": "443" else: uri.port - addresses = resolveTAddress(uri.hostname, port.parseInt.Port) - hooks = @[ - authHeaderHook(client.username, client.password), - addHeadersHook(HttpTable.init({ - "User-Agent": "buckets-relay client 1.0", - })), - ] - tls = uri.scheme == "https" or uri.scheme == "wss" or port == "443" - if addresses.len == 0: - raise ValueError.newException(&"Unable to resolve {uri.hostname}") - let address = addresses[0] - try: - let ws = if tls: - var flags: set[TLSFlags] - if not client.verifyHostname: - flags.incl(TLSFlags.NoVerifyHost) - flags.incl(TLSFlags.NoVerifyServerName) - await WebSocket.connect( - uri, - protocols = @["proto"], - flags = flags, - hooks = hooks, - version = WSDefaultVersion, - frameSize = WSDefaultFrameSize, - onPing = nil, - onPong = nil, - onClose = nil, - rng = nil, - ) - else: - await WebSocket.connect( - address, - path = uri.path, - hooks = hooks, - hostName = hostname, - ) - client.wsopt = some(ws) - except WebSocketError as exc: - let msg = getCurrentExceptionMsg() - if "403" in msg and "Forbidden" in msg: - raise RelayErrLoginFailed.newException("Failed initial authentication") - else: - raise exc - await client.handler.handleLifeEvent(ClientLifeEvent( - kind: ConnectedToServer, - ), client) - var authenticated = newFuture[void]("relay.client.dial.authenticated") - client.done = client.loop(authenticated) - let fut = client.keepAliveLoop() - client.tasks.add(fut) - await authenticated - -proc connect*(client: RelayClient, pubkey: PublicKey) {.async, raises: [RelayNotConnected].} = - ## Initiate a connection through the relay to the given public key - await client.ws.send(RelayCommand( - kind: Connect, - conn_pubkey: pubkey, - )) - -proc sendData*(client: RelayClient, dest_pubkey: PublicKey, data: string) {.async, raises: [RelayNotConnected].} = - ## Send data to a connection through the relay - await client.ws.send(RelayCommand( - kind: SendData, - send_data: data, - dest_pubkey: dest_pubkey, - )) - -proc disconnect*(client: RelayClient) {.async.} = - ## Disconnect this client from the network - if not client.done.isNil: - await client.done.cancelAndWait() - if client.wsopt.isSome: - await client.wsopt.get().close() - client.wsopt = none[WSSession]() - for task in client.tasks: - await task.cancelAndWait() - -proc disconnect*(client: RelayClient, dest_pubkey: PublicKey) {.async.} = - ## Disconnect this client from a remote client - if client.wsopt.isSome: - await client.wsopt.get().send(RelayCommand( - kind: Disconnect, - dcon_pubkey: dest_pubkey, - )) - \ No newline at end of file diff --git a/src/bucketsrelay/common.nim b/src/bucketsrelay/common.nim deleted file mode 100644 index 0b75e5d..0000000 --- a/src/bucketsrelay/common.nim +++ /dev/null @@ -1,59 +0,0 @@ -import std/macros -import std/strformat -import std/random; export random - -import chronicles - -import libsodium/sodium -import libsodium/sodium_sizes - -const - singleusermode* = defined(relaysingleusermode) - multiusermode* = not singleusermode - relayverbose* = defined(relayverbose) - -template nextDebugName*(): untyped = - $rand(100000..999999) - -template vlog*(x: varargs[string, `$`]): untyped = - when relayverbose: - debug x - -proc hash_password*(password: string): string = - # We use a lower memlimit because a stolen password is - # easy to mitigate and doesn't cause immediate harm to users. - let memlimit = max(crypto_pwhash_memlimit_min(), 10_000_000) - crypto_pwhash_str(password, memlimit = memlimit) - -proc verify_password*(pwhash: string, password: string): bool {.inline.} = - crypto_pwhash_str_verify(pwhash, password) - -macro multiuseronly*(fn: untyped): untyped = - ## Add as a pragma to procs that are only available in multiusermode - when multiusermode: - fn - else: - newStmtList() - -macro singleuseronly*(fn: untyped): untyped = - ## Add as a pragma to procs that should only be available in singleusermode - when singleusermode: - fn - else: - newStmtList() - -#------------------------------------------------------------ -# Memory-checking helpers -#------------------------------------------------------------ -var lastMem = getOccupiedMem() - -proc checkmem*(name: string) = - let newMem = getOccupiedMem() - let diffMem = newMem - lastMem - debug "checkmem", res = &"{diffMem:>10} = {newMem:>10} <- {lastMem:>10} {name}" - lastMem = newMem - -template checkMemGrowth(name: string, body: untyped): untyped = - let occ {.genSym.} = getOccupiedMem() - body - echo "Mem growth during: " & name & " " & $(getOccupiedMem() - occ) diff --git a/src/bucketsrelay/dbschema.nim b/src/bucketsrelay/dbschema.nim deleted file mode 100644 index a9c92d6..0000000 --- a/src/bucketsrelay/dbschema.nim +++ /dev/null @@ -1,41 +0,0 @@ -import std/strformat -import std/strutils -import std/logging - -import ndb/sqlite - -type - Patch* = tuple - name: string - sqls: seq[string] - -proc upgradeSchema*(db:DbConn, patches:openArray[Patch]) = - ## Apply database patches to this file - # See what patches have already been applied - db.exec(sql""" - CREATE TABLE IF NOT EXISTS _schema_version ( - id INTEGER PRIMARY KEY, - created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - name TEXT UNIQUE - )""") - var applied:seq[string] - for row in db.getAllRows(sql"SELECT name FROM _schema_version"): - applied.add(row[0].s) - if applied.len > 0: - logging.debug &"(dbpatch) existing patches: {applied}" - - # Apply patches - for patch in patches: - if patch.name in applied: - continue - logging.info &"(dbpatch) applying patch: {patch.name}" - db.exec(sql"BEGIN") - try: - for statement in patch.sqls: - db.exec(sql(statement)) - db.exec(sql"INSERT INTO _schema_version (name) VALUES (?)", patch.name) - db.exec(sql"COMMIT") - except: - logging.error &"(dbpatch) error applying patch {patch.name}: {getCurrentExceptionMsg()}" - db.exec(sql"ROLLBACK") - raise diff --git a/src/bucketsrelay/httpreq.nim b/src/bucketsrelay/httpreq.nim deleted file mode 100644 index 68be8b3..0000000 --- a/src/bucketsrelay/httpreq.nim +++ /dev/null @@ -1,88 +0,0 @@ -## HTTP client that does SSL/TLS with BearSSL (so you don't need `-d:ssl`) -## -import std/strutils -import std/options -import std/uri - -import chronos -import chronos/apps/http/httpclient -import chronos/apps/http/httpcommon; export httpcommon -import chronos/apps/http/httptable; export httptable - -export waitFor - -type - HttpResponse* = tuple - code: int - body: string - - -proc fetch*(session: HttpSessionRef, req: HttpClientRequestRef): Future[HttpResponseTuple] {.async.} = - ## Copied from nim-chronos - var - request = req - response: HttpClientResponseRef = nil - redirect: HttpClientRequestRef = nil - - while true: - try: - response = await request.send() - if response.status >= 300 and response.status < 400: - redirect = - block: - if "location" in response.headers: - let location = response.headers.getString("location") - if len(location) > 0: - let res = request.redirect(parseUri(location)) - if res.isErr(): - raiseHttpRedirectError(res.error()) - res.get() - else: - raiseHttpRedirectError("Location header with an empty value") - else: - raiseHttpRedirectError("Location header missing") - discard await response.consumeBody() - await response.closeWait() - response = nil - await request.closeWait() - request = nil - request = redirect - request.headers.set(HostHeader, request.address.hostname) - redirect = nil - else: - let data = await response.getBodyBytes() - let code = response.status - await response.closeWait() - response = nil - await request.closeWait() - request = nil - return (code, data) - except CancelledError as exc: - if not(isNil(response)): await closeWait(response) - if not(isNil(request)): await closeWait(request) - if not(isNil(redirect)): await closeWait(redirect) - raise exc - except HttpError as exc: - if not(isNil(response)): await closeWait(response) - if not(isNil(request)): await closeWait(request) - if not(isNil(redirect)): await closeWait(redirect) - raise exc - -proc request*(url: string, meth: HttpMethod, body = "", headers = HttpTable.init()): Future[HttpResponse] {.async.} = - ## High level request - var session = HttpSessionRef.new() - let address = session.getAddress(url).tryGet() - var req = HttpClientRequestRef.new(session, address, meth, - body = body.toOpenArrayByte(0, body.len-1), - headers = headers.toList()) - let (code, bytes) = await session.fetch(req) - return (code, bytes.bytesToString) - -when isMainModule: - import std/os - import std/strformat - let url = paramStr(1) - echo &"requesting {url}" - let resp = waitFor request(url, MethodGet) - echo "resp: ", resp[0] - echo resp[1] diff --git a/src/bucketsrelay/jwtrsaonly.nim b/src/bucketsrelay/jwtrsaonly.nim deleted file mode 100644 index dbfef21..0000000 --- a/src/bucketsrelay/jwtrsaonly.nim +++ /dev/null @@ -1,253 +0,0 @@ -# This code comes from https://github.com/yglukhov/nim-jwt -# but with modifications to work with the version of BearSSL -# included with this project and only support RSA256 JWTs. - -# The MIT License (MIT) - -# Copyright (c) 2017 Yuriy Glukhov - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import std/base64 -import std/json -import std/strutils - -import bearssl -import bearssl_pkey_decoder - -#-------------------------------------- -# jwt/private/utils -#-------------------------------------- - -proc encodeUrlSafe(s: openarray[byte]): string = - when NimMajor >= 1 and (NimMinor >= 1 or NimPatch >= 2): - result = base64.encode(s) - else: - result = base64.encode(s, newLine="") - while result.endsWith("="): - result.setLen(result.len - 1) - result = result.replace('+', '-').replace('/', '_') - -proc encodeUrlSafe(s: openarray[char]): string {.inline.} = - encodeUrlSafe(s.toOpenArrayByte(s.low, s.high)) - -proc decodeUrlSafeAsString(s: string): string = - var s = s.replace('-', '+').replace('_', '/') - while s.len mod 4 > 0: - s &= "=" - base64.decode(s) - -proc decodeUrlSafe(s: string): seq[byte] = - cast[seq[byte]](decodeUrlSafeAsString(s)) - -#-------------------------------------- -# jwt/private/jose -#-------------------------------------- - -proc toBase64(j: JsonNode): string = - encodeUrlSafe($j) - -#-------------------------------------- -# jwt/crypto -#-------------------------------------- - -# This pragma should be the same as in nim-bearssl/decls.nim -{.pragma: bearSslFunc, cdecl, gcsafe, noSideEffect, raises: [].} - -#-------------------------------------- -# Custom PEM-decoding -#-------------------------------------- - -proc invalidPemKey() = - raise newException(ValueError, "Invalid PEM encoding") - -proc pemDecoderLoop(pem: string, prc: proc(ctx: pointer, pbytes: pointer, nbytes: uint) {.bearSslFunc.}, ctx: pointer) = - var pemCtx: PemDecoderContext - pemDecoderInit(pemCtx) - var length = len(pem) - var offset = 0 - var inobj = false - while length > 0: - var tlen = pemDecoderPush(pemCtx, - unsafeAddr pem[offset], length.uint).int - offset = offset + tlen - length = length - tlen - - let event = pemDecoderEvent(pemCtx) - if event == PEM_BEGIN_OBJ: - inobj = true - pemDecoderSetdest(pemCtx, prc, ctx) - elif event == PEM_END_OBJ: - if inobj: - inobj = false - else: - break - elif event == 0 and length == 0: - break - else: - invalidPemKey() - -proc decodeFromPem(skCtx: var SkeyDecoderContext, pem: string) = - skeyDecoderInit(skCtx) - pemDecoderLoop(pem, cast[proc(ctx: pointer, pbytes: pointer, nbytes: uint) {.bearSslFunc.}](skeyDecoderPush), addr skCtx) - if skeyDecoderLastError(skCtx) != 0: invalidPemKey() - -proc decodeFromPem(pkCtx: var PkeyDecoderContext, pem: string) = - pkeyDecoderInit(addr pkCtx) - pemDecoderLoop(pem, cast[proc(ctx: pointer, pbytes: pointer, nbytes: uint) {.bearSslFunc.}](pkeyDecoderPush), addr pkCtx) - if pkeyDecoderLastError(addr pkCtx) != 0: invalidPemKey() - -proc calcHash(alg: ptr HashClass, data: string, output: var array[64, byte]) = - var ctx: array[512, byte] - let pCtx = cast[ptr ptr HashClass](addr ctx[0]) - assert(alg.contextSize <= sizeof(ctx).uint) - alg.init(pCtx) - if data.len > 0: - alg.update(pCtx, unsafeAddr data[0], data.len.uint) - alg.`out`(pCtx, addr output[0]) - -proc bearSignRSPem(data, key: string, alg: ptr HashClass, hashOid: cstring, hashLen: int): seq[byte] = - # Step 1. Extract RSA key from `key` in PEM format - var skCtx: SkeyDecoderContext - decodeFromPem(skCtx, key) - if skeyDecoderKeyType(skCtx) != KEYTYPE_RSA: - invalidPemKey() - - template privateKey(): RsaPrivateKey = skCtx.key.rsa - - # Step 2. Hash! - var digest: array[64, byte] - calcHash(alg, data, digest) - - let sigLen = (privateKey.nBitlen + 7) div 8 - result = newSeqUninitialized[byte](sigLen) - let s = rsaPkcs1SignGetDefault() - assert(not s.isNil) - if s(cast[ptr byte](hashOid), addr digest[0], hashLen.uint, addr privateKey, addr result[0]) != 1: - raise newException(ValueError, "Could not sign") - -proc bearVerifyRSPem(data, key: string, sig: openarray[byte], alg: ptr HashClass, hashOid: cstring, hashLen: int): bool = - # Step 1. Extract RSA key from `key` in PEM format - var pkCtx: PkeyDecoderContext - decodeFromPem(pkCtx, key) - if pkeyDecoderKeyType(addr pkCtx) != KEYTYPE_RSA: - invalidPemKey() - template publicKey(): RsaPublicKey = pkCtx.key.rsa - - var digest: array[64, byte] - calcHash(alg, data, digest) - - let s = rsaPkcs1VrfyGetDefault() - var digest2: array[64, byte] - - if s(unsafeAddr sig[0], sig.len.uint, cast[ptr byte](hashOid), hashLen.uint, addr publicKey, addr digest2[0]) != 1: - return false - - digest == digest2 - - -#-------------------------------------- -# jwt main -#-------------------------------------- - -type - InvalidToken* = object of ValueError - - JWT* = object - headerB64: string - claimsB64: string - header*: JsonNode - claims*: JsonNode - signature*: seq[byte] - - -proc splitToken(s: string): seq[string] = - let parts = s.split(".") - if parts.len != 3: - raise newException(InvalidToken, "Invalid token") - result = parts - -proc initJWT*(header: JsonNode, claims: JsonNode, signature: seq[byte] = @[]): JWT = - JWT( - headerB64: header.toBase64, - claimsB64: claims.toBase64, - header: header, - claims: claims, - signature: signature - ) - -# Load up a b64url string to JWT -proc toJWT*(s: string): JWT = - var parts = splitToken(s) - let - headerB64 = parts[0] - claimsB64 = parts[1] - headerJson = parseJson(decodeUrlSafeAsString(headerB64)) - claimsJson = parseJson(decodeUrlSafeAsString(claimsB64)) - signature = decodeUrlSafe(parts[2]) - - JWT( - headerB64: headerB64, - claimsB64: claimsB64, - header: headerJson, - claims: claimsJson, - signature: signature - ) - -proc toJWT*(node: JsonNode): JWT = - initJWT(node["header"], node["claims"]) - -# Encodes the raw signature to b64url -proc signatureToB64(token: JWT): string = - assert token.signature.len != 0 - result = encodeUrlSafe(token.signature) - -proc loaded(token: JWT): string = - token.headerB64 & "." & token.claimsB64 - -proc parsed(token: JWT): string = - result = token.header.toBase64 & "." & token.claims.toBase64 - -# Signs a string with a secret -proc signString(toSign: string, secret: string): seq[byte] = - template rsSign(hc, oid: typed, hashLen: int): seq[byte] = - bearSignRSPem(toSign, secret, addr hc, oid, hashLen) - return rsSign(sha256Vtable, HASH_OID_SHA256, sha256SIZE) - -# Verify that the token is not tampered with -proc verifySignature(data: string, signature: seq[byte], secret: string): bool = - result = bearVerifyRSPem(data, secret, signature, addr sha256Vtable, HASH_OID_SHA256, sha256SIZE) - -proc sign*(token: var JWT, secret: string) = - assert token.signature.len == 0 - token.signature = signString(token.parsed, secret) - -# Verify a token typically an incoming request -proc verify*(token: JWT, secret: string): bool = - verifySignature(token.loaded, token.signature, secret) - -proc toString(token: JWT): string = - token.header.toBase64 & "." & token.claims.toBase64 & "." & token.signatureToB64 - -proc `$`*(token: JWT): string = - token.toString - -proc `%`*(token: JWT): JsonNode = - let s = $token - %s diff --git a/src/bucketsrelay/licenses.nim b/src/bucketsrelay/licenses.nim deleted file mode 100644 index 452849c..0000000 --- a/src/bucketsrelay/licenses.nim +++ /dev/null @@ -1,80 +0,0 @@ -import std/json -import std/strformat -import std/strutils -import std/times - -import ./jwtrsaonly - -proc formatForEmail*(x: string): string = - ## Format a base64-encoded string nicely for email delivery - for i,c in x: - result.add(c) - if (i+1) mod 40 == 0: - result.add "\n" - elif (i+1) mod 10 == 0: - result.add " " - if result[^1] != '\n': - result.add "\n" - -#------------------------------------------------------ -# V1 RSA License -#------------------------------------------------------ -const - rsaPrefix = "-----BEGIN RSA PRIVATE KEY-----" - rsaSuffix = "-----END RSA PRIVATE KEY-----" - licensePrefix = "------------- START LICENSE ---------------" - licenseSuffix = "------------- END LICENSE -----------------" - -type - BucketsV1License* = distinct string - -proc unformatLicense*(x: string): BucketsV1License = - var tmp = x.replace(licensePrefix, "").replace(licenseSuffix, "") - var res: string - for c in tmp: - case c - of 'a'..'z','A'..'Z','0'..'9','+','=','_','-','/','.': - res.add c - else: - discard - return res.BucketsV1License - -proc createV1License*(privateKey: string, email: string): BucketsV1License = - ## Generate a new license - var privateKey = privateKey.replace(rsaPrefix, "") - privateKey = privateKey.replace(rsaSuffix, "") - privateKey = privateKey.strip().replace(" ", "\n") - privateKey = &"{rsaPrefix}\n{privateKey}\n{rsaSuffix}" - var token = toJWT( %* { - "header": { - "alg": "RS256", - "typ": "JWT" - }, - "claims": { - "email": email, - "iat": getTime().toUnix(), - } - }) - token.sign(privateKey) - return ($token).BucketsV1License - -proc `$`*(license: BucketsV1License): string = - ## Format a license for delivery in email - result.add licensePrefix & "\n" - result.add license.string.formatForEmail() - result.add licenseSuffix - -proc verify*(license: BucketsV1License, pubkey: string): bool = - ## Return true if the license is valid, raise an exception if not - result = false - let jwtToken = license.string.toJWT() - result = jwtToken.verify(pubkey) - -proc extractEmail*(license: BucketsV1License): string = - ## Extract the email address this license was issued to - try: - let jwt = license.string.toJWT() - return $jwt.claims["email"].getStr() - except: - discard - diff --git a/src/bucketsrelay/mailer.nim b/src/bucketsrelay/mailer.nim deleted file mode 100644 index 5fab560..0000000 --- a/src/bucketsrelay/mailer.nim +++ /dev/null @@ -1,56 +0,0 @@ -import std/logging -import std/os -import std/strformat -import std/strutils - -import chronos - -import ./common - -const usepostmark = multiusermode and not defined(nopostmark) -const fromEmail {.strdefine.} = "relay@budgetwithbuckets.com" -when usepostmark: - const POSTMARK_API_KEY {.strdefine.} = "env:POSTMARK_API_KEY" - - import std/json - import ./httpreq - - -proc valueRef(location: string): string = - ## Get a value from the given location. `location` is a string - ## prefixed with one of the following, which determines where - ## the value comes from: - runnableExamples: - assert getValue("env:FOO") == getEnv("FOO") - assert getValue("embed:someval") == "someval" - if location.startsWith("env:"): - getEnv(location.substr("env:".len)) - elif location.startsWith("embed:"): - location.substr("embed:".len) - else: - raise ValueError.newException("Unknown variable ref type") - -proc sendEmail*(toEmail, subject, text: string) {.async, raises: [CatchableError].} = - when usepostmark: - let data = $(%* { - "From": fromEmail, - "To": toEmail, - "Subject": subject, - "MessageStream": "outbound", - "TextBody": text, - }) - var headers = HttpTable.init() - headers.add("Accept", "application/json") - headers.add("Content-Type", "application/json") - headers.add("X-Postmark-Server-Token", POSTMARK_API_KEY.valueRef) - let (code, res) = await request("https://api.postmarkapp.com/email", MethodPost, data, headers = headers) - if code != 200: - try: - error "Error sending email: " & $res - except: - discard - raise CatchableError.newException("Email sending failed") - else: - # logging only - info "EMAIL FAKE SENDER:\nFrom: " & fromEmail & "\nTo: " & toEmail & "\nSubject: " & subject & "\n\n" & text & "\n------------------------------------" - stderr.flushFile() diff --git a/src/bucketsrelay/netstring.nim b/src/bucketsrelay/netstring.nim deleted file mode 100644 index a12b1f5..0000000 --- a/src/bucketsrelay/netstring.nim +++ /dev/null @@ -1,116 +0,0 @@ -import std/deques -import std/strformat -import std/strutils - -type - NSDecoderState = enum - LookingForNumber, - ReadingData, - LookingForComma, - NetstringDecoder* = object - buf: string - expectedLen: int - state: NSDecoderState - maxlen: int - output: Deque[string] - terminalChar*: char - -const - COLONCHAR = ':' - TERMINALCHAR = ',' - DEFAULTMAXLEN = 1_000_000 - -proc nsencode*(msg:string, terminalChar = TERMINALCHAR):string {.inline.} = - $msg.len & COLONCHAR & msg & terminalChar - -proc newNetstringDecoder*(terminalChar = TERMINALCHAR):NetstringDecoder = - result.output = initDeque[string]() - result.terminalChar = terminalChar - result.maxlen = DEFAULTMAXLEN - -when defined(testmode): - proc reset*(p: var NetstringDecoder) = - ## Reset the parser. For testing only. - p.buf = "" - p.expectedLen = 0 - p.state = LookingForNumber - -proc `maxlen=`*(p: var NetstringDecoder, length:int) = - ## Set the maximum message length - p.maxlen = length - -proc `len`*(p: var NetstringDecoder):int = - p.output.len - -proc consume*(p: var NetstringDecoder, data:string) = - ## Send some netstring data (perhaps incomplete as yet) - var cursor:int = 0 - while cursor < data.len: - case p.state: - of LookingForNumber: - let ch = data[cursor] - cursor.inc() - case ch - of '0'..'9': - p.buf.add(ch) - if p.buf.len == 2 and p.buf[0] == '0': - raise newException(ValueError, &"Length may not start with 0") - if p.maxlen != 0: - if p.buf.parseInt() > p.maxlen: - raise newException(ValueError, &"Message too long") - of COLONCHAR: - p.expectedLen = p.buf.parseInt() - p.buf = "" - if p.expectedLen == 0: - p.state = LookingForComma - else: - p.state = ReadingData - else: - raise newException(ValueError, &"Invalid netstring length char: {ch.repr}") - - of ReadingData: - let toread = p.expectedLen - int(p.buf.len) - var sidx = cursor - var eidx = sidx + toread - 1 - if eidx >= int(data.len): - eidx = int(data.len-1) - let snippet = data[sidx..eidx] - - p.buf.add(snippet) - cursor += toread - if int(p.buf.len) == p.expectedLen: - # message possibly complete - p.state = LookingForComma - of LookingForComma: - let ch = data[cursor] - cursor.inc() - if ch == p.terminalChar: - # message complete! - # Is this a copy? I'd rather it be a move - let msg = p.buf - p.buf = "" - p.output.addLast(msg) - p.state = LookingForNumber - else: - raise newException(ValueError, &"Missing terminal comma") - -proc bytesToRead*(p: var NetstringDecoder): int = - ## Return how many bytes the decoder needs to read - case p.state - of LookingForNumber: - return 1 - of LookingForComma: - return 1 - of ReadingData: - return p.expectedLen - int(p.buf.len) - -proc hasMessage*(p: var NetstringDecoder): bool = - return p.output.len > 0 - -proc nextMessage*(p: var NetstringDecoder): string = - ## Get the next decoded message - if p.output.len > 0: - p.output.popFirst() - else: - raise newException(IndexError, &"No message available") - diff --git a/src/bucketsrelay/proto.nim b/src/bucketsrelay/proto.nim deleted file mode 100644 index 89cdaf6..0000000 --- a/src/bucketsrelay/proto.nim +++ /dev/null @@ -1,403 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import std/base64 -import std/hashes -import std/logging -import std/options -import std/sets; export sets -import std/strformat -import std/strutils -import std/tables - -import ./common - -import libsodium/sodium -import ndb/sqlite - -template TODO*(msg: string) = - when defined(release): - {.error: msg .} - -type - PublicKey* = distinct string - SecretKey* = distinct string - - KeyPair* = tuple - pk: PublicKey - sk: SecretKey - - ## Relay event types - EventKind* = enum - Who = "?" - Authenticated = "+" - Connected = "c" - Disconnected = "x" - Data = "d" - Entered = ">" - Exited = "^" - ErrorEvent = "E" - - ## RelayEvent error types - ErrorCode* = enum - Generic = 0 - DestNotPresent - - ## Relay events -- server to client message - RelayEvent* = object - case kind*: EventKind - of Who: - who_challenge*: string - of Authenticated: - discard - of Connected: - conn_pubkey*: PublicKey - of Disconnected: - dcon_pubkey*: PublicKey - of Data: - data*: string - sender_pubkey*: PublicKey - of Entered: - entered_pubkey*: PublicKey - of Exited: - exited_pubkey*: PublicKey - of ErrorEvent: - err_code*: ErrorCode - err_message*: string - - ## Relay command types - CommandKind* = enum - Iam = "i" - Connect = "c" - Disconnect = "x" - SendData = "d" - - ## Relay command - client to server message - RelayCommand* = object - case kind*: CommandKind - of Iam: - iam_signature*: string - iam_pubkey*: PublicKey - of Connect: - conn_pubkey*: PublicKey - of Disconnect: - dcon_pubkey*: PublicKey - of SendData: - send_data*: string - dest_pubkey*: PublicKey - - RelayConnection*[T] = ref object - challenge: string - pubkey*: PublicKey - channel*: string - peer_connections: HashSet[PublicKey] - sender*: T - - Relay*[T] = ref object - conns: TableRef[PublicKey, RelayConnection[T]] - channels: TableRef[string, HashSet[PublicKey]] - conn_requests: TableRef[PublicKey, HashSet[PublicKey]] - db: DbConn - - RelayErr* = object of CatchableError - -proc newRelay*[T](): Relay[T] = - new(result) - result.conns = newTable[PublicKey, RelayConnection[T]]() - result.channels = newTable[string, HashSet[PublicKey]]() - result.conn_requests = newTable[PublicKey, HashSet[PublicKey]]() - -proc `$`*(a: PublicKey): string = - a.string.encode() - -proc abbr*(s: string, size = 6): string = - if s.len > size: - result.add s.substr(0, size) & "..." - else: - result.add(s) - -proc abbr*(a: PublicKey): string = - a.string.encode().abbr - -proc `$`*(conn: RelayConnection): string = - result.add "[RConn " - if conn.pubkey.string == "": - result.add "----------" - else: - result.add conn.pubkey.abbr - result.add "]" - -proc `==`*(a, b: PublicKey): bool {.borrow.} - -proc hash*(p: PublicKey): Hash {.borrow.} - -proc `==`*(a, b: RelayEvent): bool = - if a.kind != b.kind: - return false - else: - case a.kind - of Who: - return a.who_challenge == b.who_challenge - of Authenticated: - return true - of Connected: - return a.conn_pubkey == b.conn_pubkey - of Disconnected: - return a.dcon_pubkey == b.dcon_pubkey - of Data: - return a.sender_pubkey == b.sender_pubkey and a.data == b.data - of Entered: - return a.entered_pubkey == b.entered_pubkey - of Exited: - return a.exited_pubkey == b.exited_pubkey - of ErrorEvent: - return a.err_message == b.err_message - -proc `==`*(a, b: RelayCommand): bool = - if a.kind != b.kind: - return false - else: - case a.kind: - of Iam: - return a.iam_signature == b.iam_signature and a.iam_pubkey == b.iam_pubkey - of Connect: - return a.conn_pubkey == b.conn_pubkey - of Disconnect: - return a.dcon_pubkey == b.dcon_pubkey - of SendData: - return a.send_data == b.send_data and a.dest_pubkey == b.dest_pubkey - -proc `$`*(ev: RelayEvent): string = - result.add "(" - case ev.kind - of Who: - result.add "Who challenge=" & ev.who_challenge.encode().abbr - of Authenticated: - result.add "Authenticated" - of Connected: - result.add "Connected " & ev.conn_pubkey.abbr - of Disconnected: - result.add "Disconnected " & ev.dcon_pubkey.abbr - of Data: - result.add "Data " & ev.sender_pubkey.abbr & " data=" & $ev.data.len - of Entered: - result.add "Entered " & ev.entered_pubkey.abbr - of Exited: - result.add "Exited " & ev.exited_pubkey.abbr - of ErrorEvent: - result.add "Error " & ev.err_message - result.add ")" - -template dbg*(ev: RelayEvent): string = $ev - -proc `$`*(cmd: RelayCommand): string = - result.add "(" - case cmd.kind - of Iam: - result.add &"Iam {cmd.iam_pubkey.abbr} sig={cmd.iam_signature.encode.abbr}" - of Connect: - result.add &"Connect {cmd.conn_pubkey.abbr}" - of Disconnect: - result.add &"Disconnect {cmd.dcon_pubkey.abbr}" - of SendData: - result.add &"SendData {cmd.dest_pubkey.abbr} data={cmd.send_data.len}" - result.add ")" - -template dbg*(cmd: RelayCommand): string = $cmd - -when defined(testmode): - # proc dump*(relay: Relay): string = - # for row in relay.db.getAllRows(sql"SELECT * FROM clients"): - # result.add $row & "\l" - # for row in relay.db.getAllRows(sql"SELECT * FROM pending_conns"): - # result.add $row & "\l" - - proc testmode_conns*[T](relay: Relay[T]): TableRef[PublicKey, RelayConnection[T]] = - relay.conns - - proc testmode_conns*(conn: RelayConnection): HashSet[PublicKey] = - conn.peer_connections - -proc newRelayConnection*[T](sender: T): RelayConnection[T] = - new(result) - result.sender = sender - result.peer_connections = initHashSet[PublicKey]() - -template sendEvent(conn: RelayConnection, ev: RelayEvent) = - case ev.kind - of Data: - when relayverbose: - debug $conn & "< " & ev.dbg - else: - discard - else: - debug $conn & "< " & ev.dbg - conn.sender.sendEvent(ev) - -template sendError(conn: RelayConnection, message: string) = - debug $conn & "< error: " & message - conn.sender.sendEvent(RelayEvent( - kind: ErrorEvent, - err_message: message, - )) - -proc initAuth*[T](relay: var Relay[T], client: T, channel = ""): RelayConnection[T] = - ## Ask the client to authenticate itself. After it succeeds, it will - ## be added as a connected client. - ## If channel is provided, this is the channel to which this client - ## will be subscribed for Entered/Exited events. - var conn = newRelayConnection[T](client) - conn.challenge = randombytes(32) - conn.channel = channel - conn.sendEvent(RelayEvent( - kind: Who, - who_challenge: conn.challenge, - )) - return conn - -proc connectPair[T](a, b: var RelayConnection[T]) = - ## Connect two clients together - a.peer_connections.incl(b.pubkey) - b.peer_connections.incl(a.pubkey) - a.sendEvent(RelayEvent(kind: Connected, conn_pubkey: b.pubkey)) - b.sendEvent(RelayEvent(kind: Connected, conn_pubkey: a.pubkey)) - -proc addConnRequest(relay: var Relay, alice_pubkey: PublicKey, bob_pubkey: PublicKey) = - ## Add or fulfil a connection request from alice to bob - var alice = relay.conns[alice_pubkey] - relay.conn_requests.mgetOrPut(alice_pubkey, initHashSet[PublicKey]()).incl(bob_pubkey) - if bob_pubkey in alice.peer_connections: - # They're already connected - return - var bob_requests = relay.conn_requests.getOrDefault(bob_pubkey, initHashSet[PublicKey]()) - if alice_pubkey in bob_requests: - # They both want to connect! - var bob = relay.conns[bob_pubkey] - connectPair(alice, bob) - -proc removeConnRequest(relay: var Relay, alice_pubkey: PublicKey, bob_pubkey: PublicKey) = - ## Remove a connection request from alice to bob - relay.conn_requests.mgetOrPut(alice_pubkey, initHashSet[PublicKey]()).excl(bob_pubkey) - -proc removeConnection*[T](relay: var Relay[T], conn: RelayConnection[T]) = - ## Remove a conn from the relay if it exists. - if conn.pubkey in relay.conn_requests: - relay.conn_requests.del(conn.pubkey) - # disconnect all peer connections - var commands: seq[RelayCommand] - for other_pubkey in conn.peer_connections: - commands.add(RelayCommand( - kind: Disconnect, - dcon_pubkey: other_pubkey, - )) - for command in commands: - relay.handleCommand(conn, command) - # notify the channel (if any) - if conn.channel != "": - relay.channels.mgetOrPut(conn.channel, initHashSet[PublicKey]()).excl(conn.pubkey) - for other in relay.channels[conn.channel].items: - if other in relay.conns: - relay.conns[other].sendEvent(RelayEvent( - kind: Exited, - exited_pubkey: conn.pubkey, - )) - # remove it from the registry - if conn.pubkey in relay.conns: - relay.conns.del(conn.pubkey) - debug &"{conn} gone" - -proc handleCommand*[T](relay: var Relay[T], conn: RelayConnection[T], command: RelayCommand) = - case command.kind - of SendData: - when defined(verbose): - debug &"{conn} > {command.dbg}" - else: - discard - else: - debug &"{conn} > {command.dbg}" - case command.kind - of Iam: - if conn.challenge == "": - conn.sendError "Authentication cannot proceed. Reconnect and try again." - try: - crypto_sign_verify_detached(command.iam_pubkey.string, conn.challenge, command.iam_signature) - except: - conn.challenge = "" # disable authentication - conn.sendError "Invalid signature" - return - conn.pubkey = command.iam_pubkey - if conn.pubkey in relay.conns: - # this pubkey is already connected; boot the old conn - relay.removeConnection(relay.conns[conn.pubkey]) - relay.conns[conn.pubkey] = conn - conn.sendEvent(RelayEvent( - kind: Authenticated, - )) - if conn.channel != "": - relay.channels.mgetOrPut(conn.channel, initHashSet[PublicKey]()).incl(conn.pubkey) - for other in relay.channels[conn.channel].items: - if other != conn.pubkey: - conn.sendEvent(RelayEvent( - kind: Entered, - entered_pubkey: other, - )) - if other in relay.conns: - relay.conns[other].sendEvent(RelayEvent( - kind: Entered, - entered_pubkey: conn.pubkey, - )) - of Connect: - if conn.pubkey.string == "": - conn.sendError "Connection forbidden" - elif command.conn_pubkey.string == conn.pubkey.string: - conn.sendError "Can't connect to self" - else: - relay.addConnRequest(conn.pubkey, command.conn_pubkey) - of Disconnect: - relay.removeConnRequest(conn.pubkey, command.dcon_pubkey) - if command.dcon_pubkey in conn.peer_connections: - if command.dcon_pubkey in relay.conns: - var other = relay.conns[command.dcon_pubkey] - # disassociate - other.peer_connections.excl(conn.pubkey) - conn.peer_connections.excl(other.pubkey) - # notify - conn.sendEvent(RelayEvent( - kind: Disconnected, - dcon_pubkey: other.pubkey, - )) - other.sendEvent(RelayEvent( - kind: Disconnected, - dcon_pubkey: conn.pubkey, - )) - of SendData: - if conn.pubkey.string == "": - conn.sendError "Sending forbidden" - elif command.dest_pubkey notin conn.peer_connections: - conn.sendError "No such connection" - else: - if command.dest_pubkey notin relay.conns: - conn.sendEvent(RelayEvent( - kind: ErrorEvent, - err_message: "Other side disconnected", - )) - else: - let remote = relay.conns[command.dest_pubkey] - remote.sendEvent(RelayEvent( - kind: Data, - sender_pubkey: conn.pubkey, - data: command.send_data, - )) - -#------------------------------------------------------------ -# utilities -#------------------------------------------------------------ -proc genkeys*(): KeyPair = - let (pk, sk) = crypto_sign_keypair() - result = (pk.PublicKey, sk.SecretKey) - -proc sign*(key: SecretKey, message: string): string = - ## Sign a message with the given secret key - result = crypto_sign_detached(key.string, message) diff --git a/src/bucketsrelay/server.nim b/src/bucketsrelay/server.nim deleted file mode 100644 index 35a9f9e..0000000 --- a/src/bucketsrelay/server.nim +++ /dev/null @@ -1,922 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import std/base64 -import std/json -import std/logging -import std/mimetypes -import std/options; export options -import std/os -import std/sha1 -import std/sqlite3 -import std/strformat -import std/strutils -import std/tables - -import chronicles except debug, info, warn, error -import chronos -import httputils -import libsodium/sodium -import mustache -import ndb/sqlite -import stew/byteutils -import websock/extensions/compression/deflate -import websock/websock - -import ./common -import ./dbschema -import ./netstring -import ./proto -import ./stringproto -import ./mailer -import ./licenses - -type - WSClient = ref object - debugname*: string - ws: WSSession - user_id: int64 - ip: string - relayserver: RelayServer - eventQueue: AsyncQueue[RelayEvent] - - RelayHttpServer = ref object - case tls: bool - of true: - httpsServer: TlsHttpServer - of false: - httpServer: HttpServer - - RelayServer* = ref object - debugname*: string - nextid: int - relay: Relay[WSClient] - http: RelayHttpServer - mcontext*: proc(): mustache.Context - longrunservices: seq[Future[void]] - runningRequests: TableRef[int, HttpRequest] - when multiusermode: - pubkey*: string - dbfilename: string - updateSchema: bool - userdb: Option[DbConn] - elif singleusermode: - usernameHash: string - passwordHash: string - - NotFound* = object of CatchableError - WrongPassword* = object of CatchableError - DuplicateUser* = object of CatchableError - -const - partialsDir = currentSourcePath.parentDir.parentDir / "partials" - staticDir = currentSourcePath.parentDir.parentDir / "static" - -when multiusermode: - let - AUTH_LICENSE_PUBKEY* = getEnv("AUTH_LICENSE_PUBKEY", "") - LICENSE_HASH_SALT = getEnv("LICENSE_HASH_SALT", "yououghttochangethis") - -const versionSupport = static: - var jnode = %* { - "versions": [], - } - var authMethods = %* ["usernamepassword"] - when multiusermode: - authMethods.add newJString("v1license") - jnode["versions"].add(%* { - "version": "1", - "authMethods": authMethods, - }) - $jnode - -var mimedb = newMimetypes() - -when defined(release) or defined(embedassets): - # embed templates and static data - const partialsData = static: - var tab = initTable[string, string]() - echo "Embedding templates from ", partialsDir - for item in walkDir(partialsDir): - if item.kind == pcFile: - let - parts = item.path.splitFile - name = parts.name - echo " + ", name, ": ", item.path - tab[name] = slurp(item.path) - tab - proc addDefaultContext*(c: var Context) = - c.searchTable(partialsData) - - const staticData = static: - var tab = initTable[string, string]() - echo "Embedding static data from ", staticDir - for item in walkDir(staticDir): - if item.kind == pcFile: - let name = "/" & item.path.extractFilename - echo " + ", name, ": ", item.path - tab[name] = slurp(item.path) - tab - - template readStaticFile(path: string): string = - staticData[path] -else: - # read templates and static data from disk - proc addDefaultContext*(c: var Context) = - c.searchDirs(@[partialsDir]) - - proc readStaticFile(path: string): string = - let fullpath = normalizedPath(staticDir / path) - if fullpath.isRelativeTo(staticDir) and fullpath.fileExists(): - readFile(fullpath) - else: - raise NotFound.newException("No such file: " & path) - -template logname*(rs: RelayServer): string = - "(" & rs.debugname & ") " - -proc start*(rhs: RelayHttpServer) = - case rhs.tls - of true: - rhs.httpsServer.start() - of false: - rhs.httpServer.start() - -proc stop*(rhs: RelayHttpServer) = - case rhs.tls - of true: - rhs.httpsServer.stop() - of false: - rhs.httpServer.stop() - -proc close*(rhs: RelayHttpServer) = - case rhs.tls - of true: - rhs.httpsServer.close() - of false: - rhs.httpServer.close() - -proc join*(rhs: RelayHttpServer): Future[void] = - case rhs.tls - of true: - rhs.httpsServer.join() - of false: - rhs.httpServer.join() - -proc `handler=`*(rhs: RelayHttpServer, handler: HttpAsyncCallback) = - case rhs.tls - of true: - rhs.httpsServer.handler = handler - of false: - rhs.httpServer.handler = handler - -#------------------------------------------------------------- -# netstrings -#------------------------------------------------------------- -const - COLONCHAR = ':' - TERMINALCHAR = ',' - DEFAULTMAXLEN = 1_000_000 - -proc nsencode*(msg:string, terminalChar = TERMINALCHAR):string {.inline.} = - $msg.len & COLONCHAR & msg & terminalChar - -#------------------------------------------------------------- -# User management -#------------------------------------------------------------- -type - LowerString* = distinct string - -converter toLowercase*(s: string): LowerString = s.toLower().LowerString -converter toString*(s: LowerString): string = s.string - -const userdbSchema = [ - ("initial", @[ - """CREATE TABLE IF NOT EXISTS iplog ( - day TEXT NOT NULL, - ip TEXT NOT NULL, - bytes_sent INT DEFAULT 0, - bytes_recv INT DEFAULT 0, - PRIMARY KEY (day, ip) - )""", - """CREATE TABLE IF NOT EXISTS user ( - id INTEGER PRIMARY KEY, - created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - email TEXT NOT NULL, - pwhash TEXT NOT NULL, - emailverified TINYINT DEFAULT 0, - blocked TINYINT DEFAULT 0, - recentlicensehash TEXT DEFAULT '', - UNIQUE(email) - )""", - """CREATE TABLE IF NOT EXISTS userlog ( - day TEXT NOT NULL, - user_id INTEGER, - bytes_sent INT DEFAULT 0, - bytes_recv INT DEFAULT 0, - PRIMARY KEY (day, user_id), - FOREIGN KEY (user_id) REFERENCES user(id) - )""", - """CREATE TABLE IF NOT EXISTS emailtoken ( - id INTEGER PRIMARY KEY, - expires TIMESTAMP DEFAULT (datetime('now', '+1 hour')), - user_id INTEGER NOT NULL, - token TEXT, - FOREIGN KEY (user_id) REFERENCES user(id) - )""", - """CREATE TABLE IF NOT EXISTS pwreset ( - id INTEGER PRIMARY KEY, - expires TIMESTAMP DEFAULT (datetime('now', '+1 hour')), - user_id INTEGER NOT NULL, - token TEXT, - FOREIGN KEY (user_id) REFERENCES user(id) - )""", - """CREATE TABLE IF NOT EXISTS disabledlicense ( - licensehash TEXT PRIMARY KEY - )""", - ]) -] - -template boolVal*(d: DbValue): bool = - d.i == 1 - -proc db*(rs: RelayServer): DbConn {.multiuseronly.} = - ## Get the user-data database for this server - if not rs.userdb.isSome: - var db = open(rs.dbfilename, "", "", "") - discard db.busy_timeout(1000) - db.exec(sql"PRAGMA foreign_keys = ON") - if rs.updateSchema: - db.upgradeSchema(userdbSchema) - rs.userdb = some(db) - rs.userdb.get() - -proc newRelayServer*(dbfilename: string, updateSchema = true, pubkey = ""): RelayServer {.multiuseronly.} = - ## Make a new multi-user relay server - new(result) - result.relay = newRelay[WSClient]() - result.mcontext = proc(): Context = - result = newContext() - result.addDefaultContext() - result.dbfilename = dbfilename - result.updateSchema = updateSchema - result.pubkey = pubkey - result.runningRequests = newTable[int, HttpRequest]() - discard result.db() - result.debugname = "RelayServer" & nextDebugName() - -proc stop*(rs: RelayServer) {.async.} = - for fut in rs.longrunservices: - await fut.cancelAndWait() - for req in rs.runningRequests.values(): - try: - await req.sendError(Http503) - except: - discard - -proc newRelayServer*(username, password: string): RelayServer {.singleuseronly.} = - ## Make a new single-user relay server - new(result) - result.relay = newRelay[WSClient]() - result.mcontext = proc(): Context = - result = newContext() - result.addDefaultContext() - result.usernameHash = hash_password(username) - result.passwordHash = hash_password(password) - result.runningRequests = newTable[int, HttpRequest]() - -proc get_user_id*(rs: RelayServer, email: LowerString): int64 {.multiuseronly.} = - ## Get a user's id from their email - try: - rs.db.getRow(sql"SELECT id FROM user WHERE email=?", email).get()[0].i - except: - raise NotFound.newException("No such user") - -proc register_user*(rs: RelayServer, email: LowerString, password: string): int64 {.multiuseronly.} = - ## Register a user with a password - let pwhash = try: - hash_password(password) - except: - logging.error "Error hashing password", getCurrentExceptionMsg() - raise CatchableError.newException("Crypto error") - try: - result = rs.db.insertID(sql"INSERT INTO user (email, pwhash) VALUES (?,?)", - email, pwhash) - except: - logging.error rs.logname, "failed registering", getCurrentExceptionMsg() - raise DuplicateUser.newException("Account already exists") - -proc password_auth*(rs: RelayServer, email: LowerString, password: string): int64 {.multiuseronly.} = - ## Return the userid if the password is correct, else raise an exception - let orow = rs.db.getRow(sql"SELECT id, pwhash FROM user WHERE email = ?", email) - if orow.isNone: - raise NotFound.newException("No such user") - else: - let row = orow.get() - let user_id = row[0].i - let pwhash = row[1].s - if verify_password(pwhash, password): - return user_id - raise WrongPassword.newException("Wrong password") - -proc password_auth*(rs: RelayServer, email: LowerString, password: string): int64 {.singleuseronly.} = - ## Return 1 if the password is correct, or else raise an exception - if not verify_password(rs.usernameHash, email): - raise NotFound.newException("No such user") - if verify_password(rs.passwordHash, password): - return 1 - raise WrongPassword.newException("Wrong password") - -proc strHash(lic: BucketsV1License, email: LowerString): string {.multiuseronly.} = - $secureHash( - $secureHash(LICENSE_HASH_SALT) & $secureHash(email.string) & $secureHash($lic) - ) - -proc license_auth*(rs: RelayServer, license: string): int64 {.multiuseronly.} = - ## Return the userid if the license is valid, else raise an error - if rs.pubkey == "": - raise WrongPassword.newException("License auth not supported") - let lic = license.unformatLicense() - if lic.verify(rs.pubkey) == false: - raise WrongPassword.newException("Invalid license") - let email = lic.extractEmail().toLowercase() - let lichash = strHash(lic, email) - let disabled = rs.db.getRow(sql"SELECT count(*) FROM disabledlicense WHERE licensehash=?", lichash).get()[0].i - if disabled != 0: - raise WrongPassword.newException("License disabled") - try: - result = rs.get_user_id(email) - except NotFound: - # create the user - result = rs.db.insertID(sql"INSERT INTO user (email, pwhash, emailverified) VALUES (?, '', 1)", email) - # disable former passwords - rs.db.exec(sql"UPDATE user SET pwhash='' WHERE recentlicensehash='' AND id=?", result) - # add license - rs.db.exec(sql"UPDATE user SET recentlicensehash=?, emailverified=1 WHERE id=?", lichash, result) - -proc is_email_verified*(rs: RelayServer, user_id: int64): bool {.multiuseronly.} = - ## Return true if the user has verified their email address - let row = rs.db.getRow(sql"SELECT emailverified FROM user WHERE id=?", user_id) - if row.isSome: - return row.get()[0].boolVal - -proc generate_email_verification_token*(rs: RelayServer, user_id: int64): string {.multiuseronly.} = - ## Generate a string to be emailed to a user that when returned - ## to `use_email_verification_token` will mark that user's email - ## as verified. - result = randombytes(16).toHex() - rs.db.exec(sql"INSERT INTO emailtoken (user_id, token) VALUES (?, ?)", - user_id, result) - rs.db.exec(sql"""DELETE FROM emailtoken WHERE id NOT IN - (SELECT id FROM emailtoken WHERE user_id=? ORDER BY id DESC LIMIT 3)""", - user_id) - -proc use_email_verification_token*(rs: RelayServer, user_id: int64, token: string): bool {.multiuseronly.} = - ## Verify a user's email address via token. Return `true` if they are now - ## verified and `false` if they are not. - try: - let row = rs.db.getRow(sql"SELECT count(*) FROM emailtoken WHERE user_id=? AND token=?", - user_id, token).get() - if row[0].i == 1: - rs.db.exec(sql"DELETE FROM emailtoken WHERE user_id = ?", user_id) - rs.db.exec(sql"UPDATE user SET emailverified=1 WHERE id=?", user_id) - except: - discard - return rs.is_email_verified(user_id) - -proc generate_password_reset_token*(rs: RelayServer, email: LowerString): string {.multiuseronly.} = - ## Generate a string token to be emailed to a user that can be used - ## to set their password. - result = randombytes(16).toHex() - let user_id = rs.get_user_id(email) - rs.db.exec(sql"INSERT INTO pwreset (user_id, token) VALUES (?, ?)", - user_id, result) - rs.db.exec(sql"""DELETE FROM pwreset WHERE id NOT IN - (SELECT id FROM pwreset WHERE user_id=? ORDER BY id DESC LIMIT 3)""", - user_id) - -proc delete_old_pwreset_tokens(rs: RelayServer) {.multiuseronly.} = - rs.db.exec(sql"DELETE FROM pwreset WHERE expires < datetime('now')") - -proc user_for_password_reset_token*(rs: RelayServer, token: string): Option[int64] {.multiuseronly.} = - ## Get the user associated with a password reset token, if one exists. - rs.delete_old_pwreset_tokens() - try: - let row = rs.db.getRow(sql"SELECT user_id FROM pwreset WHERE token = ?", token).get() - return some(row[0].i) - except: - discard - -proc update_password_with_token*(rs: RelayServer, token: string, newpassword: string) {.multiuseronly.} = - ## Update a user's password using a password-reset token - let o_user_id = rs.user_for_password_reset_token(token) - if o_user_id.isNone: - raise NotFound.newException("Invalid token") - let user_id = o_user_id.get() - let pwhash = hash_password(newpassword) - rs.db.exec(sql"DELETE FROM pwreset WHERE user_id = ?", user_id) - rs.db.exec(sql"UPDATE user SET pwhash=? WHERE id=?", pwhash, user_id) - -proc block_user*(rs: RelayServer, user_id: int64) {.multiuseronly.} = - ## Block a user's access to the relay - rs.db.exec(sql"UPDATE user SET blocked=1 WHERE id=?", user_id) - -proc block_user*(rs: RelayServer, email: LowerString) {.multiuseronly.} = - ## Block a user's access to the relay - rs.block_user(rs.get_user_id(email)) - -proc unblock_user*(rs: RelayServer, user_id: int64) {.multiuseronly.} = - ## Unblock a user's access to the relay - rs.db.exec(sql"UPDATE user SET blocked=0 WHERE id=?", user_id) - -proc unblock_user*(rs: RelayServer, email: LowerString) {.multiuseronly.} = - ## Unblock a user's access to the relay - rs.unblock_user(rs.get_user_id(email)) - -proc disable_most_recently_used_license*(rs: RelayServer, uid: int64) {.multiuseronly.} = - ## Block the most recently-used license for a user - let lichash = try: - rs.db.getRow(sql"SELECT recentlicensehash FROM user WHERE id=?", uid).get()[0].s - except: - raise NotFound.newException("No such user") - if lichash == "": - raise NotFound.newException("User has not authenticated via license") - rs.db.exec(sql"INSERT INTO disabledlicense (licensehash) VALUES (?) ON CONFLICT DO NOTHING", lichash) - -proc can_use_relay*(rs: RelayServer, user_id: int64): bool {.multiuseronly.} = - ## Return true if the user is allowed to use the relay - ## because their email is verified and they are not blocked - try: - return rs.db.getRow(sql"SELECT emailverified AND not(blocked) FROM user WHERE id=?", user_id).get()[0].boolVal - except: - discard - -type - DataSentRecv* = tuple - sent: int - recv: int - -proc log_user_data*(rs: RelayServer, user_id: int64, dlen: DataSentRecv) {.multiuseronly.} = - rs.db.exec(sql"""INSERT INTO userlog (day, user_id, bytes_sent, bytes_recv) - VALUES (date(), ?, ?, ?) - ON CONFLICT (day, user_id) DO - UPDATE SET - bytes_sent = bytes_sent + excluded.bytes_sent, - bytes_recv = bytes_recv + excluded.bytes_recv - """, user_id, dlen.sent, dlen.recv) - -when multiusermode: - template log_user_data_sent*(rs: RelayServer, user_id: int64, dlen: int) = - rs.log_user_data(user_id, (dlen, 0)) - - template log_user_data_recv*(rs: RelayServer, user_id: int64, dlen: int) = - rs.log_user_data(user_id, (0, dlen)) - -proc data_by_user*(rs: RelayServer, user_id: int64, days = 1): DataSentRecv {.multiuseronly.} = - let orow = rs.db.getRow(sql""" - SELECT - sum(bytes_sent), - sum(bytes_recv) - FROM - userlog - WHERE - user_id = ? - AND day >= date('now', '-' || ? || ' day') - """, user_id, days) - if orow.isSome: - let row = orow.get() - return (row[0].i.int, row[1].i.int) - -proc top_data_users*(rs: RelayServer, limit = 20, days = 7): seq[tuple[user: string, data: DataSentRecv]] {.multiuseronly.} = - let rows = rs.db.getAllRows(sql""" - SELECT - u.email, - sum(ll.bytes_sent), - sum(ll.bytes_recv), - sum(ll.bytes_sent + ll.bytes_recv) as total - FROM - userlog as ll - LEFT JOIN user AS u - ON ll.user_id = u.id - WHERE - ll.day >= date('now', '-' || ? || ' day') - GROUP BY 1 - ORDER BY total DESC - LIMIT ? - """, days, limit) - for row in rows: - result.add((row[0].s, (row[1].i.int, row[2].i.int))) - -proc log_ip_data*(rs: RelayServer, ip: string, dlen: DataSentRecv) {.multiuseronly.} = - rs.db.exec(sql"""INSERT INTO iplog (day, ip, bytes_sent, bytes_recv) - VALUES (date(), ?, ?, ?) - ON CONFLICT (day, ip) DO - UPDATE SET - bytes_sent = bytes_sent + excluded.bytes_sent, - bytes_recv = bytes_recv + excluded.bytes_recv - """, ip, dlen.sent, dlen.recv) - -when multiusermode: - template log_ip_data_sent*(rs: RelayServer, ip: string, dlen: int) = - rs.log_ip_data(ip, (dlen, 0)) - - template log_ip_data_recv*(rs: RelayServer, ip: string, dlen: int) = - rs.log_ip_data(ip, (0, dlen)) - -proc data_by_ip*(rs: RelayServer, ip: string, days = 1): DataSentRecv {.multiuseronly.} = - let orow = rs.db.getRow(sql""" - SELECT - sum(bytes_sent), - sum(bytes_recv) - FROM - iplog - WHERE - ip = ? - AND day >= date('now', '-' || ? || ' day') - """, ip, days) - if orow.isSome: - let row = orow.get() - return (row[0].i.int, row[1].i.int) - -proc top_data_ips*(rs: RelayServer, limit = 20, days = 7): seq[tuple[ip: string, data: DataSentRecv]] {.multiuseronly.} = - let rows = rs.db.getAllRows(sql""" - SELECT - ip, - sum(bytes_sent), - sum(bytes_recv), - sum(bytes_sent + bytes_recv) as total - FROM - iplog - WHERE - day >= date('now', '-' || ? || ' day') - GROUP BY 1 - ORDER BY total DESC - LIMIT ? - """, days, limit) - for row in rows: - result.add((row[0].s, (row[1].i.int, row[2].i.int))) - -proc delete_old_stats*(rs: RelayServer, keep_days = 90) {.gcsafe, multiuseronly.} = - ## Remote stats older than `keep_days` days - try: - info "Deleting stats older than " & $keep_days & "days" - except: - discard - {.gcsafe.}: - rs.db.exec(sql"DELETE FROM iplog WHERE day < date('now', '-' || ? || ' day')", keep_days) - rs.db.exec(sql"DELETE FROM userlog WHERE day < date('now', '-' || ? || ' day')", keep_days) - -proc clear_stat_loop(rs: RelayServer) {.async, multiuseronly.} = - while true: - await sleepAsync(24.hours) - rs.delete_old_stats() - -proc periodically_delete_old_stats*(rs: RelayServer) {.multiuseronly.} = - ## Delete old stats at a regular interval - rs.longrunservices.add(rs.clear_stat_loop()) - -#------------------------------------------------------------- -# Common HTTP helpers -#------------------------------------------------------------- - -proc ipAddress(request: HttpRequest): string = - ## Return the IP Address associated with this request - # # Forwarded (TODO) - # let forwarded = request.headers.getOrDefault("forwarded") - # if forwarded != "": - # return result - # True-Client-IP (cloudflare) - result = request.headers.getString("true-client-ip") - if result != "": - return result - # X-Real-IP (nginx) - result = request.headers.getString("x-real-ip") - if result != "": - return result - result = request.stream.writer.tsource.remoteAddress().host() - -proc sendHTML(req: HttpRequest, data: string) {.async.} = - var headers = HttpTable.init() - headers.add("Content-Type", "text/html") - await req.sendResponse(Http200, headers, data = data) - -#------------------------------------------------------------- -# Version 1 -#------------------------------------------------------------- - -template logname*(c: WSClient): string = - "(" & c.debugname & ") " - -proc sendEvent*(c: WSClient, ev: RelayEvent) = - ## Queue an event to a single ws client - c.eventQueue.addLastNoWait(ev) - -proc newWSClient(rs: RelayServer, ws: WSSession, user_id: int64, ip: string): WSClient = - new(result) - result.ws = ws - result.relayserver = rs - result.user_id = user_id - result.ip = ip - result.eventQueue = newAsyncQueue[RelayEvent]() - result.debugname = "WSClient" & nextDebugName() - -proc closeWait(c: WSClient) {.async.} = - c.ws = nil - -proc authenticate(rs: RelayServer, req: HttpRequest): int64 = - ## Perform HTTP basic authentication and return the - ## user id if correct. - let authorization = req.headers.getString("authorization") - let parts = authorization.strip().split(" ") - doAssert parts.len == 2, "Authorization header should have 2 items" - doAssert parts[0] == "Basic", "Only basic HTTP auth is supported" - let credentials = base64.decode(parts[1]).split(":", maxsplit = 1) - doAssert credentials.len == 2, "Must supply username and password" - let - username = credentials[0] - password = credentials[1] - try: - when multiusermode: - if rs.pubkey != "" and username == "_license": - return rs.license_auth(password) - else: - return rs.password_auth(username, password) - else: - return rs.password_auth(username, password) - except WrongPassword: - info rs.logname, "WrongPassword: " & getCurrentExceptionMsg() - raise - except: - logging.error rs.logname, "Error during authentication: " & getCurrentExceptionMsg() - raise - -when defined(testmode): - var allHttpRequests*: seq[HttpRequest] - -proc handleRequestRelayV1(rs: RelayServer, req: HttpRequest) {.async, gcsafe.} = - # Perform HTTP basic authenciation - {.gcsafe.}: - vlog rs.logname, "starting..." - let user_id = block: - try: - vlog "authenticating..." - rs.authenticate(req) - except: - logging.error "Error authenticating: " & getCurrentExceptionMsg() - await req.sendError(Http403) - return - vlog rs.logname, "auth ok" - when multiusermode: - if not rs.can_use_relay(user_id): - info rs.logname, "Blocked from relay: " & $user_id - await req.sendError(Http403) - return - let ip = req.ipAddress() - - # Upgrade protocol to websockets - var relayconn: RelayConnection[WSClient] - try: - let deflateFactory = deflateFactory() - let server = WSServer.new(factories = [deflateFactory]) - vlog rs.logname, "opening WS..." - var ws = await server.handleRequest(req) - if ws.readyState != Open: - raise ValueError.newException("Failed to open websocket connection") - - var wsclient = newWSClient(rs, ws, user_id, ip) - wsclient.debugname = rs.debugname & "." & wsclient.debugname - vlog wsclient.logname, "starting..." - try: - relayconn = rs.relay.initAuth(wsclient, channel = $user_id) - var decoder = newNetstringDecoder() - var msgfut: Future[seq[byte]] - var evfut: Future[RelayEvent] - while ws.readyState != ReadyState.Closed: - if msgfut.isNil: - msgfut = ws.recvMsg() - if evfut.isNil: - evfut = wsclient.eventQueue.get() - await (msgfut or evfut) - if evfut.finished: - let ev = await evfut - evfut = nil - let msg = nsencode(dumps(ev)) - when multiusermode: - rs.log_user_data_recv(wsclient.user_id, msg.len) - rs.log_ip_data_recv(wsclient.ip, msg.len) - await ws.send(msg.toBytes, Opcode.Binary) - if msgfut.finished: - let buff = try: - await msgfut - except: - vlog wsclient.logname, "error getting msg: ", getCurrentExceptionMsg() - break - msgfut = nil - when multiusermode: - rs.log_user_data_sent(user_id, buff.len) - rs.log_ip_data_sent(ip, buff.len) - decoder.consume(string.fromBytes(buff)) - while decoder.hasMessage(): - let cmd = loadsRelayCommand(decoder.nextMessage()) - rs.relay.handleCommand(relayconn, cmd) - finally: - await wsclient.closeWait() - except WSClosedError: - discard - except WebSocketError as exc: - error rs.logname, "WebSocketError: ", exc.msg - await req.sendError(Http400) - except Exception as exc: - error rs.logname, "connection failed: ", exc.msg - await req.sendError(Http400) - finally: - if not relayconn.isNil: - rs.relay.removeConnection(relayconn) - -proc handleRequestAuthV1(rs: RelayServer, req: HttpRequest) {.async, multiuseronly.} = - ## Handle user registration activities - # Upgrade protocol to websockets - {.gcsafe.}: - try: - vlog "[ws.auth] starting..." - let deflateFactory = deflateFactory() - let server = WSServer.new(factories = [deflateFactory]) - var ws = await server.handleRequest(req) - if ws.readyState != Open: - raise ValueError.newException("Failed to open websocket connection") - - while ws.readyState != ReadyState.Closed: - let buff = try: - await ws.recvMsg() - except: - break - let msg = string.fromBytes(buff) - let data = parseJson(msg) - var resp = newJObject() - resp["id"] = data["id"] - try: - let command = data["command"].getStr() - vlog "[ws.auth] command: " & command - let args = data["args"] - case command - of "register": - let email = args["email"].getStr() - let password = args["password"].getStr() - vlog "[ws.auth] register_user" - let user_id = rs.register_user(email, password) - vlog "[ws.auth] generate_email_verification_token" - let email_token = rs.generate_email_verification_token(user_id) - try: - vlog "[ws.auth] sendEmail" - await sendEmail(email, "Buckets Relay - Email Verification", - &"Use this code to verify your email address:\n\n{email_token}") - resp["response"] = newJBool(true) - except: - resp["error"] = newJString("Failed to send email") - of "sendVerify": - let email = args["email"].getStr() - let user_id = rs.get_user_id(email) - let email_token = rs.generate_email_verification_token(user_id) - try: - await sendEmail(email, "Buckets Relay - Email Verification", - &"Use this code to verify your email address:\n\n{email_token}") - resp["response"] = newJBool(true) - except: - resp["error"] = newJString("Failed to send email") - of "verify": - let email = args["email"].getStr() - let code = args["code"].getStr() - let user_id = rs.get_user_id(email) - resp["response"] = newJBool(rs.use_email_verification_token(user_id, code)) - of "resetPassword": - let email = args["email"].getStr() - let pw_token = rs.generate_password_reset_token(email) - try: - await sendEmail(email, "Buckets Relay - Password Reset", - &"Use this code to change your password:\n\n{pw_token}") - except: - resp["error"] = newJString("Failed to send email") - of "updatePassword": - let pw_token = args["token"].getStr() - let new_password = args["new_password"].getStr() - rs.update_password_with_token(pw_token, new_password) - else: - resp["error"] = newJString("Unknown command"); - except NotFound: - vlog "[ws.auth] not found" - resp["error"] = newJString("Not found") - except DuplicateUser: - vlog "[ws.auth] duplicate user" - resp["error"] = newJString("Account already exists") - except WrongPassword: - vlog "[ws.auth] wrong password" - resp["error"] = newJString("Wrong password") - except Exception as exc: - vlog "[ws.auth] unexpected error" - error exc.msg - resp["error"] = newJString("Unexpected error") - finally: - vlog "[ws.auth] sending response" - await ws.send($resp) - except WSClosedError: - vlog "[ws.auth] WSClosedError" - except WebSocketError as exc: - logging.error "relay/server: WebSocketError: " & exc.msg - await req.sendError(Http400) - except Exception as exc: - logging.error "relay/server: connection failed: " & exc.msg - await req.sendError(Http400) - -proc handleRequestV1(rs: RelayServer, req: HttpRequest, subpath: string) {.async, gcsafe.} = - ## Version 1 request handling - {.gcsafe.}: - var path = req.uri.path.substr(subpath.len) - if path == "": path = "/" - if path == "/relay": - await rs.handleRequestRelayV1(req) - elif path == "/auth": - when multiusermode: - await rs.handleRequestAuthV1(req) - else: - await req.sendError(Http404) - elif path == "/": - let ctx = rs.mcontext() - ctx["openregistration"] = multiusermode - let rendered = render("{{>index}}", ctx) - await req.sendHTML(rendered) - else: - await req.sendError(Http404) - -#------------------------------------------------------------- -# HTTP routing common to all versions -#------------------------------------------------------------- - -proc handleRequest*(rs: RelayServer, req: HttpRequest): Future[void] {.async, gcsafe.} = - ## Handle a relay server websocket request. - {.gcsafe.}: - let reqid = rs.nextid - rs.nextid.inc() - rs.runningRequests[reqid] = req - defer: rs.runningRequests.del(reqid) - when defined(testmode): - allHttpRequests.add(req) - defer: - allHttpRequests.delete(allHttpRequests.find(req)) - let path = req.uri.path - if path.startsWith("/v1/"): - await rs.handleRequestV1(req, "/v1") - elif path == "/versions": - await req.sendResponse(Http200, data = versionSupport) - elif path == "/": - var headers = HttpTable.init() - headers.add("Location", "/v1/") - await req.sendResponse(Http307, headers, "") - elif path.startsWith("/static"): - let subpath = path.substr("/static".len) - try: - var headers = HttpTable.init() - headers.add("Content-Type", mimedb.getMimetype(path.splitFile.ext)) - await req.sendResponse(Http200, headers, data = readStaticFile(subpath)) - except: - await req.sendError(Http404) - else: - await req.sendError(Http404) - -proc start*(rs: RelayServer, address: TransportAddress, tlsPrivateKey = "", tlsCertificate = "") = - ## Start the relay server at the given address. - let - socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - if tlsPrivateKey != "" and tlsCertificate != "": - rs.http = RelayHttpServer( - tls: true, - httpsServer: TlsHttpServer.create( - address = address, - tlsPrivateKey = TLSPrivateKey.init(tlsPrivateKey), - tlsCertificate = TLSCertificate.init(tlsCertificate), - flags = socketFlags) - ) - else: - rs.http = RelayHttpServer( - tls: false, - httpServer: HttpServer.create(address, flags = socketFlags), - ) - - rs.http.handler = proc(request: HttpRequest) {.async.} = - try: - await rs.handleRequest(request) - except: - let msg = getCurrentExceptionMsg() - if "Stream is already closed" in msg: - discard - else: - logging.error rs.logname, "Error handling HTTP request: " & getCurrentExceptionMsg() - rs.http.start() - -proc finish*(rs: RelayServer) {.async.} = - ## Completely stop the running server - rs.http.stop() - rs.http.close() - await rs.http.join() - vlog rs.logname, "finished" diff --git a/src/bucketsrelay/stringproto.nim b/src/bucketsrelay/stringproto.nim deleted file mode 100644 index eb75eaf..0000000 --- a/src/bucketsrelay/stringproto.nim +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import strutils -import ./proto -import ./netstring - -proc dumps*(ev: RelayEvent): string = - ## Serialize a RelayEvent to a string. Opposite of loadsRelayEvent - result = $ev.kind - case ev.kind - of Who: - result.add nsencode(ev.who_challenge) - of Authenticated: - discard - of Connected: - result.add nsencode(ev.conn_pubkey.string) - of Disconnected: - result.add nsencode(ev.dcon_pubkey.string) - of Data: - result.add nsencode($ev.sender_pubkey.string) - result.add nsencode(ev.data) - of Entered: - result.add nsencode(ev.entered_pubkey.string) - of Exited: - result.add nsencode(ev.exited_pubkey.string) - of ErrorEvent: - result.add nsencode($ev.err_code) - result.add nsencode(ev.err_message) - -proc loadsRelayEvent*(msg: string): RelayEvent = - ## Deserialize a RelayEvent from a string. Opposite of dumps - let kind = case $msg[0] - of $Who: Who - of $Authenticated: Authenticated - of $Connected: Connected - of $Disconnected: Disconnected - of $Data: Data - of $Entered: Entered - of $Exited: Exited - of $ErrorEvent: ErrorEvent - else: - raise ValueError.newException("Unknown event type: " & msg[0]) - let rest = msg[1..^1] - result = RelayEvent(kind: kind) - var decoder = newNetstringDecoder() - decoder.consume(rest) - case kind - of Who: - result.who_challenge = decoder.nextMessage() - of Authenticated: - discard - of Connected: - result.conn_pubkey = decoder.nextMessage().PublicKey - of Disconnected: - result.dcon_pubkey = decoder.nextMessage().PublicKey - of Data: - result.sender_pubkey = decoder.nextMessage().PublicKey - result.data = decoder.nextMessage() - of Entered: - result.entered_pubkey = decoder.nextMessage().PublicKey - of Exited: - result.exited_pubkey = decoder.nextMessage().PublicKey - of ErrorEvent: - result.err_code = parseEnum[ErrorCode](decoder.nextMessage()) - result.err_message = decoder.nextMessage() - -proc dumps*(cmd: RelayCommand): string = - ## Serialize a RelayCommand to a string. Opposite of loadsRelayCommand. - result = $cmd.kind - case cmd.kind - of Iam: - result.add nsencode(cmd.iam_signature) - result.add nsencode(cmd.iam_pubkey.string) - of Connect: - result.add nsencode(cmd.conn_pubkey.string) - of Disconnect: - result.add nsencode(cmd.dcon_pubkey.string) - of SendData: - result.add nsencode(cmd.dest_pubkey.string) - result.add nsencode(cmd.send_data) - -proc loadsRelayCommand*(msg: string): RelayCommand = - ## Deserialize a RelayCommand from a string. Opposite of dumps. - let kind = case $msg[0] - of $Iam: Iam - of $Connect: Connect - of $Disconnect: Disconnect - of $SendData: SendData - else: - raise ValueError.newException("Unknown command type: " & msg[0]) - let rest = msg[1..^1] - result = RelayCommand(kind: kind) - var decoder = newNetstringDecoder() - decoder.consume(rest) - case kind - of Iam: - result.iam_signature = decoder.nextMessage() - result.iam_pubkey = decoder.nextMessage().PublicKey - of Connect: - result.conn_pubkey = decoder.nextMessage().PublicKey - of Disconnect: - result.dcon_pubkey = decoder.nextMessage().PublicKey - of SendData: - result.dest_pubkey = decoder.nextMessage().PublicKey - result.send_data = decoder.nextMessage() diff --git a/src/objs.nim b/src/objs.nim index 4ab947c..d2c96f6 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -207,17 +207,23 @@ proc `==`*(a, b: RelayCommand): bool = # TODO: consider if this should belong in a different file #-------------------------------------------------------------- -proc nsencode*(x: string): string = +const MAX_NETSTRING = 65536 + +type + NetstringError* = object of CatchableError + IncompleteNetstring* = object of NetstringError + +proc nsencode*(x: string, terminal = ','): string = ## Encode a string as a netstring - $len(x) & ":" & x & "," + $len(x) & ":" & x & terminal -proc nsdecode*(x: string, start: var int = 0): string = +proc nsdecode*(x: string, start: var int, maxlen = MAX_NETSTRING): string = ## Read the netstring from x starting at index `start` ## start will be moved to the next netstring location if x.len == 0: - raise ValueError.newException("Empty string is invalid netstring") + raise NetstringError.newException("Empty string is invalid netstring") var cursor = start - # get length prefix + # 1. get length prefix var expectedLength = 0 block: var buf = "" @@ -227,24 +233,35 @@ proc nsdecode*(x: string, start: var int = 0): string = case ch of '0'..'9': buf.add(ch) + if buf.parseInt > maxlen: + raise NetstringError.newException("Exceeds max length") of ':': + if buf.len == 0: + raise NetstringError.newException("Missing starting length") + if buf.len >= 2 and buf[0] == '0': + raise NetstringError.newException("Invalid starting length") expectedLength = buf.parseInt() break else: - raise ValueError.newException("Invalid length character: " & ch & " at position " & $cursor) + raise NetstringError.newException("Invalid length character: " & ch & " at position " & $cursor) - # check for terminal and length + # 2. check for terminal and length let terminalIdx = cursor + expectedLength if terminalIdx >= x.len: - raise ValueError.newException("Netstring incomplete") + raise IncompleteNetstring.newException("Netstring incomplete") let terminalCh = x[terminalIdx] if terminalCh notin {',','\n'}: - raise ValueError.newException("Invalid terminal character: " & terminalCh) + raise NetstringError.newException("Invalid terminal character: " & terminalCh) + # 3. get string result = x[cursor..(cursor + expectedLength - 1)] start = terminalIdx + 1 +proc nsdecode*(x: string, maxlen = MAX_NETSTRING): string = + var idx = 0 + return nsdecode(x, idx, maxlen = maxlen) + proc serialize*(kind: MessageKind): char = case kind diff --git a/src/partials/index.mustache b/src/partials/index.mustache deleted file mode 100644 index ea36c26..0000000 --- a/src/partials/index.mustache +++ /dev/null @@ -1,384 +0,0 @@ - - - Buckets Relay - - - - -
-

- - Buckets Relay -

- -

- If you use Buckets, this relay lets you securely share your budget among your devices. This relay doesn't store any budget info. Instead, think of it like a satellite in the sky that can bounce your data from your computer to your phone. -

- -

- Use of this service may be revoked at any time for any reason. -

- -

- The code for this is Open Source if you'd like to run your own instance. -

- - {{#openregistration}} -
-

Register

-
- - -
- -
- - -
- - -
-
- -
-

Verify email address

-
- - -
- -
- - -
- - -
-
- If you didn't receive a verification code, you can - -
- -
-

Forgot password?

-
- - -
- - -
-
- -
-

Change password

-
- - -
-
- - -
- - -
-
- {{/openregistration}} - -
- {{#openregistration}} - - {{/openregistration}} - - \ No newline at end of file diff --git a/src/server2.nim b/src/server2.nim new file mode 100644 index 0000000..f2d1b6f --- /dev/null +++ b/src/server2.nim @@ -0,0 +1,71 @@ +import std/asyncdispatch +import std/asynchttpserver +import std/logging +import std/strformat +import std/strutils + +import nimja +import ws + +const + favicon_png = slurp"static/favicon.png" + logo_png = slurp"static/logo.png" + version = slurp"../CHANGELOG.md".split(" ")[1] +static: + echo "version: ", version + +var connections = newSeq[WebSocket]() + +proc cb(req: Request) {.async, gcsafe.} = + if req.url.path == "/": + var html = "" + compileTemplateFile("templates/index.html", baseDir = getScriptDir(), autoEscape = true, varname = "html") + await req.respond(Http200, html) + elif req.url.path == "/static/favicon.png": + await req.respond(Http200, favicon_png) + elif req.url.path == "/static/logo.png": + await req.respond(Http200, logo_png) + elif req.url.path == "/ws": + try: + var ws = await newWebSocket(req) + connections.add ws + # await ws.send("Welcome to simple chat server") + while ws.readyState == Open: + let packet = await ws.receiveStrPacket() + # echo "Received packet: " & packet + # for other in connections: + # if other.readyState == Open: + # asyncCheck other.send(packet) + except WebSocketClosedError: + echo "Socket closed. " + except WebSocketProtocolMismatchError: + echo "Socket tried to use an unknown protocol: ", getCurrentExceptionMsg() + except WebSocketError: + echo "Unexpected socket error: ", getCurrentExceptionMsg() + await req.respond(Http200, "done") + else: + await req.respond(Http404, "Not found") + +proc main(database: string, port: Port, address = "127.0.0.1") = + var L = newConsoleLogger() + addHandler(L) + var server = newAsyncHttpServer() + info &"Serving on {address}:{port.int}" + waitFor server.serve(port, cb, address = address) + +when isMainModule: + import argparse + var p = newParser: + option("-d", "--database", help="Database") + command("server"): + option("-p", "--port", default=some("9000")) + option("-a", "--address", default=some("127.0.0.1")) + run: + main(opts.parentOpts.database, opts.port.parseInt.Port, opts.address) + + try: + p.run() + except UsageError as e: + stderr.writeLine getCurrentExceptionMsg() + quit(1) + \ No newline at end of file diff --git a/src/templates/index.nimja b/src/templates/index.nimja new file mode 100644 index 0000000..e69de29 diff --git a/tests/all.sh b/tests/all.sh new file mode 100755 index 0000000..80c0ace --- /dev/null +++ b/tests/all.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +RC=0 +for filename in $(ls tests/t*.nim); do + echo $filename + nim r "$filename" + rc1=$? + if [ ! "$rc1" == "0" ]; then + RC="$rc1" + fi +done + +exit "$RC" diff --git a/tests/tbrelay.nim b/tests/tbrelay.nim deleted file mode 100644 index e934cb2..0000000 --- a/tests/tbrelay.nim +++ /dev/null @@ -1,75 +0,0 @@ -import std/unittest -import std/logging -import ./util - -import chronos - -import brelay -import bclient - -import bucketsrelay/common -import bucketsrelay/proto - -proc tlog(msg: string) = - debug "TEST: " & msg - -when multiusermode: - test "copy": - withinTmpDir: - tlog "Adding users ..." - addverifieduser("data.sqlite", "alice", "alice") - addverifieduser("data.sqlite", "bob", "bob") - let relayurl = "http://127.0.0.1:9001/v1/relay" - tlog "Starting relay ..." - let server = startRelay("data.sqlite", 9001.Port, "127.0.0.1") - tlog "Generating keys ..." - let akeys = genkeys() - let bkeys = genkeys() - tlog "Sending from sender to receiver ..." - let sendres = relaySend("hello", bkeys.pk, - relayurl = relayurl, - mykeys = akeys, - username = "alice", - password = "alice", - ) - tlog "Receiving from sender ..." - let recvres = relayReceive(akeys.pk, - relayurl = relayurl, - mykeys = bkeys, - username = "bob", - password = "bob", - ) - tlog "Waiting for send to resolve" - waitFor sendres - tlog "Waiting for recv to resolve" - let res = waitFor recvres - check res == "hello" - -when singleusermode: - test "copy": - withinTmpDir: - let relayurl = "http://127.0.0.1:9001/v1/relay" - tlog "Starting relay ..." - let server = startRelaySingleUser("alice", "password", 9001.Port, "127.0.0.1") - tlog "Generating keys ..." - let akeys = genkeys() - let bkeys = genkeys() - tlog "Sending from sender to receiver ..." - let sendres = relaySend("hello", bkeys.pk, - relayurl = relayurl, - mykeys = akeys, - username = "alice", - password = "password", - ) - tlog "Receiving from sender ..." - let recvres = relayReceive(akeys.pk, - relayurl = relayurl, - mykeys = bkeys, - username = "alice", - password = "password", - ) - tlog "Waiting for send to resolve" - waitFor sendres - tlog "Waiting for recv to resolve" - let res = waitFor recvres - check res == "hello" diff --git a/tests/tclient.nim b/tests/tclient.nim deleted file mode 100644 index d9929ec..0000000 --- a/tests/tclient.nim +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import std/unittest -import std/strutils - -import ./util - -import bucketsrelay/common -import bucketsrelay/client -import bucketsrelay/server - -type - ClientHandler = ref object - events: seq[RelayEvent] - lifeEvents: seq[ClientLifeEvent] - -proc handleEvent(handler: ClientHandler, ev: RelayEvent, remote: RelayClient) {.async.} = - handler.events.add(ev) - -proc handleLifeEvent(handler: ClientHandler, ev: ClientLifeEvent, remote: RelayClient) {.async.} = - handler.lifeEvents.add(ev) - -proc newClientHandler(): ClientHandler = - new(result) - -proc popEvent(client: ClientHandler, k: EventKind): Future[RelayEvent] {.async, gcsafe.} = - ## Wait for and remove particular event type from the queue - # Since this is just for tests, this does dumb polling - var res: RelayEvent - var delay = 10 - while true: - var idx = -1 - for i,ev in client.events: - if ev.kind == k: - idx = i - res = ev - break - if idx >= 0: - client.events.del(idx) - return res - else: - if delay > 1000: - echo "Waiting for event: " & $k - await sleepAsync(delay.milliseconds) - delay += 100 - -proc popEvent(client: ClientHandler, k: ClientLifeEventKind): Future[ClientLifeEvent] {.async.} = - var delay = 10 - while true: - var idx = -1 - for i,ev in client.lifeEvents: - if ev.kind == k: - idx = i - result = ev - break - if idx >= 0: - client.lifeEvents.del(idx) - return result - else: - if delay > 1000: - echo "Waiting for event: " & $k - await sleepAsync(delay.milliseconds) - delay += 100 - -when multiusermode: - - proc verified_user(rs: RelayServer, email: string, password = ""): int64 = - result = rs.register_user(email, password) - let token = rs.generate_email_verification_token(result) - assert rs.use_email_verification_token(result, token) == true - - test "basic": - withinTmpDir: - var server = newRelayServer(":memory:") - server.start(initTAddress("127.0.0.1", 9001)) - defer: - waitFor server.finish() - let user1 = server.verified_user("alice", "password") - let user2 = server.verified_user("bob", "password") - - var c1h = newClientHandler() - var keys1 = genkeys() - var client1 = newRelayClient(keys1, c1h, "alice", "password") - waitFor client1.connect("ws://127.0.0.1:9001/v1/relay") - discard waitFor c1h.popEvent(ConnectedToServer) - - var c2h = newClientHandler() - var keys2 = genkeys() - var client2 = newRelayClient(keys2, c2h, "bob", "password") - waitFor client2.connect("ws://127.0.0.1:9001/v1/relay") - discard waitFor c2h.popEvent(ConnectedToServer) - - waitFor client1.connect(keys2.pk) - waitFor client2.connect(keys1.pk) - - var atob = (waitFor c1h.popEvent(Connected)).conn_pubkey - var btoa = (waitFor c2h.popEvent(Connected)).conn_pubkey - check atob.string != "" - check btoa.string != "" - - waitFor client1.sendData(atob, "hello") - check (waitFor c2h.popEvent(Data)).data == "hello" - waitFor client2.sendData(btoa, "a".repeat(4096)) - check (waitFor c1h.popEvent(Data)).data == "a".repeat(4096) - - waitFor client1.disconnect(keys2.pk) - waitFor client2.disconnect(keys2.pk) - - check (waitFor c1h.popEvent(Disconnected)).dcon_pubkey == keys2.pk - check (waitFor c2h.popEvent(Disconnected)).dcon_pubkey == keys1.pk - - test "NotConnected": - withinTmpDir: - var server = newRelayServer(":memory:") - server.start(initTAddress("127.0.0.1", 9002)) - let user1 = server.verified_user("alice", "password") - - var ch = newClientHandler() - var keys1 = genkeys() - var client1 = newRelayClient(keys1, ch, "alice", "password") - waitFor client1.connect("ws://127.0.0.1:9002/v1/relay") - echo "Stopping relay server ..." - waitFor server.finish() - echo "Relay server stopped" - for req in allHttpRequests: - # req.stream.writer.tsource.close() - req.stream.reader.tsource.close() - echo "Closed stream" - discard waitFor ch.popEvent(DisconnectedFromServer) - expect RelayNotConnected: - waitFor client1.connect("foobar".PublicKey) - expect RelayNotConnected: - waitFor client1.sendData("foobar".PublicKey, "some data") - - test "server goes down": - withinTmpDir: - var server = newRelayServer(":memory:") - server.start(initTAddress("127.0.0.1", 9002)) - let user1 = server.verified_user("alice", "password") - - var ch = newClientHandler() - var keys1 = genkeys() - var client1 = newRelayClient(keys1, ch, "alice", "password") - waitFor client1.connect("ws://127.0.0.1:9002/v1/relay") - discard waitFor ch.popEvent(ConnectedToServer) - echo "Stopping relay server ..." - waitFor server.finish() - echo "Relay server stopped" - for req in allHttpRequests: - # req.stream.writer.tsource.close() - req.stream.reader.tsource.close() - echo "Closed stream" - discard waitFor ch.popEvent(DisconnectedFromServer) - expect RelayNotConnected: - waitFor client1.connect("foobar".PublicKey) - expect RelayNotConnected: - waitFor client1.sendData("foobar".PublicKey, "some data") - - test "wrong credentials": - var server = newRelayServer(":memory:") - server.start(initTAddress("127.0.0.1", 9003)) - defer: - waitFor server.finish() - let user1 = server.verified_user("alice", "password") - - var ch = newClientHandler() - var keys1 = genkeys() - var client1 = newRelayClient(keys1, ch, "alice", "wrongpassword") - expect RelayErrLoginFailed: - waitFor client1.connect("ws://127.0.0.1:9003/v1/relay") diff --git a/tests/tlicenses.nim b/tests/tlicenses.nim deleted file mode 100644 index 177edf6..0000000 --- a/tests/tlicenses.nim +++ /dev/null @@ -1,89 +0,0 @@ -import std/strutils -import std/unittest - -import bucketsrelay/licenses - -const PRIVATEKEY1 = """ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAkFVXBWA85bBdFOpdwusXL5hELbGh9u7cg/ZeoV1ToDD02Tw2 -BEetGBUSzXsp3fKPbx89wigTjGAJNHAXVGcdbAbBCve+ARhTJTHIrXZ3lXNxvl0j -KfXNa0VqV79WPeZaRtlvC0e8G9A8sL1wjvZn0nL2DG3gGBLeyAeSYiSCE8ROx5op -oDylJRj5RVTWDtCsQFU4j5h7+Jk2nFCfIsaLyDDKquiycIRcXAt8f32RaMEZn0qh -OXnqjHRAEF8V8hvDVfvVwx4iJXcAdnbDHKIl/aD7ssk2fYqeh0kFRH0zyLmPgFoJ -UYOb+opA1NsWBcZCmaeyLC+RWjJDDXQW9H4NgwIDAQABAoIBACEBkwvkrShtg2vE -CLsJXd0Beh3k8D/y8bSvw4YtPHF2oJeJAGVMKtZGA23AC5v42zozL8FVvtqsH47B -T2R6zCynAsBKVUYU1Pa9gsHARKqFou5AiEkRL++nCSGV3Nf89IodMRqoRekqXqag -O7xFtwpWRdQj0EpRDmc57AzLgn+YWrdhwy/2IklJpkmbXiE/lr2Hmgt1eLPb+F5Q -zJ3JGpLKyFmgQZEuShhSVFJnqnFdJGhpK6DDI9XTEufxoBOhEbgJyJrc3FKqjQ4s -Fro4GGNBOjFzOM8nAVWjAeMTMDh/6DSFDP0DDhbQlCHvKfv78UK7oIDEylGOwSha -ODaTVWECgYEA+6gRIR5M9hy8E4/09ZwWjCgOqOSEEL4dluZWSS8a3nMLLTz/Q18u -disfJVNP/rFPO40eRP5FdNHXtXDVbOFclCm0tERcWzQNFYNi54vT9yGQrMpdqJsd -a5/vX3vztvr0Kw7O3jPwzCOkMne04BGZKW/TJelguEBN6d0wNNYh1dMCgYEAktMS -CLM+tyf/tULVAkapYr6kr/fi5ZyNn7S6YkZQx8soH54JfhMqbBCzRwSeF2NKnj7D -XBzjp4FFQoFCoqNwo0G9nTAoOoY6y3S9lLTZr4LjTthW4Zgo1JpgD6/jIEL/v2mC -zpEpvfUWWKjVCn77QBj9Zxoda9v0DRa40DiAq5ECgYEAvXsJEree2Pw/vDb7COcy -rusGRrJwoa6T1uetdkMKZw2WD8TKqi6DbCQBunflVm6oqr0RWn9dSp0pXosLl4SD -0WcpkUWbiGxDobwgfxjwSzYxmXhxVp8cYsm0UV+h3FdN+xGWPwY6u2nmmr05KjD1 -8pYpFHWJBpIcWAbb4hyMs1MCgYA0tnTODNRiW4jxocnp5Eah/gIQbzXV68vo37De -4ZHU+Toxh8KuseDUJXbH839ytCIxCCWJZ5HQLJgaFWBAFd+1rT+PNJ/syw5Gx2Xd -AsT4v0wunXsryT43fiko2KP5jDRXm2DsGq/a1CgusoayGv7Hd3Fa18RiWfiXzmWR -1AdWEQKBgQCGjvSH/6cVzf5L8qpdhGcZ1jIalc5K0eO5//qvCYv8HysdXB8mUfdh -63oK3QONPrYql3KKLgWjQxaffPEWshm4c02tuJQanSGa2yTQwhSJsk4hg1iY+7lX -ihlOePC/fSmBJCmr9f0n0DNn1MxUL6GuIU7peFGm5Q1TJ0toimWG/w== ------END RSA PRIVATE KEY----- -""".strip() - -const PUBLICKEY1 = """ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAkFVXBWA85bBdFOpdwusX -L5hELbGh9u7cg/ZeoV1ToDD02Tw2BEetGBUSzXsp3fKPbx89wigTjGAJNHAXVGcd -bAbBCve+ARhTJTHIrXZ3lXNxvl0jKfXNa0VqV79WPeZaRtlvC0e8G9A8sL1wjvZn -0nL2DG3gGBLeyAeSYiSCE8ROx5opoDylJRj5RVTWDtCsQFU4j5h7+Jk2nFCfIsaL -yDDKquiycIRcXAt8f32RaMEZn0qhOXnqjHRAEF8V8hvDVfvVwx4iJXcAdnbDHKIl -/aD7ssk2fYqeh0kFRH0zyLmPgFoJUYOb+opA1NsWBcZCmaeyLC+RWjJDDXQW9H4N -gwIDAQAB ------END PUBLIC KEY----- -""".strip() - -const PRIVATEKEY2 = """ ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQDCqQMftQvDX2B2oJl1t7eXRSMhviklJx00olcqI/4okB2WLX18 -3wNUM+O+DZiMAkOlMk96Z6y1Rs03CmV4wJmu4fwrOGFrcS1nsOky8z9KLPENmzxp -0FAL2xwdG6TEhGOlHSRloDQQN58CEjegPYGcLwiysL30fmK69GbVE6f1ZwIDAQAB -AoGBAJCtVzIIuH5z89kXUhdo/V3Dt/HLSP9hC9bj1Y7vg2YYfrTwiHT3t5ysmFbX -+goNYMN2GhYq2fU9cya2ZmaSF2XR9fD5zGINSFltSSOQTUtokhUUx6pVDk06CmjJ -vetu7//nhVp1xP4T2IHXIOuaOB1FxfMlUk8LV+TNsmhsXHgxAkEA/WbYpDp8ukLw -ryhpOaqZiZW06aTe8seLNS2U7cGlTe+VsA9uGwS1HHIvAiOQ9/4f5rjM3XZtNwZD -NrH+2BambwJBAMSn+bNuoFUtwVGzKSaAMGOg/IERQN8uH73iSCSaFnfM7Kwzl4o7 -u96nEYi0B2R7UMa/UwbgpBDplnvZ8QRXXIkCPw/WXbPl8+WwSVqpK+puvynaMXRo -2YZS8mBgeO5jK/GzB6f5TuhhYvBkMovvrR/SwiupYSR2Ql0uBwVkGolm4QJANFHs -YQyho4fU0wOzgwa/2QHPrBcHB1miIEa/ot1L9PuUTAw92Q0jYo1YYOJkxRr51qa4 -VDAX9lfvLWxCb0E+4QJBAJlhxvujrPotY6/rXMVAY6Zt+MmQiUiYNDVm4eEaH6t5 -jMiVvR+d8aAzTRTV1U8jg+LwhM7t0lyN5gIC8NeuHuU= ------END RSA PRIVATE KEY----- -""".strip() - -const PUBLICKEY2 {.used.} = """ ------BEGIN PUBLIC KEY----- -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDCqQMftQvDX2B2oJl1t7eXRSMh -viklJx00olcqI/4okB2WLX183wNUM+O+DZiMAkOlMk96Z6y1Rs03CmV4wJmu4fwr -OGFrcS1nsOky8z9KLPENmzxp0FAL2xwdG6TEhGOlHSRloDQQN58CEjegPYGcLwiy -sL30fmK69GbVE6f1ZwIDAQAB ------END PUBLIC KEY----- -""".strip() - -suite "BucketsV1RSALicense": - - test "works": - let license = createV1License(PRIVATEKEY1, "foo@foo.com") - check $license is string - checkpoint $license - check verify(license, PUBLICKEY1) == true - check extractEmail(license) == "foo@foo.com" - - test "wrong key": - let license = createV1License(PRIVATEKEY2, "bad@foo.com") - check $license is string - checkpoint $license - check verify(license, PUBLICKEY1) == false - check extractEmail(license) == "bad@foo.com" diff --git a/tests/tnetstring.nim b/tests/tnetstring.nim index db57520..89a304d 100644 --- a/tests/tnetstring.nim +++ b/tests/tnetstring.nim @@ -1,88 +1,61 @@ import std/unittest -import bucketsrelay/netstring +import objs -test "nsencode": - check nsencode("apple") == "5:apple," - check nsencode("") == "0:," - check nsencode("banana\x00,") == "8:banana\x00,," +suite "encode": + test "nsencode": + check nsencode("apple") == "5:apple," + check nsencode("") == "0:," + check nsencode("banana\x00,") == "8:banana\x00,," -test "nsencode newline allowed instead of comma": - check nsencode("apple", '\n') == "5:apple\n" - check nsencode("", '\n') == "0:\n" - check nsencode("banana\x00\n", '\n') == "8:banana\x00\n\n" + test "nsencode newline allowed instead of comma": + check nsencode("apple", '\n') == "5:apple\n" + check nsencode("", '\n') == "0:\n" + check nsencode("banana\x00\n", '\n') == "8:banana\x00\n\n" -suite "NetstringDecoder": +suite "decode": - test "netstring in, message out": - var ns = newNetstringDecoder() - ns.consume("5:apple,") - check ns.len == 1 - ns.consume("7:bana") - check ns.len == 1 - ns.consume("na\x00,3:foo,3:bar") - check ns.len == 3 - ns.consume(",") - check ns.len == 4 - check ns.nextMessage() == "apple" - check ns.nextMessage() == "banana\x00" - check ns.nextMessage() == "foo" - check ns.nextMessage() == "bar" + test "basic": + check nsdecode("5:apple,") == "apple" + + test "incomplete": + expect(IncompleteNetstring): + discard nsdecode("7:bana") + + test "2 strings": + var idx = 0 + check nsdecode("5:apple,3:f\x00o,", idx) == "apple" + check nsdecode("5:apple,3:f\x00o,", idx) == "f\x00o" test "newline delimiter": - var ns = newNetstringDecoder('\n') - ns.consume("5:apple\n") - check ns.len == 1 - ns.consume("7:bana") - check ns.len == 1 - ns.consume("na\x00\n3:foo\n3:bar") - check ns.len == 3 - ns.consume("\n") - check ns.len == 4 - check ns.nextMessage() == "apple" - check ns.nextMessage() == "banana\x00" - check ns.nextMessage() == "foo" - check ns.nextMessage() == "bar" + check nsdecode(nsencode("apple", '\n')) == "apple" test "empty string": - var ns = newNetstringDecoder() - ns.consume("0:,") - check ns.nextMessage() == "" + check nsdecode("0:,") == "" test "can't start with 0": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume("01:,") + expect(NetstringError): + discard nsdecode("01:a,") test "can't include non-numerics": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume("1a:,") + expect(NetstringError): + discard nsdecode("1a:,") test ": required": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume("1f,") + expect(NetstringError): + discard nsdecode("1f,") test ", required": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume("1:a2:ab,") + expect(NetstringError): + discard nsdecode("1:a2:ab,") test "len required": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume(":s,") + expect(NetstringError): + discard nsdecode(":s,") test "max message length": - var ns = newNetstringDecoder() - - ns.maxlen = 4 - ns.consume("4:fooa,") - expect(Exception): - ns.consume("5:") - ns.reset() - - ns.maxlen = 10000 - expect(Exception): - ns.consume("100000") + check nsdecode("4:boom,", maxlen=4) == "boom" + expect(NetstringError): + discard nsdecode("5:apple,", maxlen=4) + expect(NetstringError): + discard nsdecode("200:a", maxlen=100) diff --git a/tests/tproto.nim b/tests/tproto.nim deleted file mode 100644 index f31461b..0000000 --- a/tests/tproto.nim +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import unittest -import os -import options -import tables -import sets -import logging - -import bucketsrelay/proto -import libsodium/sodium -import ./util - -type - KeyPair = tuple - pk: PublicKey - sk: SecretKey - StringClient = ref object - id: int - received: seq[RelayEvent] - pk: PublicKey - sk: SecretKey - -proc newClient(): StringClient = - new(result) - result.received = newSeq[RelayEvent]() - -proc popEvent(client: StringClient): RelayEvent = - doAssert client.received.len > 0, "Expected an event" - result = client.received[0] - client.received.del(0) - -proc popEvent(client: StringClient, kind: EventKind): RelayEvent = - result = client.popEvent() - doAssert result.kind == kind, "Expected " & $kind & " but found " & $result - -proc sendEvent(client: StringClient, ev: RelayEvent) = - client.received.add(ev) - -proc popEvent(conn: RelayConnection[StringClient]): RelayEvent = - conn.sender.popEvent() - -proc popEvent(conn: RelayConnection[StringClient], kind: EventKind): RelayEvent = - conn.sender.popEvent(kind) - -proc mkConnection(relay: var Relay, keys = none[KeyPair](), channel = ""): RelayConnection[StringClient] = - var keys = keys - if keys.isNone: - keys = some(genkeys()) - var client = newClient() - client.pk = keys.get().pk - client.sk = keys.get().sk - var conn = relay.initAuth(client, channel = channel) - let who = client.popEvent() - let signature = sign(client.sk, who.who_challenge) - relay.handleCommand(conn, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: client.pk)) - let ok = client.popEvent() - result = conn - -template sendData*(relay: var Relay, src: RelayConnection, dst: PublicKey, data: string) = - relay.handleCommand(src, RelayCommand(kind: SendData, send_data: data, dest_pubkey: dst)) - -test "basic": - var relay = newRelay[StringClient]() - let (pk, sk) = genkeys() - var aclient = newClient() - aclient.pk = pk - aclient.sk = sk - - checkpoint "who?" - var alice = relay.initAuth(aclient) - let who = alice.popEvent() - check who.kind == Who - check who.who_challenge != "" - - checkpoint "iam" - let signature = sign(sk, who.who_challenge) - relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: pk)) - let ok = alice.popEvent() - check ok.kind == Authenticated - - checkpoint "connect" - let bob = relay.mkConnection() - check bob.pubkey != alice.pubkey - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - block: - let ev = bob.popEvent() - check ev.kind == Connected - check ev.conn_pubkey == alice.pubkey - - block: - let ev = alice.popEvent() - check ev.kind == Connected - check ev.conn_pubkey == bob.pubkey - - checkpoint "data" - relay.handleCommand(bob, RelayCommand(kind: SendData, send_data: "hello, alice!", dest_pubkey: alice.pubkey)) - let adata = alice.popEvent() - check adata.kind == Data - check adata.data == "hello, alice!" - check adata.sender_pubkey == bob.pubkey - - relay.handleCommand(alice, RelayCommand(kind: SendData, send_data: "hello, bob!", dest_pubkey: bob.pubkey)) - let bdata = bob.popEvent() - check bdata.kind == Data - check bdata.data == "hello, bob!" - check bdata.sender_pubkey == alice.pubkey - -test "multiple conns to same pubkey": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - check bob.sender.received.len == 0 - check alice.sender.received.len == 0 - -test "no crosstalk": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - var cathy = relay.mkConnection() - var dave = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - check cathy.sender.received.len == 0 - check dave.sender.received.len == 0 - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: dave.pubkey)) - relay.handleCommand(dave, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard dave.popEvent(Connected) - relay.sendData(alice, bob.pubkey, "hi, bob") - check bob.popEvent(Data).data == "hi, bob" - check cathy.sender.received.len == 0 - check dave.sender.received.len == 0 - -test "disconnect multiple times": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - relay.removeConnection(alice) - relay.removeConnection(alice) - -test "disconnect, remove from remote client.connections": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - relay.removeConnection(alice) - let edcon = bob.popEvent(Disconnected) - check edcon.dcon_pubkey == alice.pubkey - let bobclient = relay.testmode_conns()[bob.pubkey] - check bobclient.testmode_conns.len == 0 - -test "send data to invalid id": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - relay.sendData(alice, "goober".PublicKey, "testing?") - discard alice.popEvent(ErrorEvent) - relay.sendData(alice, alice.pubkey, "feedback") - discard alice.popEvent(ErrorEvent) - -test "send data to unconnected id": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.sendData(alice, bob.pubkey, "hello") - discard alice.popEvent(ErrorEvent) - check bob.sender.received.len == 0 - -test "connect to self": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(ErrorEvent) - -test "not authenticated": - var relay = newRelay[StringClient]() - let (pk, sk) = genkeys() - let aclient = newClient() - - checkpoint "who?" - var alice = relay.initAuth(aclient) - discard alice.popEvent(Who) - - let bob = relay.mkConnection() - - checkpoint "connect" - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - discard alice.popEvent(ErrorEvent) - check bob.sender.received.len == 0 - - checkpoint "send" - relay.sendData(alice, bob.pubkey, "something") - discard alice.popEvent(ErrorEvent) - check bob.sender.received.len == 0 - -test "disconnect command": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - - relay.handleCommand(alice, RelayCommand(kind: Disconnect, dcon_pubkey: bob.pubkey)) - check bob.popEvent(Disconnected).dcon_pubkey == alice.pubkey - check alice.popEvent(Disconnected).dcon_pubkey == bob.pubkey - -test "remember connection requests": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - - relay.handleCommand(alice, RelayCommand(kind: Disconnect, dcon_pubkey: bob.pubkey)) - check bob.popEvent(Disconnected).dcon_pubkey == alice.pubkey - check alice.popEvent(Disconnected).dcon_pubkey == bob.pubkey - - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - -test "forget connection requests on disconnect": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - - relay.handleCommand(alice, RelayCommand(kind: Disconnect, dcon_pubkey: bob.pubkey)) - check bob.popEvent(Disconnected).dcon_pubkey == alice.pubkey - check alice.popEvent(Disconnected).dcon_pubkey == bob.pubkey - - relay.handleCommand(bob, RelayCommand(kind: Disconnect, dcon_pubkey: alice.pubkey)) - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.sendData(alice, bob.pubkey, "something") - discard alice.popEvent(ErrorEvent) - check bob.sender.received.len == 0 - -test "pub/sub": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection(channel = "alicenbob") - var bob = relay.mkConnection(channel = "alicenbob") - block: - let ev = alice.popEvent(Entered) - check ev.entered_pubkey == bob.pubkey - block: - let ev = bob.popEvent(Entered) - check ev.entered_pubkey == alice.pubkey - relay.removeConnection(alice) - block: - let ev = bob.popEvent(Exited) - check ev.exited_pubkey == alice.pubkey diff --git a/tests/tserver.nim b/tests/tserver.nim deleted file mode 100644 index 2ad62c7..0000000 --- a/tests/tserver.nim +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import std/unittest -import std/strutils -import ./util - -import chronos - -import bucketsrelay/common -import bucketsrelay/server -import bucketsrelay/licenses - -when multiusermode: - test "add user": - withinTmpDir: - var rs = newRelayServer("test.db") - let uid = rs.register_user("foo", "password") - check rs.is_email_verified(uid) == false - check rs.can_use_relay(uid) == false - check rs.password_auth("foo", "password") == uid - expect WrongPassword: - discard rs.password_auth("foo", "something else") - - let token = rs.generate_email_verification_token(uid) - checkpoint "token: " & token - check rs.use_email_verification_token(uid, token) == true - check rs.is_email_verified(uid) == true - check rs.can_use_relay(uid) == true - - test "duplicate users not allowed": - withinTmpDir: - var rs = newRelayServer("test.db") - discard rs.register_user("foo", "password") - expect DuplicateUser: - discard rs.register_user("foo", "another") - - test "email verification only 5 latest codes work": - withinTmpDir: - var rs = newRelayServer("test.db") - let uid = rs.register_user("foo", "password") - let t1 = rs.generate_email_verification_token(uid) - discard rs.generate_email_verification_token(uid) - let t3 = rs.generate_email_verification_token(uid) - discard rs.generate_email_verification_token(uid) - check rs.use_email_verification_token(uid, "invalid token") == false - check rs.is_email_verified(uid) == false - - check rs.use_email_verification_token(uid, t1) == false # failed because only 5 are valid - check rs.is_email_verified(uid) == false - - check rs.use_email_verification_token(uid, t3) == true - check rs.is_email_verified(uid) == true - - proc verified_user(rs: RelayServer, email: string, password = ""): int64 = - result = rs.register_user(email, password) - let token = rs.generate_email_verification_token(result) - assert rs.use_email_verification_token(result, token) == true - - test "reset password": - withinTmpDir: - var rs = newRelayServer(":memory:") - let uid = rs.register_user("foo", "password") - let t1 = rs.generate_password_reset_token("foo") - check rs.user_for_password_reset_token(t1).get() == uid - rs.update_password_with_token(t1, "newpassword") - check rs.password_auth("foo", "newpassword") == uid - - test "reset password once only": - withinTmpDir: - var rs = newRelayServer(":memory:") - let uid = rs.register_user("foo", "password") - let t1 = rs.generate_password_reset_token("foo") - check rs.user_for_password_reset_token(t1).get() == uid - rs.update_password_with_token(t1, "newpassword") - expect NotFound: - rs.update_password_with_token(t1, "another password") - check rs.password_auth("foo", "newpassword") == uid - - test "block user": - withinTmpDir: - var rs = newRelayServer("test.db") - var uid = rs.verified_user("foo") - var other = rs.verified_user("bar") - rs.block_user("foo") - check rs.can_use_relay(uid) == false - check rs.can_use_relay(other) == true - rs.unblock_user("foo") - check rs.can_use_relay(uid) == true - check rs.can_use_relay(other) == true - - test "log user data": - withinTmpDir: - var rs = newRelayServer("test.db") - var uid = rs.verified_user("foo") - rs.log_user_data_sent(uid, 10) - rs.log_user_data_recv(uid, 20) - rs.log_user_data_sent(uid, 30) - rs.log_user_data_recv(uid, 40) - check rs.data_by_user(uid, days = 1) == (40, 60) - rs.log_ip_data_sent("10.0.0.5", 10) - rs.log_ip_data_recv("10.0.0.4", 20) - check rs.data_by_ip("10.0.0.5", days = 1) == (10, 0) - check rs.data_by_ip("10.0.0.4", days = 1) == (0, 20) - - check rs.top_data_users(10, days = 7) == @[ - ("foo", (40, 60)), - ] - check rs.top_data_ips(10, days = 7) == @[ - ("10.0.0.4", (0, 20)), - ("10.0.0.5", (10, 0)), - ] - - const SAMPLE_PRIVATE_KEY = """ - -----BEGIN RSA PRIVATE KEY----- - MIICWwIBAAKBgH29pIKU/P8ELlA4ofzSiq4InGzd45hxqE/vfqqUOP70Sa5R5s6W - ntYVz6x5Btp8uc2vwWcDg4gFDkyBJ9xShROaCrtvncRIbJ5m3uka46yAObWYkKxP - +e3AMud/8tu5DvnJRiPq9Nu0wbdWXePriajk/Nc4CQAl8tB1Ka1QWLhhAgMBAAEC - gYBLP5aX3vmY07OzpnCqkIUVqWmTbSarMDl9vOGcy59gVGlTvQfXUiQ0ElF58eO8 - FTBMe4XOVDf+yqfH+PMV0vx348rxXG1z2SBAB9tDk4G54bldeoEOtU3kcZCQhstr - A7i4bjFwEK73URRMFaa8/NFGAC9KQ+eD1o/kptB8GPFWEQJBAOuyvEl1mFZtCXHB - THLL92isP/o/q6+dLpI6WlmakFawV1Aof5QPrGeYf50R+c9XebkAxKRtSfzMeQ5u - wLUuPMsCQQCIkkpzrsMtvEPV8iuIphJHcQ7T4DP+Z0iRwN+AjWN9FL6tlTuuIdYW - EsGX6SBeKLbgpsIPsXz+ipsISTnhO8YDAkAuOJrb/QemyzMy76lCSeV2zXCubpYI - llZvrqnRMJJlracxvP9n1bsFhc5gywmmM41XTmNBq3z66k5DGk0IOs0JAkBcZsYy - 0NpDdm5bMZdcxCf36DmFBtuG0/CYlOtjOcZHWaLNJPwVC9WiZ5xOILACpP9erdT8 - 8zRDsBnGmGytxFhrAkEAlrmBpqG8yQyLnyls/R6CPbewf4iqy9utub2ZeiqjYMO4 - aXJie+gnu0Oc8WMQF4BFshD6Lr76QgwUcvjIraNJZw== - -----END RSA PRIVATE KEY----- - """.dedent.strip - - const SAMPLE_PUBLIC_KEY = """ - -----BEGIN PUBLIC KEY----- - MIGeMA0GCSqGSIb3DQEBAQUAA4GMADCBiAKBgH29pIKU/P8ELlA4ofzSiq4InGzd - 45hxqE/vfqqUOP70Sa5R5s6WntYVz6x5Btp8uc2vwWcDg4gFDkyBJ9xShROaCrtv - ncRIbJ5m3uka46yAObWYkKxP+e3AMud/8tu5DvnJRiPq9Nu0wbdWXePriajk/Nc4 - CQAl8tB1Ka1QWLhhAgMBAAE= - -----END PUBLIC KEY----- - """.dedent.strip - - test "authenticate with Buckets license": - withinTmpDir: - var rs = newRelayServer("test.db", pubkey = SAMPLE_PUBLIC_KEY) - let license = createV1License(SAMPLE_PRIVATE_KEY, "jim@jim.com") - checkpoint "license:" - checkpoint $license - let uid = rs.license_auth($license) - let uid2 = rs.get_user_id("jim@jim.com") - check uid == uid2 - check rs.is_email_verified(uid) == true - check rs.can_use_relay(uid) == true - let uid3 = rs.license_auth($license) - check uid3 == uid2 - - test "disable Buckets license": - withinTmpDir: - var rs = newRelayServer("test.db", pubkey = SAMPLE_PUBLIC_KEY) - let license = createV1License(SAMPLE_PRIVATE_KEY, "jim@jim.com") - checkpoint "first license" - checkpoint $license - let uid = rs.license_auth($license) - rs.disable_most_recently_used_license(uid) - expect WrongPassword: - discard rs.license_auth($license) - sleep(1001) - let license2 = createV1License(SAMPLE_PRIVATE_KEY, "jim@jim.com") - checkpoint "second license" - checkpoint $license2 - let uid2 = rs.license_auth($license2) - check uid == uid2 - - test "auth w/ password, then license, should disable password": - withinTmpDir: - var rs = newRelayServer("test.db", pubkey = SAMPLE_PUBLIC_KEY) - let uid = rs.register_user("jim@jim.com", "password") - check rs.is_email_verified(uid) == false - check rs.can_use_relay(uid) == false - check rs.password_auth("jim@jim.com", "password") == uid - expect WrongPassword: - discard rs.password_auth("jim@jim.com", "something else") - - let license = createV1License(SAMPLE_PRIVATE_KEY, "jim@jim.com") - let uid2 = rs.license_auth($license) - check uid == uid2 - check rs.is_email_verified(uid2) == true - check rs.can_use_relay(uid2) == true - - # The password is invalidated by the license authentication - # to prevent preemptive account takeover. Without this, - # an attacker could register a user, setting a password, - # but not verify the email address. Then, when the user - # authenticates with a license it counts that as - # email verification and the password chosen by the attacker - # would work. - expect WrongPassword: - discard rs.password_auth("jim@jim.com", "password") - let t1 = rs.generate_password_reset_token("jim@jim.com") - check rs.user_for_password_reset_token(t1).get() == uid - rs.update_password_with_token(t1, "newpassword") - - # Now, both password and license auth work - check rs.password_auth("jim@jim.com", "newpassword") == uid - check rs.license_auth($license) == uid - check rs.password_auth("jim@jim.com", "newpassword") == uid - check rs.license_auth($license) == uid diff --git a/tests/tstringproto.nim b/tests/tstringproto.nim deleted file mode 100644 index 26630bd..0000000 --- a/tests/tstringproto.nim +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import unittest -import strformat - -import bucketsrelay/proto -import bucketsrelay/stringproto - -proc cev(ev: RelayEvent) = - doAssert ev == ev - checkpoint "original: " & $ev - let intermediate = ev.dumps() - let actual = intermediate.loadsRelayEvent() - checkpoint "actual: " & $actual - checkpoint "intermed: " & intermediate - doAssert ev == actual, &"Expected {ev} to equal {actual}" - -proc ccmd(cmd: RelayCommand) = - doAssert cmd == cmd - checkpoint "original: " & $cmd - let intermediate = cmd.dumps() - let actual = intermediate.loadsRelayCommand() - checkpoint "actual: " & $actual - checkpoint "intermed: " & intermediate - doAssert cmd == actual, &"Expected {cmd} to equal {actual}" - -suite "RelayEvent": - test "Who": - cev RelayEvent(kind: Who, who_challenge: "something\x00!") - test "Authenticated": - cev RelayEvent(kind: Authenticated) - test "Connected": - cev RelayEvent(kind: Connected, conn_pubkey: "hi".PublicKey) - test "Disconnected": - cev RelayEvent(kind: Disconnected, dcon_pubkey: "hi".PublicKey) - test "Data": - cev RelayEvent(kind: Data, sender_pubkey: "hey".PublicKey, data: "bob") - test "ErrorEvent": - cev RelayEvent(kind: ErrorEvent, err_message: "foo") - test "Entered": - cev RelayEvent(kind: Entered, entered_pubkey: "alice".PublicKey) - test "Exited": - cev RelayEvent(kind: Exited, exited_pubkey: "bob".PublicKey) - -suite "RelayCommand": - - test "Iam": - ccmd RelayCommand(kind: Iam) - test "Connect": - ccmd RelayCommand(kind: Connect) - test "Disconnect": - ccmd RelayCommand(kind: Disconnect) - test "SendData": - ccmd RelayCommand(kind: SendData) \ No newline at end of file diff --git a/tests/util.nim b/tests/util.nim index e936daa..8de44c2 100644 --- a/tests/util.nim +++ b/tests/util.nim @@ -1,6 +1,5 @@ import std/logging import std/os; export os -import std/random import std/strformat if os.getEnv("SHOW_LOGS") != "": @@ -8,30 +7,3 @@ if os.getEnv("SHOW_LOGS") != "": addHandler(L) else: echo "set SHOW_LOGS=something to see logs" - -randomize() - -proc tmpDir*(): string = - result = os.getTempDir() / &"test{random.rand(10000000)}" - result.createDir() - -template withinTmpDir*(body:untyped):untyped = - let - tmp = tmpDir() - olddir = getCurrentDir() - setCurrentDir(tmp) - body - setCurrentDir(olddir) - try: - tmp.removeDir() - except: - echo "WARNING: failed to remove temporary test directory: ", getCurrentExceptionMsg() - -template cd*(dirname: string, body: untyped): untyped = - let olddir = getCurrentDir() - try: - createDir(dirname) - setCurrentDir(dirname) - body - finally: - setCurrentDir(olddir) From 1f33915e7e5a061786575224bb8abc516fefdf7b Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 16:39:11 -0400 Subject: [PATCH 10/46] Better changelog entry --- changes/new-Move-to-store-20251028-153340.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/new-Move-to-store-20251028-153340.md b/changes/new-Move-to-store-20251028-153340.md index 77bf092..accc1d0 100644 --- a/changes/new-Move-to-store-20251028-153340.md +++ b/changes/new-Move-to-store-20251028-153340.md @@ -1 +1 @@ -Move to store and forward +Change the relay from being a pure relay to having store-and-forward capabilities. From cf5c9e205a95b7d712b61dc27f99e224b8c7ea35 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 16:41:29 -0400 Subject: [PATCH 11/46] CI --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 225e539..5b92bab 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,8 +7,9 @@ RUN nimble install -y https://github.com/iffy/pkger/ COPY pkger.json . COPY ./pkger ./pkger RUN pkger fetch -COPY . . +COPY ./ ./ COPY docker/config.nims . +RUN find . RUN nim c -d:release -o:brelay src/server2.nim RUN strip brelay From 9daadacfba9ae23c43351bf6bd9d553da6f383e9 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 16:43:14 -0400 Subject: [PATCH 12/46] Fix template path --- src/server2.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server2.nim b/src/server2.nim index f2d1b6f..86c4381 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -19,7 +19,7 @@ var connections = newSeq[WebSocket]() proc cb(req: Request) {.async, gcsafe.} = if req.url.path == "/": var html = "" - compileTemplateFile("templates/index.html", baseDir = getScriptDir(), autoEscape = true, varname = "html") + compileTemplateFile("templates/index.nimja", baseDir = getScriptDir(), autoEscape = true, varname = "html") await req.respond(Http200, html) elif req.url.path == "/static/favicon.png": await req.respond(Http200, favicon_png) From 3df2815f22198467bf68b3cda30df1241535e79e Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 16:44:35 -0400 Subject: [PATCH 13/46] CI --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 29bf7ad..4c238da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,6 +21,10 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: version: ${{ matrix.version }} + - name: Install libsodium + run: | + sudo apt update -q + sudo apt install -y libsodium-dev - name: Install pkger run: nimble install -y https://github.com/iffy/pkger/ - name: Install deps From 9553ea7531553a9e2530ffb8caec2aeec80ecdad Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 28 Oct 2025 21:44:40 -0400 Subject: [PATCH 14/46] Add credentials field for future use --- src/objs.nim | 7 +++++-- tests/tserde2.nim | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/objs.nim b/src/objs.nim index d2c96f6..57d9247 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -64,6 +64,7 @@ type of Iam: iam_pubkey*: PublicKey iam_signature*: string + iam_credentials*: string ## For the future possibility of credentials of PublishNote: pub_topic*: string pub_data*: string @@ -164,7 +165,7 @@ proc `$`*(cmd: RelayCommand): string = result.add $cmd.kind & "(" case cmd.kind of Iam: - result.add &"{cmd.iam_pubkey.nice.abbr} sig={cmd.iam_signature.nicelong}" + result.add &"{cmd.iam_pubkey.nice.abbr} sig={cmd.iam_signature.nicelong} creds={cmd.iam_credentials.nicelong}" of PublishNote: result.add &"'{cmd.pub_topic.nice.abbr}' val={cmd.pub_data.nicelong}" of FetchNote: @@ -189,7 +190,7 @@ proc `==`*(a, b: RelayCommand): bool = else: case a.kind of Iam: - return a.iam_pubkey == b.iam_pubkey and a.iam_signature == b.iam_signature + return a.iam_pubkey == b.iam_pubkey and a.iam_signature == b.iam_signature and a.iam_credentials == b.iam_credentials of PublishNote: return a.pub_topic == b.pub_topic and a.pub_data == b.pub_data of FetchNote: @@ -409,6 +410,7 @@ proc serialize*(cmd: RelayCommand): string = of Iam: result &= cmd.iam_pubkey.string.nsencode result &= cmd.iam_signature.nsencode + result &= cmd.iam_credentials.nsencode of PublishNote: result &= cmd.pub_topic.nsencode result &= cmd.pub_data.nsencode @@ -436,6 +438,7 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = kind: Iam, iam_pubkey: s.nsdecode(idx).PublicKey, iam_signature: s.nsdecode(idx), + iam_credentials: s.nsdecode(idx), ) of PublishNote: var idx = 1 diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 40248c6..74fc3d5 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -1,6 +1,8 @@ import std/unittest +import std/logging import std/options +import ./util import proto2 test "MessageKind": @@ -21,13 +23,19 @@ test "RelayMessage": of Data: RelayMessage(kind: Data, data_src: "hey".PublicKey, data_val: "foo") of Chunk: RelayMessage(kind: Chunk, chunk_src: "hey".PublicKey, chunk_key: "key", chunk_val: some("theval")) let serialized = example.serialize() - checkpoint "serialized: " & serialized + info $example + info "serialized: " & serialized check RelayMessage.deserialize(serialized) == example test "RelayCommand": for kind in low(CommandKind)..high(CommandKind): let example = case kind - of Iam: RelayCommand(kind: Iam, iam_pubkey: "hey".PublicKey, iam_signature: "foo") + of Iam: RelayCommand( + kind: Iam, + iam_pubkey: "hey".PublicKey, + iam_signature: "foo", + iam_credentials: "somecreds", + ) of PublishNote: RelayCommand(kind: PublishNote, pub_topic: "topic", pub_data: "data") of FetchNote: RelayCommand(kind: FetchNote, fetch_topic: "topic") of SendData: RelayCommand(kind: SendData, send_dst: "one".PublicKey, send_val: "data") @@ -39,7 +47,8 @@ test "RelayCommand": ) of GetChunks: RelayCommand(kind: GetChunks, chunk_src: "hey".PublicKey, chunk_keys: @["foo", "bar"]) let serialized = example.serialize() - checkpoint "serialized: " & serialized + info $example + info "serialized: " & serialized check RelayCommand.deserialize(serialized) == example test "Chunk with none": @@ -48,7 +57,7 @@ test "Chunk with none": chunk_key: "key", chunk_val: none[string](), ) - checkpoint $chunk + info $chunk let serialized = chunk.serialize() - checkpoint "serialized: " & serialized + info "serialized: " & serialized check RelayMessage.deserialize(serialized) == chunk From ed8c21d5c0ca685fadd71323c17429f6d970a758 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 29 Oct 2025 10:08:45 -0400 Subject: [PATCH 15/46] duplicate note keys --- .gitignore | 2 ++ src/objs.nim | 7 ++++-- src/proto2.nim | 17 +++++++++---- tests/func1.sh | 60 -------------------------------------------- tests/tnetstring.nim | 8 ++++++ tests/tproto2.nim | 29 +++++++++++++++++++++ 6 files changed, 56 insertions(+), 67 deletions(-) delete mode 100755 tests/func1.sh diff --git a/.gitignore b/.gitignore index 062926b..545f16b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ _tests fly.toml relay.sqlite libs +*.keys +*.sqlite diff --git a/src/objs.nim b/src/objs.nim index 57d9247..22a6f20 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -222,14 +222,17 @@ proc nsdecode*(x: string, start: var int, maxlen = MAX_NETSTRING): string = ## Read the netstring from x starting at index `start` ## start will be moved to the next netstring location if x.len == 0: - raise NetstringError.newException("Empty string is invalid netstring") + raise IncompleteNetstring.newException("Empty string is invalid netstring") var cursor = start # 1. get length prefix var expectedLength = 0 block: var buf = "" while true: - let ch = x[cursor] + let ch = try: + x[cursor] + except IndexDefect: + raise IncompleteNetstring.newException("Not complete") cursor.inc() case ch of '0'..'9': diff --git a/src/proto2.nim b/src/proto2.nim index 0dee190..1860e6b 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -339,11 +339,15 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay relay.delNoteSub(cmd.pub_topic) else: # no one is waiting - relay.db.exec(sql"INSERT INTO note (topic, data) VALUES (?, ?)", - cmd.pub_topic.DbBlob, - cmd.pub_data.DbBlob, - ) - conn.sendOkay cmd.kind + try: + relay.db.exec(sql"INSERT INTO note (topic, data) VALUES (?, ?)", + cmd.pub_topic.DbBlob, + cmd.pub_data.DbBlob, + ) + conn.sendOkay cmd.kind + except: + conn.sendError("Duplicate topic", cmd.kind, Generic) + echo "FRANK post sendError" of FetchNote: if cmd.fetch_topic.len > RELAY_MAX_TOPIC_SIZE: conn.sendError("Topic too long", cmd.kind, TooLarge) @@ -439,6 +443,9 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay chunk_key: key, chunk_val: none[string](), )) + echo "FRANK post case statement" + when LOG_COMMS: + info "[" & conn.pubkey.abbr & "] DONE " & $cmd #------------------------------------------------------------------- # Utilities #------------------------------------------------------------------- diff --git a/tests/func1.sh b/tests/func1.sh deleted file mode 100755 index c52febe..0000000 --- a/tests/func1.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -waitForOpenPort() { - - PORT=$1 - HOSTTOCHECK="127.0.0.1" - TIMEOUT=5 - echo "Waiting for $HOSTTOCHECK:$PORT to open" - sleep "$TIMEOUT" & - SLEEPPID=$! - while ! nc -z "$HOSTTOCHECK" "$PORT"; do - sleep 0.1 - if ! kill -0 "$SLEEPPID" 2>/dev/null; then - echo "Timed out waiting for $HOSTTOCHECK:$PORT to open" - return 1 - fi - done - kill "$SLEEPPID" 2>/dev/null - echo "Port $HOSTTOCHECK:$PORT is open!" - return 0 -} - -dotest() { - echo "Adding a user ..." - printf 'foobar' | brelay adduser me@me.com --password-stdin - - echo "Generating keys ..." - (mkdir -p client1 && cd client1 && bclient genkeys) - (mkdir -p client2 && cd client2 && bclient genkeys) - echo "hello, world" > testfile - - echo "Starting the server ..." - brelay server --port 8080 & - CHILDPID=$! - trap "kill $CHILDPID" exit - - waitForOpenPort 8080 - - echo "Starting the clients ..." - printf "hello, world" > client2/testfile - (cd client1 && bclient receive -u me@me.com -p foobar http://127.0.0.1:8080/v1/relay "$(cat ../client2/relay.key.public)" > output) & - CLIENT1PID=$! - (cd client2 && cat testfile | bclient send -u me@me.com -p foobar http://127.0.0.1:8080/v1/relay "$(cat ../client1/relay.key.public)") - wait $CLIENT1PID - cat client1/output - - if [ "$(cat client2/testfile)" != "$(cat client1/output)" ]; then - echo "input != output" - exit 1 - fi - - echo "Showing some stats ..." - echo '.timeout 1000' | sqlite3 buckets_relay.sqlite - brelay stats -} - -rm -r _tests -set -xe -mkdir -p _tests -(cd _tests && dotest) diff --git a/tests/tnetstring.nim b/tests/tnetstring.nim index 89a304d..823e70e 100644 --- a/tests/tnetstring.nim +++ b/tests/tnetstring.nim @@ -21,6 +21,14 @@ suite "decode": test "incomplete": expect(IncompleteNetstring): discard nsdecode("7:bana") + expect(IncompleteNetstring): + discard nsdecode("") + expect(IncompleteNetstring): + discard nsdecode("1") + expect(IncompleteNetstring): + discard nsdecode("10:") + expect(IncompleteNetstring): + discard nsdecode("10:1234567890") test "2 strings": var idx = 0 diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 1032ec4..9b79999 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -151,6 +151,35 @@ suite "PublishNote": let data = bob.pop(Note) check data.note_data == "somedata" check data.note_topic == "sometopic" + + test "same topic": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "somedata", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "new data", + )) + let err = alice.pop(Error) + check err.err_cmd == PublishNote + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + let data = bob.pop(Note) + check data.note_data == "somedata" + check data.note_topic == "sometopic" test "fetch first": let relay = testRelay() From 922230c43415900a30ded62291907afd72afdbe4 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 29 Oct 2025 13:42:05 -0400 Subject: [PATCH 16/46] Start of functional tests, but there's a SIGSEGV --- src/objs.nim | 6 +++ src/proto2.nim | 4 -- src/sampleclient.nim | 81 +++++++++++++++++++++++++++++++++++++++ src/server2.nim | 91 ++++++++++++++++++++++++++++++-------------- tests/functional.sh | 19 +++++++++ tests/tnetstring.nim | 51 +++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 33 deletions(-) create mode 100644 src/sampleclient.nim create mode 100644 tests/functional.sh diff --git a/src/objs.nim b/src/objs.nim index 22a6f20..8361f48 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -266,6 +266,12 @@ proc nsdecode*(x: string, maxlen = MAX_NETSTRING): string = var idx = 0 return nsdecode(x, idx, maxlen = maxlen) +proc nschop*(x: var string, maxlen = MAX_NETSTRING): string = + ## Get the first netstring from a string and return it. + ## Also remove the first netstring from the passed-in string + var idx = 0 + result = nsdecode(x, idx, maxlen = maxlen) + x.delete(0..(idx-1)) proc serialize*(kind: MessageKind): char = case kind diff --git a/src/proto2.nim b/src/proto2.nim index 1860e6b..9a6dda4 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -347,7 +347,6 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay conn.sendOkay cmd.kind except: conn.sendError("Duplicate topic", cmd.kind, Generic) - echo "FRANK post sendError" of FetchNote: if cmd.fetch_topic.len > RELAY_MAX_TOPIC_SIZE: conn.sendError("Topic too long", cmd.kind, TooLarge) @@ -443,9 +442,6 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay chunk_key: key, chunk_val: none[string](), )) - echo "FRANK post case statement" - when LOG_COMMS: - info "[" & conn.pubkey.abbr & "] DONE " & $cmd #------------------------------------------------------------------- # Utilities #------------------------------------------------------------------- diff --git a/src/sampleclient.nim b/src/sampleclient.nim new file mode 100644 index 0000000..399d5b1 --- /dev/null +++ b/src/sampleclient.nim @@ -0,0 +1,81 @@ +import std/asyncdispatch +import std/base64 +import std/json +import std/unittest + +import ws + +import ./objs +import ./proto2 +import ./server2 + +proc saveKeys(filename: string, keys: KeyPair) = + writeFile(filename, pretty(%* { + "pk": base64.encode(keys.pk.string), + "sk": base64.encode(keys.sk.string), + })) + +proc loadKeys(filename: string): KeyPair = + let data = readFile(filename).parseJson + ( + base64.decode(data["pk"].getStr()).PublicKey, + base64.decode(data["sk"].getStr()).SecretKey, + ) + +proc authenticatedWS(url: string, keys: KeyPair): NetstringSocket = + let ws = waitFor newWebSocket(url) + var ns = newNetstringSocket(ws) + let who = waitFor ns.receiveMessage() + let sig = keys.sk.sign(who.who_challenge) + ns.sendCommand(RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: keys.pk)) + let ok = waitFor ns.receiveMessage() + doAssert ok.kind == Okay, $ok + return ns + +proc publishNote(ns: NetstringSocket, topic: string, data: string) = + ns.sendCommand(RelayCommand( + kind: PublishNote, + pub_topic: topic, + pub_data: data, + )) + let res = waitFor ns.receiveMessage() + if res.kind == Okay: + discard + elif res.kind == Error: + raise ValueError.newException("Error publishing note: " & $res.err_code & " " & res.err_message) + +proc batteryOfTests(url: string) = + let alice = genkeys() + let bob = genkeys() + let ws_alice = authenticatedWS(url, alice) + ws_alice.publishNote("topic", "data") + try: + ws_alice.publishNote("topic", "data") + raise CatchableError.newException("Failed to raise an error") + except ValueError: + discard + +when isMainModule: + import argparse + var p = newParser: + option("-k", "--keyfile", default=some("client1.keys")) + option("-u", "--url", default=some("ws://127.0.0.1:9000/ws")) + command("genkeys"): + run: + saveKeys(opts.parentOpts.keyfile, genkeys()) + command("publishnote"): + arg("topic") + arg("data") + run: + let keys = loadKeys(opts.parentOpts.keyfile) + let ws = authenticatedWS(opts.parentOpts.url, keys) + ws.publishNote(opts.topic, opts.data) + command("tests"): + run: + batteryOfTests(opts.parentOpts.url) + try: + p.run() + except UsageError as e: + stderr.writeLine getCurrentExceptionMsg() + quit(1) + \ No newline at end of file diff --git a/src/server2.nim b/src/server2.nim index 86c4381..bd9c673 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -6,49 +6,82 @@ import std/strutils import nimja import ws +import lowdb/sqlite -const - favicon_png = slurp"static/favicon.png" - logo_png = slurp"static/logo.png" - version = slurp"../CHANGELOG.md".split(" ")[1] -static: - echo "version: ", version +import ./proto2 +import ./objs -var connections = newSeq[WebSocket]() +type + NetstringSocket* = ref object + buf: string + socket: WebSocket -proc cb(req: Request) {.async, gcsafe.} = - if req.url.path == "/": - var html = "" - compileTemplateFile("templates/index.nimja", baseDir = getScriptDir(), autoEscape = true, varname = "html") - await req.respond(Http200, html) - elif req.url.path == "/static/favicon.png": - await req.respond(Http200, favicon_png) - elif req.url.path == "/static/logo.png": - await req.respond(Http200, logo_png) - elif req.url.path == "/ws": +proc newNetstringSocket*(sock: WebSocket): NetstringSocket = + new(result) + result.socket = sock + +proc receiveString*(ns: NetstringSocket): Future[string] {.async.} = + while true: try: - var ws = await newWebSocket(req) - connections.add ws - # await ws.send("Welcome to simple chat server") - while ws.readyState == Open: - let packet = await ws.receiveStrPacket() - # echo "Received packet: " & packet - # for other in connections: - # if other.readyState == Open: - # asyncCheck other.send(packet) + return ns.buf.nschop() + except IncompleteNetstring: + discard + let packet = await ns.socket.receiveStrPacket() + ns.buf &= packet + +proc sendString*(ns: NetstringSocket, msg: string): Future[void] {.async.} = + ns.socket.send(nsencode(msg)) + +proc sendCommand*(ns: NetstringSocket, cmd: RelayCommand) = + asyncCheck ns.sendString(cmd.serialize()) + +proc receiveCommand*(ns: NetstringSocket): Future[RelayCommand] {.async.} = + let s = await ns.receiveString() + return RelayCommand.deserialize(s) + +proc sendMessage*(ns: NetstringSocket, msg: RelayMessage) = + asyncCheck ns.sendString(msg.serialize()) + +proc receiveMessage*(ns: NetstringSocket): Future[RelayMessage] {.async.} = + let s = await ns.receiveString() + return RelayMessage.deserialize(s) + +var relay: Relay[NetstringSocket] + +proc handleWebsocket(req: Request) {.async, gcsafe.} = + var ws = await newWebSocket(req) + var ns = newNetstringSocket(ws) + var conn = relay.initAuth(ns) + while ns.socket.readyState == Open: + let cmd = try: + await ns.receiveCommand() except WebSocketClosedError: - echo "Socket closed. " + break except WebSocketProtocolMismatchError: echo "Socket tried to use an unknown protocol: ", getCurrentExceptionMsg() + break except WebSocketError: echo "Unexpected socket error: ", getCurrentExceptionMsg() - await req.respond(Http200, "done") + break + except CatchableError: + echo "CatchableError: ", getCurrentExceptionMsg() + break + relay.handleCommand(conn, cmd) + relay.disconnect(conn) + await req.respond(Http200, "done") + +proc cb(req: Request) {.async, gcsafe.} = + if req.url.path == "/ws": + await req.handleWebsocket() else: await req.respond(Http404, "Not found") proc main(database: string, port: Port, address = "127.0.0.1") = var L = newConsoleLogger() addHandler(L) + info "Database: ", database + var db = open(database, "", "", "") + relay = newRelay[NetstringSocket](db) var server = newAsyncHttpServer() info &"Serving on {address}:{port.int}" waitFor server.serve(port, cb, address = address) @@ -56,7 +89,7 @@ proc main(database: string, port: Port, address = "127.0.0.1") = when isMainModule: import argparse var p = newParser: - option("-d", "--database", help="Database") + option("-d", "--database", default=some("brelay.sqlite"), help="Database") command("server"): option("-p", "--port", default=some("9000")) option("-a", "--address", default=some("127.0.0.1")) diff --git a/tests/functional.sh b/tests/functional.sh new file mode 100644 index 0000000..948289f --- /dev/null +++ b/tests/functional.sh @@ -0,0 +1,19 @@ + +set -x +rm brelay.sqlite + +PORT=9000 +nim c -o:/tmp/server2 src/server2.nim +/tmp/server2 server & +SERVERPID="$!" +while ! nc -z 127.0.0.1 $PORT; do + echo "Waiting for $PORT" + sleep 1 +done +nim r src/sampleclient.nim publishnote topic1 value1 +nim r src/sampleclient.nim publishnote topic1 value1 || echo failed on purpose +RC=$? + + +pkill -P "$SERVERPID" +exit "$RC" diff --git a/tests/tnetstring.nim b/tests/tnetstring.nim index 823e70e..9b2cd54 100644 --- a/tests/tnetstring.nim +++ b/tests/tnetstring.nim @@ -67,3 +67,54 @@ suite "decode": discard nsdecode("5:apple,", maxlen=4) expect(NetstringError): discard nsdecode("200:a", maxlen=100) + +suite "chop": + + test "basic": + var s = "5:apple," + check nschop(s) == "apple" + check s == "" + + test "incomplete": + var s = "" + expect(IncompleteNetstring): + s = "7:bana" + discard nschop(s) + check s == "7:bana" + expect(IncompleteNetstring): + s = "" + discard nschop(s) + check s == "" + expect(IncompleteNetstring): + s = "1" + discard nschop(s) + check s == "1" + expect(IncompleteNetstring): + s = "10:" + discard nschop(s) + check s == "10:" + expect(IncompleteNetstring): + s = "10:1234567890" + discard nschop(s) + check s == "10:1234567890" + + test "2 strings": + var s = "5:apple,3:f\x00o," + check nschop(s) == "apple" + check nschop(s) == "f\x00o" + check s == "" + + test "leftover": + var s = "3:foo,2:ba" + check nschop(s) == "foo" + check s == "2:ba" + + test "newline delimiter": + var s = "5:apple\n" + check nschop(s) == "apple" + check s == "" + + test "empty string": + var s = "0:," + check s.nschop() == "" + check s == "" From 4ddde1c9b4045ffb2ba7771303d25ab113261a31 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 29 Oct 2025 15:34:53 -0400 Subject: [PATCH 17/46] Before reply-returning --- src/proto2.nim | 8 ++++---- src/sampleclient.nim | 4 ++-- src/server2.nim | 19 +++++++++++++------ tests/tproto2.nim | 4 ++-- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/proto2.nim b/src/proto2.nim index 9a6dda4..6955e1a 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -164,10 +164,10 @@ proc newRelay*[T](db: DbConn): Relay[T] = result.clients = newTable[PublicKey, RelayConnection[T]]() db.updateSchema() -template sendMessage*[T](conn: RelayConnection[T], msg: RelayMessage) = - when LOG_COMMS: - info "[" & conn.pubkey.abbr & "] <- " & $msg - conn.sender.sendMessage(msg) +# template sendMessage*[T](conn: RelayConnection[T], msg: RelayMessage) = +# when LOG_COMMS: +# info "[" & conn.pubkey.abbr & "] <- " & $msg +# conn.sender.sendMessage(msg) template sendError*[T](conn: RelayConnection[T], msg: string, cmd: CommandKind, code: ErrorCode) = conn.sendMessage(RelayMessage( diff --git a/src/sampleclient.nim b/src/sampleclient.nim index 399d5b1..cec5ee6 100644 --- a/src/sampleclient.nim +++ b/src/sampleclient.nim @@ -27,13 +27,13 @@ proc authenticatedWS(url: string, keys: KeyPair): NetstringSocket = var ns = newNetstringSocket(ws) let who = waitFor ns.receiveMessage() let sig = keys.sk.sign(who.who_challenge) - ns.sendCommand(RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: keys.pk)) + waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: keys.pk)) let ok = waitFor ns.receiveMessage() doAssert ok.kind == Okay, $ok return ns proc publishNote(ns: NetstringSocket, topic: string, data: string) = - ns.sendCommand(RelayCommand( + waitFor ns.sendCommand(RelayCommand( kind: PublishNote, pub_topic: topic, pub_data: data, diff --git a/src/server2.nim b/src/server2.nim index bd9c673..2ba41ea 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -30,17 +30,23 @@ proc receiveString*(ns: NetstringSocket): Future[string] {.async.} = ns.buf &= packet proc sendString*(ns: NetstringSocket, msg: string): Future[void] {.async.} = - ns.socket.send(nsencode(msg)) + echo "ns.socket.send ", msg.nice + await ns.socket.send(nsencode(msg)) + echo "ns.socket.send DONE ", msg.nice -proc sendCommand*(ns: NetstringSocket, cmd: RelayCommand) = - asyncCheck ns.sendString(cmd.serialize()) +proc sendCommand*(ns: NetstringSocket, cmd: RelayCommand): Future[void] {.async.} = + echo "asyncCheck sendString ", $cmd + await ns.sendString(cmd.serialize()) + echo "asyncCheck sendString DONE ", $cmd proc receiveCommand*(ns: NetstringSocket): Future[RelayCommand] {.async.} = let s = await ns.receiveString() return RelayCommand.deserialize(s) -proc sendMessage*(ns: NetstringSocket, msg: RelayMessage) = - asyncCheck ns.sendString(msg.serialize()) +proc sendMessage*(conn: RelayConnection[NetstringSocket], msg: RelayMessage) = + echo "asyncCheck sendMessage ", $msg + asyncCheck conn.sender.sendString(msg.serialize()) + echo "asyncCheck sendMessage DONE ", $msg proc receiveMessage*(ns: NetstringSocket): Future[RelayMessage] {.async.} = let s = await ns.receiveString() @@ -84,7 +90,8 @@ proc main(database: string, port: Port, address = "127.0.0.1") = relay = newRelay[NetstringSocket](db) var server = newAsyncHttpServer() info &"Serving on {address}:{port.int}" - waitFor server.serve(port, cb, address = address) + asyncCheck server.serve(port, cb, address = address) + runForever() when isMainModule: import argparse diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 9b79999..4d1a67f 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -36,8 +36,8 @@ proc newTestClient*(keys: KeyPair): TestClient = result.pk = keys.pk result.sk = keys.sk -proc sendMessage*(c: var TestClient, msg: RelayMessage) = - c.received.addLast(msg) +proc sendMessage*(conn: RelayConnection[TestClient], msg: RelayMessage) = + conn.sender.received.addLast(msg) proc pop*(c: var TestClient): RelayMessage = c.received.popFirst() From 883c7d910e26fc43b8d3ddb056881c0a18fdd1ce Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 29 Oct 2025 16:45:18 -0400 Subject: [PATCH 18/46] It's not pretty, but it works --- src/proto2.nim | 5 ----- src/server2.nim | 33 +++++++++++++++++++++------------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/proto2.nim b/src/proto2.nim index 6955e1a..8bf91a7 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -164,11 +164,6 @@ proc newRelay*[T](db: DbConn): Relay[T] = result.clients = newTable[PublicKey, RelayConnection[T]]() db.updateSchema() -# template sendMessage*[T](conn: RelayConnection[T], msg: RelayMessage) = -# when LOG_COMMS: -# info "[" & conn.pubkey.abbr & "] <- " & $msg -# conn.sender.sendMessage(msg) - template sendError*[T](conn: RelayConnection[T], msg: string, cmd: CommandKind, code: ErrorCode) = conn.sendMessage(RelayMessage( kind: Error, diff --git a/src/server2.nim b/src/server2.nim index 2ba41ea..593e235 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -3,6 +3,7 @@ import std/asynchttpserver import std/logging import std/strformat import std/strutils +import std/deques import nimja import ws @@ -15,6 +16,13 @@ type NetstringSocket* = ref object buf: string socket: WebSocket + + QueuedMessage* = tuple + socket: NetstringSocket + msg: RelayMessage + +var relay: Relay[NetstringSocket] +var message_queue = initDeque[QueuedMessage]() proc newNetstringSocket*(sock: WebSocket): NetstringSocket = new(result) @@ -30,34 +38,35 @@ proc receiveString*(ns: NetstringSocket): Future[string] {.async.} = ns.buf &= packet proc sendString*(ns: NetstringSocket, msg: string): Future[void] {.async.} = - echo "ns.socket.send ", msg.nice await ns.socket.send(nsencode(msg)) - echo "ns.socket.send DONE ", msg.nice proc sendCommand*(ns: NetstringSocket, cmd: RelayCommand): Future[void] {.async.} = - echo "asyncCheck sendString ", $cmd await ns.sendString(cmd.serialize()) - echo "asyncCheck sendString DONE ", $cmd proc receiveCommand*(ns: NetstringSocket): Future[RelayCommand] {.async.} = let s = await ns.receiveString() return RelayCommand.deserialize(s) -proc sendMessage*(conn: RelayConnection[NetstringSocket], msg: RelayMessage) = - echo "asyncCheck sendMessage ", $msg - asyncCheck conn.sender.sendString(msg.serialize()) - echo "asyncCheck sendMessage DONE ", $msg - proc receiveMessage*(ns: NetstringSocket): Future[RelayMessage] {.async.} = let s = await ns.receiveString() return RelayMessage.deserialize(s) -var relay: Relay[NetstringSocket] +proc sendMessage*(ns: NetstringSocket, msg: RelayMessage) {.async.} = + await ns.sendString(msg.serialize()) + +proc sendMessage*(conn: RelayConnection[NetstringSocket], msg: RelayMessage) = + message_queue.addLast((conn.sender, msg)) + +proc sendQueuedMessages*() {.async.} = + while message_queue.len > 0: + let (sock, msg) = message_queue.popFirst() + await sock.sendMessage(msg) proc handleWebsocket(req: Request) {.async, gcsafe.} = var ws = await newWebSocket(req) var ns = newNetstringSocket(ws) var conn = relay.initAuth(ns) + await sendQueuedMessages() while ns.socket.readyState == Open: let cmd = try: await ns.receiveCommand() @@ -73,6 +82,7 @@ proc handleWebsocket(req: Request) {.async, gcsafe.} = echo "CatchableError: ", getCurrentExceptionMsg() break relay.handleCommand(conn, cmd) + await sendQueuedMessages() relay.disconnect(conn) await req.respond(Http200, "done") @@ -90,8 +100,7 @@ proc main(database: string, port: Port, address = "127.0.0.1") = relay = newRelay[NetstringSocket](db) var server = newAsyncHttpServer() info &"Serving on {address}:{port.int}" - asyncCheck server.serve(port, cb, address = address) - runForever() + waitFor server.serve(port, cb, address = address) when isMainModule: import argparse From d16bc3a586e989a9a07f2fe0a5ff27da65abc6c5 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 29 Oct 2025 16:47:32 -0400 Subject: [PATCH 19/46] Run functional tests --- .github/workflows/main.yml | 4 ++++ tests/functional.sh | 10 ++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c238da..4b524be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,6 +33,10 @@ jobs: run: | export SHOW_LOGS=1 tests/all.sh + - name: Functional tests + run: | + export SHOW_LOGS=1 + tests/functional.sh docker: runs-on: ubuntu-latest diff --git a/tests/functional.sh b/tests/functional.sh index 948289f..dd42ba4 100644 --- a/tests/functional.sh +++ b/tests/functional.sh @@ -1,19 +1,17 @@ -set -x -rm brelay.sqlite +set -xe +rm brelay.sqlite || echo okay to fail PORT=9000 nim c -o:/tmp/server2 src/server2.nim +nim c -o:/tmp/sampleclient src/sampleclient.nim /tmp/server2 server & SERVERPID="$!" while ! nc -z 127.0.0.1 $PORT; do echo "Waiting for $PORT" sleep 1 done -nim r src/sampleclient.nim publishnote topic1 value1 -nim r src/sampleclient.nim publishnote topic1 value1 || echo failed on purpose +/tmp/sampleclient tests RC=$? - - pkill -P "$SERVERPID" exit "$RC" From b2e6f3e8b737411f12148c626ba770bbf5069725 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 29 Oct 2025 17:01:26 -0400 Subject: [PATCH 20/46] Fix test runner --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4b524be..27b5a4e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,9 +34,10 @@ jobs: export SHOW_LOGS=1 tests/all.sh - name: Functional tests + shell: bash run: | export SHOW_LOGS=1 - tests/functional.sh + /bin/bash tests/functional.sh docker: runs-on: ubuntu-latest From 4f95e3a3f02de713c356cac61a78cd9739c86651 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Thu, 30 Oct 2025 09:33:04 -0400 Subject: [PATCH 21/46] Not sure cli is worth the time --- .gitignore | 1 + src/sampleclient.nim | 91 +++++++++++++++++++++++++++++------------- tests/tfunctional.nim | 92 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 28 deletions(-) create mode 100644 tests/tfunctional.nim diff --git a/.gitignore b/.gitignore index 545f16b..e2750f3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ relay.sqlite libs *.keys *.sqlite +tests/bin diff --git a/src/sampleclient.nim b/src/sampleclient.nim index cec5ee6..f7f04a4 100644 --- a/src/sampleclient.nim +++ b/src/sampleclient.nim @@ -1,30 +1,57 @@ import std/asyncdispatch import std/base64 import std/json -import std/unittest +import std/sequtils +import argparse import ws import ./objs import ./proto2 import ./server2 +export KeyPair, genkeys + proc saveKeys(filename: string, keys: KeyPair) = writeFile(filename, pretty(%* { "pk": base64.encode(keys.pk.string), "sk": base64.encode(keys.sk.string), })) -proc loadKeys(filename: string): KeyPair = - let data = readFile(filename).parseJson - ( - base64.decode(data["pk"].getStr()).PublicKey, - base64.decode(data["sk"].getStr()).SecretKey, +proc serializeKeys*(keys: KeyPair): string = + base64.encode($(%* { + "pk": keys.pk.string, + "sk": keys.sk.string, + })) + +proc deserializeKeys*(text: string): KeyPair = + let data = base64.decode(text).parseJson() + return ( + data["pk"].getStr().PublicKey, + data["sk"].getStr().SecretKey, ) -proc authenticatedWS(url: string, keys: KeyPair): NetstringSocket = +proc loadKeys(location: string): KeyPair = + if location.startsWith("file:"): + let parts = location.split(":", maxsplit = 1) + let filename = parts[1] + let data = readFile(filename).parseJson + return ( + base64.decode(data["pk"].getStr()).PublicKey, + base64.decode(data["sk"].getStr()).SecretKey, + ) + elif location.startsWith("inline:"): + let parts = location.split(":", maxsplit = 1) + return deserializeKeys(parts[1]) + else: + raise ValueError.newException("Invalid key location") + +proc newWS*(url: string): NetstringSocket = let ws = waitFor newWebSocket(url) - var ns = newNetstringSocket(ws) + return newNetstringSocket(ws) + +proc authenticatedWS*(url: string, keys: KeyPair): NetstringSocket = + var ns = newWS(url) let who = waitFor ns.receiveMessage() let sig = keys.sk.sign(who.who_challenge) waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: keys.pk)) @@ -32,50 +59,58 @@ proc authenticatedWS(url: string, keys: KeyPair): NetstringSocket = doAssert ok.kind == Okay, $ok return ns -proc publishNote(ns: NetstringSocket, topic: string, data: string) = +proc publishNote*(ns: NetstringSocket, topic: string, data: string) = waitFor ns.sendCommand(RelayCommand( kind: PublishNote, pub_topic: topic, pub_data: data, )) let res = waitFor ns.receiveMessage() + echo "publishNote res: ", $res if res.kind == Okay: discard elif res.kind == Error: raise ValueError.newException("Error publishing note: " & $res.err_code & " " & res.err_message) -proc batteryOfTests(url: string) = - let alice = genkeys() - let bob = genkeys() - let ws_alice = authenticatedWS(url, alice) - ws_alice.publishNote("topic", "data") - try: - ws_alice.publishNote("topic", "data") - raise CatchableError.newException("Failed to raise an error") - except ValueError: - discard +proc fetchNote*(ns: NetstringSocket, topic: string): string = + waitFor ns.sendCommand(RelayCommand( + kind: FetchNote, + fetch_topic: topic, + )) + let res = waitFor ns.receiveMessage() + if res.kind == Note: + return res.note_data + else: + raise ValueError.newException("No such note: " & topic) -when isMainModule: - import argparse - var p = newParser: - option("-k", "--keyfile", default=some("client1.keys")) +proc cli*(args: openArray[string], outp = stdout) = + var cliparser = newParser: + option("-k", "--keys", default=some("file:client1.keys")) option("-u", "--url", default=some("ws://127.0.0.1:9000/ws")) command("genkeys"): + arg("filename") run: - saveKeys(opts.parentOpts.keyfile, genkeys()) + saveKeys(opts.filename, genkeys()) command("publishnote"): arg("topic") arg("data") run: - let keys = loadKeys(opts.parentOpts.keyfile) + let keys = loadKeys(opts.parentOpts.keys) let ws = authenticatedWS(opts.parentOpts.url, keys) ws.publishNote(opts.topic, opts.data) - command("tests"): + command("fetchnote"): + arg("topic") run: - batteryOfTests(opts.parentOpts.url) + let keys = loadKeys(opts.parentOpts.keys) + let ws = authenticatedWS(opts.parentOpts.url, keys) + outp.write(ws.fetchNote(opts.topic)) try: - p.run() + cliparser.run(toSeq(args)) except UsageError as e: stderr.writeLine getCurrentExceptionMsg() quit(1) + +when isMainModule: + cli(commandLineParams()) + \ No newline at end of file diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim new file mode 100644 index 0000000..33a7636 --- /dev/null +++ b/tests/tfunctional.nim @@ -0,0 +1,92 @@ +import std/net +import std/unittest +import std/osproc +import std/os + +import sampleclient + +const TESTPORT = 12222.Port + +proc startServer(port: Port): Process = + let database = absolutePath(currentSourcePath().parentDir() / "func.sqlite") + if database.fileExists: + echo "removing ", database.relativePath(".") + removeFile(database) + + let bin = absolutePath(currentSourcePath().parentDir() / "bin" / "server2") + bin.parentDir.createDir() + echo "compiling ", bin.relativePath(".") + echo execProcess("nim", + workingDir = currentSourcePath().parentDir().parentDir(), + args = [ + "c", "-o:" & bin, "src"/"server2.nim", + ], + options = {poStdErrToStdOut, poUsePath} + ) + echo "compiled ", bin.relativePath(".") + + startProcess(bin, + workingDir = currentSourcePath().parentDir(), + args = [ + "--database", database, + "server", + "--port", $port, + ], options = {poStdErrToStdOut, poUsePath} + ) + +proc waitForPort(port: Port) = + while true: + sleep(100) + try: + let socket = newSocket() + socket.connect("127.0.0.1", port) + socket.close() + break + except: + echo "waiting for port ", $port + echo "port open: ", $port + +proc stop(p: Process) = + p.terminate() + +var server = startServer(TESTPORT) +waitForPort(TESTPORT) + +proc runcli(keys: KeyPair, args: openArray[string]): string = + var allargs = @[ + "--keys", "inline:" & serializeKeys(keys), + "--url", "ws://127.0.0.1:" & $TESTPORT & "/ws", + ] + allargs.add(args) + echo "> ", $args + cli(allargs) + +proc runcliq(keys: KeyPair, args: openArray[string]) = + ## Run a command and discard output + discard runcli(keys, args) + +var alice = genkeys() +var bob = genkeys() +var carl = genkeys() + +suite "publishnote": + + test "basic": + runcliq(alice, ["publishnote", "basic", "data"]) + check runcli(bob, ["fetchnote", "basic"]) == "data" + expect(CatchableError): + runcliq(bob, ["fetchnote", "basic"]) + + + test "duplicate": + runcliq(alice, ["publishnote", "topic", "data"]) + expect(CatchableError): + runcliq(alice, ["publishnote", "topic", "data2"]) + + + +test "smoke": + check true + + +server.terminate() \ No newline at end of file From 9d1c6fddaed072e172cb7fc858e997f64401a818 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Thu, 30 Oct 2025 13:13:12 -0400 Subject: [PATCH 22/46] Functional tests work --- .github/workflows/main.yml | 5 -- src/objs.nim | 22 +++--- src/proto2.nim | 6 +- src/sampleclient.nim | 118 +++++++++++------------------- src/server2.nim | 13 ++-- tests/functional.sh | 17 ----- tests/tfunctional.nim | 142 ++++++++++++++++++++++++++++++------- tests/tproto2.nim | 79 ++++++++++++++++++++- tests/tserde2.nim | 1 - 9 files changed, 257 insertions(+), 146 deletions(-) delete mode 100644 tests/functional.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27b5a4e..4c238da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,11 +33,6 @@ jobs: run: | export SHOW_LOGS=1 tests/all.sh - - name: Functional tests - shell: bash - run: | - export SHOW_LOGS=1 - /bin/bash tests/functional.sh docker: runs-on: ubuntu-latest diff --git a/src/objs.nim b/src/objs.nim index 8361f48..814102c 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -28,7 +28,8 @@ type ErrorCode* = enum Generic = 0 - TooLarge = 1 + NotAllowed = 1 + TooLarge = 2 RelayMessage* = object case kind*: MessageKind @@ -64,7 +65,6 @@ type of Iam: iam_pubkey*: PublicKey iam_signature*: string - iam_credentials*: string ## For the future possibility of credentials of PublishNote: pub_topic*: string pub_data*: string @@ -165,7 +165,7 @@ proc `$`*(cmd: RelayCommand): string = result.add $cmd.kind & "(" case cmd.kind of Iam: - result.add &"{cmd.iam_pubkey.nice.abbr} sig={cmd.iam_signature.nicelong} creds={cmd.iam_credentials.nicelong}" + result.add &"{cmd.iam_pubkey.nice.abbr} sig={cmd.iam_signature.nicelong}" of PublishNote: result.add &"'{cmd.pub_topic.nice.abbr}' val={cmd.pub_data.nicelong}" of FetchNote: @@ -174,13 +174,11 @@ proc `$`*(cmd: RelayCommand): string = result.add &"{cmd.send_dst.nice.abbr} val={cmd.send_val.nicelong}" of StoreChunk: result.add &"{cmd.chunk_key.nice.abbr}={cmd.chunk_val.nicelong} dst=[" - for dst in cmd.chunk_dst: - result.add dst.nice.abbr & ", " + result.add cmd.chunk_dst.mapIt(it.nice.abbr).join(", ") result.add "]" of GetChunks: result.add &"{cmd.chunk_src.nice.abbr} keys=[" - for key in cmd.chunk_keys: - result.add key.nice.abbr & ", " + result.add cmd.chunk_keys.mapIt(it.nice.abbr).join(", ") result.add "]" result.add ")" @@ -190,7 +188,7 @@ proc `==`*(a, b: RelayCommand): bool = else: case a.kind of Iam: - return a.iam_pubkey == b.iam_pubkey and a.iam_signature == b.iam_signature and a.iam_credentials == b.iam_credentials + return a.iam_pubkey == b.iam_pubkey and a.iam_signature == b.iam_signature of PublishNote: return a.pub_topic == b.pub_topic and a.pub_data == b.pub_data of FetchNote: @@ -314,12 +312,14 @@ proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = proc serialize*(err: ErrorCode): char = case err of Generic: '0' - of TooLarge: '1' + of NotAllowed: '1' + of TooLarge: '2' proc deserialize*(typ: typedesc[ErrorCode], ch: char): ErrorCode = case ch of '0': Generic - of '1': TooLarge + of '1': NotAllowed + of '2': TooLarge else: raise ValueError.newException("Unknown ErrorCode: " & ch) proc serialize*(keys: seq[PublicKey]): string = @@ -419,7 +419,6 @@ proc serialize*(cmd: RelayCommand): string = of Iam: result &= cmd.iam_pubkey.string.nsencode result &= cmd.iam_signature.nsencode - result &= cmd.iam_credentials.nsencode of PublishNote: result &= cmd.pub_topic.nsencode result &= cmd.pub_data.nsencode @@ -447,7 +446,6 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = kind: Iam, iam_pubkey: s.nsdecode(idx).PublicKey, iam_signature: s.nsdecode(idx), - iam_credentials: s.nsdecode(idx), ) of PublishNote: var idx = 1 diff --git a/src/proto2.nim b/src/proto2.nim index 8bf91a7..bc8dae6 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -16,7 +16,7 @@ import libsodium/sodium import ./objs; export objs -const LOG_COMMS = not defined(release) +const LOG_COMMS* = not defined(release) const TESTMODE = defined(testmode) and not defined(release) type @@ -290,6 +290,10 @@ proc delExpiredChunks(relay: Relay) = proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: RelayCommand) = when LOG_COMMS: info "[" & conn.pubkey.abbr & "] DO " & $cmd + if conn.pubkey.string == "" and cmd.kind != Iam: + conn.sendError("Not allowed", cmd.kind, NotAllowed) + return + case cmd.kind of Iam: try: diff --git a/src/sampleclient.nim b/src/sampleclient.nim index f7f04a4..46e6a2d 100644 --- a/src/sampleclient.nim +++ b/src/sampleclient.nim @@ -1,56 +1,19 @@ import std/asyncdispatch -import std/base64 -import std/json -import std/sequtils +import std/options -import argparse import ws import ./objs import ./proto2 import ./server2 -export KeyPair, genkeys - -proc saveKeys(filename: string, keys: KeyPair) = - writeFile(filename, pretty(%* { - "pk": base64.encode(keys.pk.string), - "sk": base64.encode(keys.sk.string), - })) - -proc serializeKeys*(keys: KeyPair): string = - base64.encode($(%* { - "pk": keys.pk.string, - "sk": keys.sk.string, - })) - -proc deserializeKeys*(text: string): KeyPair = - let data = base64.decode(text).parseJson() - return ( - data["pk"].getStr().PublicKey, - data["sk"].getStr().SecretKey, - ) - -proc loadKeys(location: string): KeyPair = - if location.startsWith("file:"): - let parts = location.split(":", maxsplit = 1) - let filename = parts[1] - let data = readFile(filename).parseJson - return ( - base64.decode(data["pk"].getStr()).PublicKey, - base64.decode(data["sk"].getStr()).SecretKey, - ) - elif location.startsWith("inline:"): - let parts = location.split(":", maxsplit = 1) - return deserializeKeys(parts[1]) - else: - raise ValueError.newException("Invalid key location") +export KeyPair, genkeys, NetstringSocket proc newWS*(url: string): NetstringSocket = let ws = waitFor newWebSocket(url) return newNetstringSocket(ws) -proc authenticatedWS*(url: string, keys: KeyPair): NetstringSocket = +proc newRelayClient*(url: string, keys: KeyPair): NetstringSocket = var ns = newWS(url) let who = waitFor ns.receiveMessage() let sig = keys.sk.sign(who.who_challenge) @@ -59,58 +22,59 @@ proc authenticatedWS*(url: string, keys: KeyPair): NetstringSocket = doAssert ok.kind == Okay, $ok return ns -proc publishNote*(ns: NetstringSocket, topic: string, data: string) = - waitFor ns.sendCommand(RelayCommand( +proc publishNote*(ns: NetstringSocket, topic: string, data: string) {.async.} = + await ns.sendCommand(RelayCommand( kind: PublishNote, pub_topic: topic, pub_data: data, )) - let res = waitFor ns.receiveMessage() - echo "publishNote res: ", $res + let res = await ns.receiveMessage() if res.kind == Okay: discard elif res.kind == Error: raise ValueError.newException("Error publishing note: " & $res.err_code & " " & res.err_message) -proc fetchNote*(ns: NetstringSocket, topic: string): string = - waitFor ns.sendCommand(RelayCommand( +proc fetchNote*(ns: NetstringSocket, topic: string): Future[string] {.async.} = + await ns.sendCommand(RelayCommand( kind: FetchNote, fetch_topic: topic, )) - let res = waitFor ns.receiveMessage() + let res = await ns.receiveMessage() if res.kind == Note: return res.note_data else: raise ValueError.newException("No such note: " & topic) -proc cli*(args: openArray[string], outp = stdout) = - var cliparser = newParser: - option("-k", "--keys", default=some("file:client1.keys")) - option("-u", "--url", default=some("ws://127.0.0.1:9000/ws")) - command("genkeys"): - arg("filename") - run: - saveKeys(opts.filename, genkeys()) - command("publishnote"): - arg("topic") - arg("data") - run: - let keys = loadKeys(opts.parentOpts.keys) - let ws = authenticatedWS(opts.parentOpts.url, keys) - ws.publishNote(opts.topic, opts.data) - command("fetchnote"): - arg("topic") - run: - let keys = loadKeys(opts.parentOpts.keys) - let ws = authenticatedWS(opts.parentOpts.url, keys) - outp.write(ws.fetchNote(opts.topic)) - try: - cliparser.run(toSeq(args)) - except UsageError as e: - stderr.writeLine getCurrentExceptionMsg() - quit(1) +proc sendData*(ns: NetstringSocket, dst: PublicKey, val: string) {.async.} = + await ns.sendCommand(RelayCommand( + kind: SendData, + send_dst: dst, + send_val: val, + )) + +proc getData*(ns: NetstringSocket): Future[string] {.async.} = + let res = await ns.receiveMessage() + if res.kind == Data: + return res.data_val + else: + raise ValueError.newException("Expecting Data but got: " & $res) + +proc storeChunk*(ns: NetstringSocket, dsts: seq[PublicKey], key: string, val: string) {.async.} = + await ns.sendCommand(RelayCommand( + kind: StoreChunk, + chunk_dst: dsts, + chunk_key: key, + chunk_val: val, + )) -when isMainModule: - cli(commandLineParams()) - - \ No newline at end of file +proc getChunk*(ns: NetstringSocket, src: PublicKey, key: string): Future[Option[string]] {.async.} = + await ns.sendCommand(RelayCommand( + kind: GetChunks, + chunk_src: src, + chunk_keys: @[key], + )) + let res = await ns.receiveMessage() + if res.kind == Chunk: + return res.chunk_val + else: + raise ValueError.newException("Expecing Chunk but got: " & $res) diff --git a/src/server2.nim b/src/server2.nim index 593e235..6cbe290 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -41,6 +41,8 @@ proc sendString*(ns: NetstringSocket, msg: string): Future[void] {.async.} = await ns.socket.send(nsencode(msg)) proc sendCommand*(ns: NetstringSocket, cmd: RelayCommand): Future[void] {.async.} = + when LOG_COMMS: + info "[client] -> " & $cmd await ns.sendString(cmd.serialize()) proc receiveCommand*(ns: NetstringSocket): Future[RelayCommand] {.async.} = @@ -49,7 +51,10 @@ proc receiveCommand*(ns: NetstringSocket): Future[RelayCommand] {.async.} = proc receiveMessage*(ns: NetstringSocket): Future[RelayMessage] {.async.} = let s = await ns.receiveString() - return RelayMessage.deserialize(s) + let res = RelayMessage.deserialize(s) + when LOG_COMMS: + info "[client] <- " & $res + return res proc sendMessage*(ns: NetstringSocket, msg: RelayMessage) {.async.} = await ns.sendString(msg.serialize()) @@ -73,13 +78,13 @@ proc handleWebsocket(req: Request) {.async, gcsafe.} = except WebSocketClosedError: break except WebSocketProtocolMismatchError: - echo "Socket tried to use an unknown protocol: ", getCurrentExceptionMsg() + warn "Socket tried to use an unknown protocol: ", getCurrentExceptionMsg() break except WebSocketError: - echo "Unexpected socket error: ", getCurrentExceptionMsg() + warn "Unexpected socket error: ", getCurrentExceptionMsg() break except CatchableError: - echo "CatchableError: ", getCurrentExceptionMsg() + warn "CatchableError: ", getCurrentExceptionMsg() break relay.handleCommand(conn, cmd) await sendQueuedMessages() diff --git a/tests/functional.sh b/tests/functional.sh deleted file mode 100644 index dd42ba4..0000000 --- a/tests/functional.sh +++ /dev/null @@ -1,17 +0,0 @@ - -set -xe -rm brelay.sqlite || echo okay to fail - -PORT=9000 -nim c -o:/tmp/server2 src/server2.nim -nim c -o:/tmp/sampleclient src/sampleclient.nim -/tmp/server2 server & -SERVERPID="$!" -while ! nc -z 127.0.0.1 $PORT; do - echo "Waiting for $PORT" - sleep 1 -done -/tmp/sampleclient tests -RC=$? -pkill -P "$SERVERPID" -exit "$RC" diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index 33a7636..538651e 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -1,9 +1,17 @@ +import std/asyncdispatch import std/net -import std/unittest -import std/osproc +import std/options import std/os +import std/osproc +import std/unittest + +import ./util import sampleclient +import server2 +import proto2 + +import ws const TESTPORT = 12222.Port @@ -31,9 +39,12 @@ proc startServer(port: Port): Process = "--database", database, "server", "--port", $port, - ], options = {poStdErrToStdOut, poUsePath} + ], options = {poStdErrToStdOut, poUsePath, poParentStreams} ) +proc isPortOpen(port: Port): bool = + discard + proc waitForPort(port: Port) = while true: sleep(100) @@ -52,41 +63,118 @@ proc stop(p: Process) = var server = startServer(TESTPORT) waitForPort(TESTPORT) -proc runcli(keys: KeyPair, args: openArray[string]): string = - var allargs = @[ - "--keys", "inline:" & serializeKeys(keys), - "--url", "ws://127.0.0.1:" & $TESTPORT & "/ws", - ] - allargs.add(args) - echo "> ", $args - cli(allargs) +proc serverURL(): string = + "ws://127.0.0.1:" & $TESTPORT & "/ws" -proc runcliq(keys: KeyPair, args: openArray[string]) = - ## Run a command and discard output - discard runcli(keys, args) +proc testClient(keys: KeyPair): NetstringSocket = + let url = serverURL() + newRelayClient(url, keys) + +proc testClient(): NetstringSocket = + testClient(genkeys()) -var alice = genkeys() -var bob = genkeys() -var carl = genkeys() suite "publishnote": test "basic": - runcliq(alice, ["publishnote", "basic", "data"]) - check runcli(bob, ["fetchnote", "basic"]) == "data" - expect(CatchableError): - runcliq(bob, ["fetchnote", "basic"]) - + var alice = testClient() + var bob = testClient() + waitFor alice.publishNote("basic", "data") + check (waitFor bob.fetchNote("basic")) == "data" + var p = bob.fetchNote("basic") + check p.finished == false + waitFor alice.publishNote("basic", "again") + check (waitFor p) == "again" test "duplicate": - runcliq(alice, ["publishnote", "topic", "data"]) + var alice = testClient() + waitFor alice.publishNote("dupe", "data") expect(CatchableError): - runcliq(alice, ["publishnote", "topic", "data2"]) + waitFor alice.publishNote("dupe", "data again") + +suite "data": + test "basic": + var akeys = genkeys() + var bkeys = genkeys() + var alice = testClient(akeys) + var bob = testClient(bkeys) + waitFor alice.sendData(bkeys.pk, "hey, bob?") + check (waitFor bob.getData()) == "hey, bob?" + waitFor bob.sendData(akeys.pk, "hi, alice!") + check (waitFor alice.getData()) == "hi, alice!" + test "offline": + var akeys = genkeys() + var bkeys = genkeys() + var alice = testClient(akeys) + waitFor alice.sendData(bkeys.pk, "message \x01") + waitFor alice.sendData(bkeys.pk, "message \x02") + waitFor alice.sendData(bkeys.pk, "message \x00null") -test "smoke": - check true + var bob = testClient(bkeys) + check (waitFor bob.getData()) == "message \x01" + check (waitFor bob.getData()) == "message \x02" + check (waitFor bob.getData()) == "message \x00null" +suite "chunks": -server.terminate() \ No newline at end of file + test "basic": + var akeys = genkeys() + var bkeys = genkeys() + var ckeys = genkeys() + var alice = testClient(akeys) + var bob = testClient(bkeys) + var carl = testClient(ckeys) + waitFor alice.storeChunk(@[bkeys.pk], "chunk1", "data1") + waitFor alice.storeChunk(@[bkeys.pk, ckeys.pk], "chunk2", "data2") + waitFor alice.storeChunk(@[bkeys.pk], "chunk3", "data3") + waitFor alice.storeChunk(@[bkeys.pk], "chunk3", "data3updated") + + check (waitFor bob.getChunk(akeys.pk, "chunk1")) == some("data1") + check (waitFor bob.getChunk(akeys.pk, "chunk2")) == some("data2") + check (waitFor bob.getChunk(akeys.pk, "chunk3")) == some("data3updated") + check (waitFor bob.getChunk(akeys.pk, "chunk4")).isNone() + + check (waitFor carl.getChunk(akeys.pk, "chunk1")).isNone() + check (waitFor carl.getChunk(akeys.pk, "chunk2")) == some("data2") + check (waitFor carl.getChunk(akeys.pk, "chunk3")).isNone() + check (waitFor carl.getChunk(akeys.pk, "chunk4")).isNone() + +suite "invalid": + + test "malformed": + let ws = waitFor newWebSocket(serverURL()) + waitFor ws.send("garbage") + waitForPort(TESTPORT) + + test "too big": + let ws = waitFor newWebSocket(serverURL()) + waitFor ws.send("123456789:foooooo") + waitForPort(TESTPORT) + + test "RelayMessage": + var keys = genkeys() + let ws = waitFor newWebSocket(serverURL()) + let ns = newNetstringSocket(ws) + let who = waitFor ns.receiveMessage() + let sig = keys.sk.sign(who.who_challenge) + waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: keys.pk)) + let ok = waitFor ns.receiveMessage() + checkpoint $ok + check ok.kind == Okay + + waitFor ws.send(nsencode(serialize(RelayMessage( + kind: Note, + note_topic: "hey", + note_data: "data", + )))) + waitFor sleepAsync(1000) + + var legit = testClient() + waitFor legit.publishNote("something", "here") + check (waitFor legit.fetchNote("something")) == "here" + check server.running() + waitForPort(TESTPORT) + +server.terminate() diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 4d1a67f..b3524ce 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -55,8 +55,8 @@ proc pop(conn: var RelayConnection[TestClient], expected: MessageKind): RelayMes try: result = conn.sender.pop() except IndexDefect: - raise IndexDefect.newException("Error getting message of kind: " & $expected) - doAssert result.kind == expected + raise IndexDefect.newException("No message found while expecting kind: " & $expected) + doAssert result.kind == expected, "Expected " & $expected & " but got " & $result proc msgCount(conn: var RelayConnection[TestClient]): int = conn.sender.received.len @@ -65,6 +65,11 @@ proc pk(conn: var RelayConnection[TestClient]): PublicKey = conn.sender.pk proc sk(conn: var RelayConnection[TestClient]): SecretKey = conn.sender.sk proc keys(conn: var RelayConnection[TestClient]): KeyPair = (conn.sender.pk, conn.sender.sk) +proc anonConn(relay: Relay): RelayConnection[TestClient] = + let client = newTestClient(genkeys()) + var conn = relay.initAuth(client) + return conn + proc authenticatedConn(relay: Relay, keys: KeyPair): RelayConnection[TestClient] = let client = newTestClient(keys) var conn = relay.initAuth(client) @@ -571,3 +576,73 @@ suite "store": let err = alice.pop(Error) check err.err_cmd == StoreChunk check err.err_code == TooLarge + +suite "anonymous": + + test "PublishNote": + let relay = testRelay() + var alice = relay.anonConn() + discard alice.pop(Who) + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "foo", + pub_data: "bar" + )) + let err = alice.pop(Error) + check err.err_cmd == PublishNote + check err.err_code == NotAllowed + + test "FetchNote": + let relay = testRelay() + var alice = relay.anonConn() + discard alice.pop(Who) + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "foo", + )) + let err = alice.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotAllowed + + test "SendData": + let relay = testRelay() + var keys = genkeys() + var alice = relay.anonConn() + discard alice.pop(Who) + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_dst: keys.pk, + send_val: "bar", + )) + let err = alice.pop(Error) + check err.err_cmd == SendData + check err.err_code == NotAllowed + + test "StoreChunk": + let relay = testRelay() + var keys = genkeys() + var alice = relay.anonConn() + discard alice.pop(Who) + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: @[keys.pk], + chunk_key: "foo", + chunk_val: "bar", + )) + let err = alice.pop(Error) + check err.err_cmd == StoreChunk + check err.err_code == NotAllowed + + test "GetChunks": + let relay = testRelay() + var keys = genkeys() + var alice = relay.anonConn() + discard alice.pop(Who) + relay.handleCommand(alice, RelayCommand( + kind: GetChunks, + chunk_src: keys.pk, + chunk_keys: @["foo"], + )) + let err = alice.pop(Error) + check err.err_cmd == GetChunks + check err.err_code == NotAllowed diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 74fc3d5..6a61582 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -34,7 +34,6 @@ test "RelayCommand": kind: Iam, iam_pubkey: "hey".PublicKey, iam_signature: "foo", - iam_credentials: "somecreds", ) of PublishNote: RelayCommand(kind: PublishNote, pub_topic: "topic", pub_data: "data") of FetchNote: RelayCommand(kind: FetchNote, fetch_topic: "topic") From 24328bffa69a146f0e64268fda5e98c3d86795b3 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Thu, 30 Oct 2025 13:30:06 -0400 Subject: [PATCH 23/46] Fix test running --- src/objs.nim | 7 +++---- tests/all.sh | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/objs.nim b/src/objs.nim index 814102c..5ff8469 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -7,7 +7,6 @@ ## This file should be kept free of dependencies other than the stdlib ## as it's meant to be referenced by outside libraries. -import std/base64 import std/hashes import std/options import std/sequtils @@ -327,9 +326,9 @@ proc serialize*(keys: seq[PublicKey]): string = result &= nsencode(key.string) proc deserializePubKeys*(val: string): seq[PublicKey] = - var idx = 0 - while idx < val.len: - result.add(val.nsdecode(idx).PublicKey) + var val = val + while val.len > 0: + result.add(val.nschop().PublicKey) proc serialize*(s: seq[string]): string = for item in s: diff --git a/tests/all.sh b/tests/all.sh index 80c0ace..521a554 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -1,13 +1,31 @@ #!/bin/bash RC=0 +FAILED= +PASSED= for filename in $(ls tests/t*.nim); do echo $filename nim r "$filename" rc1=$? if [ ! "$rc1" == "0" ]; then RC="$rc1" + FAILED="$filename $FAILED" + else + PASSED="$filename $PASSED" fi done +for filename in $PASSED; do + echo "PASSED: $filename" +done +for filename in $FAILED; do + echo "FAILED: $filename" +done + +if [ "$RC" == "0" ]; then + echo "OK" +else + echo "exit=$RC" +fi + exit "$RC" From 8fc7642242d961b6ba6f5c909e99423b19605300 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 31 Oct 2025 10:42:19 -0400 Subject: [PATCH 24/46] Add hashing challenge to clients --- README.md | 6 +- nim.cfg | 2 +- pkger/deps.json | 18 ++--- src/objs.nim | 76 ++++++++++++++----- src/proto2.nim | 122 ++++++++++++++++++++++++------ src/sampleclient.nim | 4 +- tests/tfunctional.nim | 6 +- tests/tproto2.nim | 169 ++++++++++++++++++++++++++++-------------- tests/tserde2.nim | 17 ++++- 9 files changed, 304 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index 8c39360..1ef7ff8 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Clients send the following commands: | Command | Description | |----------------|-------------| -| `Iam` | In response to a `Who` event, proves that this client has the private key | +| `Iam` | In response to a `Who` event, proves that this client has the private key and does some spam mitigation | | `PublishNote` | Send a few bytes to another client addressed by topic (good for key exchange) | | `FetchNote` | Request a note addressed by topic | | `SendData` | Store/forward bytes to other clients, addressed by relay-authenticated public keys | @@ -71,7 +71,7 @@ The relay server sends the following events: |-----------------|------------------------------------| | `Okay` | Sent when certain commands succeed | | `Error` | Sent when commands fail | -| `Who` | Challenge for authenticating a client's public/private keys | +| `Who` | Challenge for authenticating a client's public/private keys and spam mitigation | | `Note` | Data payload of a note requested by `FetchNote` | | `Data` | Data payload from another client, addressed by relay-authenticated public key | | `Chunk` | Data payload response to `GetChunk` request | @@ -81,7 +81,7 @@ The relay server sends the following events: Authentication happens like this: 1. On connection, server sends `Who(challenge=ABCD...)` -2. Client responds with `Iam(pubkey=MYPK..., signature=SIGN...)` +2. Client responds with a signed PoW hash `Iam(pubkey=MYPK..., signature=SIGN...)` 3. If the signature is correct, server sends `Okay(cmd=Iam)` ``` diff --git a/nim.cfg b/nim.cfg index a8f6599..b66a710 100644 --- a/nim.cfg +++ b/nim.cfg @@ -1,9 +1,9 @@ ### PKGER START - DO NOT EDIT BELOW ######### ---path:"pkger/lazy/libsodium" --path:"pkger/lazy/lowdb/src" --path:"pkger/lazy/ws/src" --path:"pkger/lazy/nimja/src" --path:"pkger/lazy/db_connector/src" --path:"pkger/lazy/argparse/src" +--path:"pkger/lazy/libsodium" ### PKGER END - DO NOT EDIT ABOVE ########### diff --git a/pkger/deps.json b/pkger/deps.json index 6fa80ad..596e6d2 100644 --- a/pkger/deps.json +++ b/pkger/deps.json @@ -1,14 +1,5 @@ { "pinned": { - "libsodium": { - "pkgname": "libsodium", - "parent": "", - "src": { - "url": "https://github.com/FedericoCeratto/nim-libsodium", - "kind": "git" - }, - "sha": "0258efe4e7f48e22daedf26f70e3efbe830abfb5" - }, "lowdb": { "pkgname": "lowdb", "parent": "", @@ -53,6 +44,15 @@ "kind": "git" }, "sha": "98c7c99bfbcaae750ac515a6fd603f85ed68668f" + }, + "libsodium": { + "pkgname": "libsodium", + "parent": "", + "src": { + "url": "https://github.com/iffy/nim-libsodium", + "kind": "git" + }, + "sha": "d3a116e1047b6b1363dfee08876a8f1c1bb1a128" } } } \ No newline at end of file diff --git a/src/objs.nim b/src/objs.nim index 5ff8469..db03f3a 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -17,6 +17,17 @@ type PublicKey* = distinct string SecretKey* = distinct string + Challenge* = tuple + bits: int + rand: string + opslimit: int + memlimit: int + + ChallengeAnswer* = tuple + nonce: int + output: string + signature: string + MessageKind* = enum Who Okay @@ -33,7 +44,7 @@ type RelayMessage* = object case kind*: MessageKind of Who: - who_challenge*: string + who_challenge*: Challenge of Okay: ok_cmd*: CommandKind of Error: @@ -63,7 +74,7 @@ type case kind*: CommandKind of Iam: iam_pubkey*: PublicKey - iam_signature*: string + iam_answer*: ChallengeAnswer of PublishNote: pub_topic*: string pub_data*: string @@ -125,11 +136,17 @@ proc `==`*(a,b: PublicKey): bool {.borrow.} proc abbr*(a: PublicKey): string = abbr(a.nice) +proc `$`*(ch: Challenge): string = + result = &"({ch.bits} {ch.opslimit} {ch.memlimit} rand={ch.rand.nice})" + +proc `$`*(ans: ChallengeAnswer): string = + result = &"({ans.nonce} {ans.output} {ans.signature.nicelong})" + proc `$`*(msg: RelayMessage): string = result.add $msg.kind & "(" case msg.kind of Who: - result.add "challenge=" & msg.who_challenge.nicelong + result.add "challenge=" & $msg.who_challenge of Okay: result.add &"cmd={msg.ok_cmd}" of Error: @@ -164,7 +181,7 @@ proc `$`*(cmd: RelayCommand): string = result.add $cmd.kind & "(" case cmd.kind of Iam: - result.add &"{cmd.iam_pubkey.nice.abbr} sig={cmd.iam_signature.nicelong}" + result.add &"{cmd.iam_pubkey.nice.abbr} {cmd.iam_answer}" of PublishNote: result.add &"'{cmd.pub_topic.nice.abbr}' val={cmd.pub_data.nicelong}" of FetchNote: @@ -187,7 +204,7 @@ proc `==`*(a, b: RelayCommand): bool = else: case a.kind of Iam: - return a.iam_pubkey == b.iam_pubkey and a.iam_signature == b.iam_signature + return a.iam_pubkey == b.iam_pubkey and a.iam_answer == b.iam_answer of PublishNote: return a.pub_topic == b.pub_topic and a.pub_data == b.pub_data of FetchNote: @@ -309,17 +326,40 @@ proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = else: raise ValueError.newException("Unknown CommandKind: " & val) proc serialize*(err: ErrorCode): char = - case err - of Generic: '0' - of NotAllowed: '1' - of TooLarge: '2' + chr(err.ord) proc deserialize*(typ: typedesc[ErrorCode], ch: char): ErrorCode = - case ch - of '0': Generic - of '1': NotAllowed - of '2': TooLarge - else: raise ValueError.newException("Unknown ErrorCode: " & ch) + try: + ErrorCode(ord(ch)) + except: + raise ValueError.newException("Unknown ErrorCode: " & ch) + +proc serialize*(chal: Challenge): string = + result.add nsencode($chal.bits) + result.add nsencode(chal.rand) + result.add nsencode($chal.opslimit) + result.add nsencode($chal.memlimit) + +proc deserialize*(typ: typedesc[Challenge], val: string): Challenge = + var idx = 0 + let bits = val.nsdecode(idx).parseInt() + let rand = val.nsdecode(idx) + let opslimit = val.nsdecode(idx).parseInt() + let memlimit = val.nsdecode(idx).parseInt() + return (bits, rand, opslimit, memlimit) + +proc serialize*(ans: ChallengeAnswer): string = + result &= nsencode($ans.nonce) + result &= nsencode(ans.output) + result &= nsencode(ans.signature) + +proc deserialize*(typ: typedesc[ChallengeAnswer], val: string): ChallengeAnswer = + var idx = 0 + return ( + nonce: val.nsdecode(idx).parseInt(), + output: val.nsdecode(idx), + signature: val.nsdecode(idx), + ) proc serialize*(keys: seq[PublicKey]): string = for key in keys: @@ -343,7 +383,7 @@ proc serialize*(msg: RelayMessage): string = result &= msg.kind.serialize() case msg.kind of Who: - result &= msg.who_challenge + result &= msg.who_challenge.serialize() of Okay: result &= msg.ok_cmd.serialize() of Error: @@ -369,7 +409,7 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = let kind = MessageKind.deserialize(s[0]) case kind of Who: - return RelayMessage(kind: Who, who_challenge: s[1..^1]) + return RelayMessage(kind: Who, who_challenge: Challenge.deserialize(s[1..^1])) of Okay: return RelayMessage(kind: Okay, ok_cmd: CommandKind.deserialize(s[1])) of Error: @@ -417,7 +457,7 @@ proc serialize*(cmd: RelayCommand): string = case cmd.kind of Iam: result &= cmd.iam_pubkey.string.nsencode - result &= cmd.iam_signature.nsencode + result &= cmd.iam_answer.serialize().nsencode of PublishNote: result &= cmd.pub_topic.nsencode result &= cmd.pub_data.nsencode @@ -444,7 +484,7 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = return RelayCommand( kind: Iam, iam_pubkey: s.nsdecode(idx).PublicKey, - iam_signature: s.nsdecode(idx), + iam_answer: ChallengeAnswer.deserialize(s.nsdecode(idx)), ) of PublishNote: var idx = 1 diff --git a/src/proto2.nim b/src/proto2.nim index bc8dae6..d8e3a61 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -13,6 +13,7 @@ import std/times import lowdb/sqlite import libsodium/sodium +import libsodium/sodium_sizes import ./objs; export objs @@ -31,7 +32,7 @@ type RelayConnection*[T] = ref object sender*: T pubkey*: PublicKey ## The authenticated pubkey - challenge: string + challenge: Option[Challenge] relay*: Relay[T] when TESTMODE: @@ -43,6 +44,90 @@ when TESTMODE: proc resetSkew*() = TIME_SKEW = 0 +#------------------------------------------------------------------- +# Utilities +#------------------------------------------------------------------- +proc genkeys*(): KeyPair = + let (pk, sk) = crypto_sign_keypair() + result = (pk.PublicKey, sk.SecretKey) + +proc sign*(key: SecretKey, message: string): string = + ## Sign a message with the given secret key + result = crypto_sign_detached(key.string, message) + +proc is_valid_signature*(key: PublicKey, plaintext: string, signature: string): bool = + try: + crypto_sign_verify_detached(key.string, plaintext, signature) + return true + except SodiumError: + return false + except CatchableError: + return false + +const + CHALLENGE_BITS = when TESTMODE: 1 else: 5 + +proc generateChallenge*(bits = CHALLENGE_BITS, opslimit = crypto_pwhash_opslimit_interactive().int, memlimit = crypto_pwhash_memlimit_interactive().int): Challenge = + return ( + bits: bits, + rand: randombytes(32), + opslimit: opslimit, + memlimit: memlimit, + ) + +proc sigContents*(ch: Challenge, nonce: int, output: string): string = + ch.serialize & nsencode($nonce) & output + +proc firstBits(s: string, n: int): string = + ## Returns the first `n` bits of the string `s` as a binary string. + if s.len * 8 < n: + raise ValueError.newException("String not long enough") + var bitsLeft = n + for i in 0.. 0: + result.add(if (byte and (1'u8 shl bit)) != 0: '1' else: '0') + dec bitsLeft + else: + return result + if bitsLeft > 0: + # this should never happen, but just in case + raise ValueError.newException("String not long enough") + return result + +proc answer*(ch: Challenge, sk: SecretKey): ChallengeAnswer = + ## Answer a hashcash challenge and sign the result + var nonce = 0 + let serialized = ch.serialize() + var start = getTime() + var expected_prefix = '0'.repeat(ch.bits) + while true: + let inp = serialized & ":" & $nonce + let output = crypto_pwhash_str(inp, + opslimit = ch.opslimit.csize_t, + memlimit = ch.memlimit.csize_t) + let hashpart = base64.decode(output.split('$')[^1]) + let bits = hashpart.firstBits(ch.bits) + if bits == expected_prefix: + var diff = getTime() - start + return ( + nonce: nonce, + output: output, + signature: sk.sign(sigContents(ch, nonce, output)), + ) + nonce.inc() + +proc is_valid_answer*(pk: PublicKey, ch: Challenge, answer: ChallengeAnswer): bool = + ## Verify the signed challenge answer + if not pk.is_valid_signature(sigContents(ch, answer.nonce, answer.output), answer.signature): + return false + let inp = ch.serialize() & ":" & $answer.nonce + if crypto_pwhash_str_verify(answer.output, inp) == false: + return false + return true + + #------------------------------------------------------------------- # Database #------------------------------------------------------------------- @@ -146,8 +231,8 @@ proc `$`*[T](conn: RelayConnection[T]): string = result = "RelayConnectiong(" result &= &"pubkey={conn.pubkey.abbr} " result &= &"sender={conn.sender}" - if conn.challenge != "": - result &= " cha=" & base64.encode(conn.challenge) + if conn.challenge.isSome: + result &= " cha=" & base64.encode(conn.challenge.get()) result &= ")" proc `$`*[T](tab: TableRef[PublicKey, RelayConnection[T]]): string = @@ -181,10 +266,10 @@ template sendOkay*[T](conn: RelayConnection[T], cmd: CommandKind) = proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = new(result) result.sender = client - result.challenge = randombytes(32) + result.challenge = some(generateChallenge()) result.sendMessage(RelayMessage( kind: Who, - who_challenge: result.challenge, + who_challenge: result.challenge.get(), )) result.relay = relay @@ -296,18 +381,23 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay case cmd.kind of Iam: - try: - crypto_sign_verify_detached(cmd.iam_pubkey.string, conn.challenge, cmd.iam_signature) - except SodiumError: - conn.sendError("Invalid signature", cmd.kind, Generic) + if conn.challenge.isNone: + conn.sendError("Already authenticated", cmd.kind, Generic) return + let challenge = conn.challenge.get() + conn.challenge = none[Challenge]() # disable future authentication attempts + + try: + if not is_valid_answer(cmd.iam_pubkey, challenge, cmd.iam_answer): + conn.sendError("Invalid answer", cmd.kind, Generic) + return except CatchableError: - conn.sendError("Error validating signature", cmd.kind, Generic) + conn.sendError("Invalid answer", cmd.kind, Generic) return + # successful connection conn.pubkey = cmd.iam_pubkey relay.clients[conn.pubkey] = conn - conn.challenge = "" # disable authentication info &"[{conn.pubkey.abbr}] connected" conn.sendOkay cmd.kind @@ -441,13 +531,3 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay chunk_key: key, chunk_val: none[string](), )) -#------------------------------------------------------------------- -# Utilities -#------------------------------------------------------------------- -proc genkeys*(): KeyPair = - let (pk, sk) = crypto_sign_keypair() - result = (pk.PublicKey, sk.SecretKey) - -proc sign*(key: SecretKey, message: string): string = - ## Sign a message with the given secret key - result = crypto_sign_detached(key.string, message) diff --git a/src/sampleclient.nim b/src/sampleclient.nim index 46e6a2d..feb5914 100644 --- a/src/sampleclient.nim +++ b/src/sampleclient.nim @@ -16,8 +16,8 @@ proc newWS*(url: string): NetstringSocket = proc newRelayClient*(url: string, keys: KeyPair): NetstringSocket = var ns = newWS(url) let who = waitFor ns.receiveMessage() - let sig = keys.sk.sign(who.who_challenge) - waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: keys.pk)) + let answer = who.who_challenge.answer(keys.sk) + waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: keys.pk)) let ok = waitFor ns.receiveMessage() doAssert ok.kind == Okay, $ok return ns diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index 538651e..3699d1a 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -27,7 +27,7 @@ proc startServer(port: Port): Process = echo execProcess("nim", workingDir = currentSourcePath().parentDir().parentDir(), args = [ - "c", "-o:" & bin, "src"/"server2.nim", + "c", "-d:testmode", "-o:" & bin, "src"/"server2.nim", ], options = {poStdErrToStdOut, poUsePath} ) @@ -158,8 +158,8 @@ suite "invalid": let ws = waitFor newWebSocket(serverURL()) let ns = newNetstringSocket(ws) let who = waitFor ns.receiveMessage() - let sig = keys.sk.sign(who.who_challenge) - waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: keys.pk)) + let answer = who.who_challenge.answer(keys.sk) + waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: keys.pk)) let ok = waitFor ns.receiveMessage() checkpoint $ok check ok.kind == Okay diff --git a/tests/tproto2.nim b/tests/tproto2.nim index b3524ce..155f57e 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -37,6 +37,8 @@ proc newTestClient*(keys: KeyPair): TestClient = result.sk = keys.sk proc sendMessage*(conn: RelayConnection[TestClient], msg: RelayMessage) = + when LOG_COMMS: + info "[" & conn.pubkey.abbr & "] <- " & $msg conn.sender.received.addLast(msg) proc pop*(c: var TestClient): RelayMessage = @@ -75,8 +77,13 @@ proc authenticatedConn(relay: Relay, keys: KeyPair): RelayConnection[TestClient] var conn = relay.initAuth(client) let who = conn.pop() doAssert who.kind == Who - let sig = client.sk.sign(who.who_challenge) - relay.handleCommand(conn, RelayCommand(kind: Iam, iam_signature: sig, iam_pubkey: client.pk)) + let answer = who.who_challenge.answer(client.sk) + echo "answer: ", $answer + relay.handleCommand(conn, RelayCommand( + kind: Iam, + iam_answer: answer, + iam_pubkey: client.pk + )) let ok = conn.pop() doAssert ok.kind == Okay doAssert ok.ok_cmd == Iam @@ -97,12 +104,12 @@ suite "Auth": checkpoint "who?" var alice = relay.initAuth(aclient) let who = alice.pop(Who) - check who.who_challenge != "" + check who.who_challenge != default(Challenge) checkpoint $who checkpoint "iam" - let signature = aclient.sk.sign(who.who_challenge) - relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + let answer = who.who_challenge.answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) discard alice.pop(Okay) test "iam twice": @@ -112,25 +119,71 @@ suite "Auth": checkpoint "who?" var alice = relay.initAuth(aclient) let who = alice.pop(Who) - check who.who_challenge != "" + check who.who_challenge != default(Challenge) checkpoint "iam" - let signature = aclient.sk.sign(who.who_challenge) - relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + let answer = who.who_challenge.answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) discard alice.pop(Okay) - relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) check alice.pop().kind == Error - test "iam invalid sig": + test "iam invalid answer": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + let answer = generateChallenge().answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + + test "iam invalid signature": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + var bobkeys = genkeys() + let answer = generateChallenge().answer(bobkeys.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + + test "invalid opslimit": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + let answer = generateChallenge(opslimit = who.who_challenge.opslimit - 1).answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + + test "invalid memlimit": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + let answer = generateChallenge(memlimit = who.who_challenge.memlimit - 32).answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + + test "invalid bits": let relay = testRelay() let aclient = newTestClient(genkeys()) var alice = relay.initAuth(aclient) let who = alice.pop(Who) - let signature = aclient.sk.sign(who.who_challenge & "garbage") - relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: aclient.pk)) + let answer = generateChallenge(bits = who.who_challenge.bits - 1).answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) let err = alice.pop(Error) check err.err_cmd == Iam @@ -237,23 +290,24 @@ suite "PublishNote": check err.err_code == TooLarge check err.err_cmd == PublishNote - test "expiration": - let relay = testRelay() - var alice = relay.authenticatedConn() - relay.handleCommand(alice, RelayCommand( - kind: PublishNote, - pub_topic: "topic", - pub_data: "a", - )) - check alice.pop(Okay).ok_cmd == PublishNote - - skewTime(RELAY_NOTE_DURATION) - skewTime(1) - relay.handleCommand(alice, RelayCommand( - kind: FetchNote, - fetch_topic: "topic", - )) - check alice.msgCount == 0 + when not defined(release): + test "expiration": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "topic", + pub_data: "a", + )) + check alice.pop(Okay).ok_cmd == PublishNote + + skewTime(RELAY_NOTE_DURATION) + skewTime(1) + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "topic", + )) + check alice.msgCount == 0 test "fetch note again": let relay = testRelay() @@ -375,21 +429,22 @@ suite "data": check err.err_code == TooLarge check err.err_cmd == SendData - test "expiration": - let relay = testRelay() - var alice = relay.authenticatedConn() - var bob = relay.authenticatedConn() - relay.disconnect(bob) + when not defined(release): + test "expiration": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + relay.disconnect(bob) - relay.handleCommand(alice, RelayCommand( - kind: SendData, - send_dst: bob.pk, - send_val: "hello", - )) + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_dst: bob.pk, + send_val: "hello", + )) - skewTime(RELAY_MESSAGE_DURATION + 1) - var bob2 = relay.authenticatedConn(bob.keys) - check bob2.msgCount == 0 + skewTime(RELAY_MESSAGE_DURATION + 1) + var bob2 = relay.authenticatedConn(bob.keys) + check bob2.msgCount == 0 proc storeChunk(conn: var RelayConnection[TestClient], key: string, val: string, dst = newSeq[PublicKey]()) = @@ -494,21 +549,23 @@ suite "store": alice.storeChunk("key", "first") check bob.getChunk(alice, "key").isNone() - test "expiration": - let relay = testRelay() - var alice = relay.authenticatedConn() - alice.storeChunk("key", "foo") - skewTime(RELAY_MESSAGE_DURATION + 1) - check alice.getChunk(alice, "key").isNone() + when not defined(release): + test "expiration": + let relay = testRelay() + var alice = relay.authenticatedConn() + alice.storeChunk("key", "foo") + skewTime(RELAY_MESSAGE_DURATION + 1) + check alice.getChunk(alice, "key").isNone() - test "expiration update": - let relay = testRelay() - var alice = relay.authenticatedConn() - alice.storeChunk("key", "foo") - skewTime(RELAY_MESSAGE_DURATION - 1) - alice.storeChunk("key", "foo") - skewTime(3) - check alice.getChunk(alice, "key").get() == "foo" + when not defined(release): + test "expiration update": + let relay = testRelay() + var alice = relay.authenticatedConn() + alice.storeChunk("key", "foo") + skewTime(RELAY_MESSAGE_DURATION - 1) + alice.storeChunk("key", "foo") + skewTime(3) + check alice.getChunk(alice, "key").get() == "foo" test "remove dst": let relay = testRelay() diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 6a61582..762fde6 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -4,6 +4,7 @@ import std/options import ./util import proto2 +import objs test "MessageKind": for kind in low(MessageKind)..high(MessageKind): @@ -16,7 +17,7 @@ test "CommandKind": test "RelayMessage": for kind in low(MessageKind)..high(MessageKind): let example = case kind - of Who: RelayMessage(kind: Who, who_challenge: "test") + of Who: RelayMessage(kind: Who, who_challenge: generateChallenge()) of Okay: RelayMessage(kind: Okay, ok_cmd: SendData) of Error: RelayMessage(kind: Error, err_cmd: SendData, err_code: TooLarge, err_message: "foo") of Note: RelayMessage(kind: Note, note_topic: "something", note_data: "data") @@ -24,7 +25,7 @@ test "RelayMessage": of Chunk: RelayMessage(kind: Chunk, chunk_src: "hey".PublicKey, chunk_key: "key", chunk_val: some("theval")) let serialized = example.serialize() info $example - info "serialized: " & serialized + info "serialized: " & serialized.nice check RelayMessage.deserialize(serialized) == example test "RelayCommand": @@ -33,7 +34,11 @@ test "RelayCommand": of Iam: RelayCommand( kind: Iam, iam_pubkey: "hey".PublicKey, - iam_signature: "foo", + iam_answer: ( + nonce: 1, + output: "foo", + signature: "hey", + ), ) of PublishNote: RelayCommand(kind: PublishNote, pub_topic: "topic", pub_data: "data") of FetchNote: RelayCommand(kind: FetchNote, fetch_topic: "topic") @@ -60,3 +65,9 @@ test "Chunk with none": let serialized = chunk.serialize() info "serialized: " & serialized check RelayMessage.deserialize(serialized) == chunk + +test "ErrorCodes": + for err in low(ErrorCode)..high(ErrorCode): + let serialized = err.serialize() + checkpoint "serialized.nice: " & nice($serialized) + check ErrorCode.deserialize(serialized) == err From 837647c2532dfac81d6b8f110abeac77bbeff085 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 31 Oct 2025 10:45:38 -0400 Subject: [PATCH 25/46] Add a timestamp to the challenge --- src/proto2.nim | 2 +- tests/tproto2.nim | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/proto2.nim b/src/proto2.nim index d8e3a61..ad205c3 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -70,7 +70,7 @@ const proc generateChallenge*(bits = CHALLENGE_BITS, opslimit = crypto_pwhash_opslimit_interactive().int, memlimit = crypto_pwhash_memlimit_interactive().int): Challenge = return ( bits: bits, - rand: randombytes(32), + rand: randombytes(32) & $epochTime(), opslimit: opslimit, memlimit: memlimit, ) diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 155f57e..0ddad50 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -78,7 +78,6 @@ proc authenticatedConn(relay: Relay, keys: KeyPair): RelayConnection[TestClient] let who = conn.pop() doAssert who.kind == Who let answer = who.who_challenge.answer(client.sk) - echo "answer: ", $answer relay.handleCommand(conn, RelayCommand( kind: Iam, iam_answer: answer, From 38733ac477f9a460b829c77f127c395d9a001d17 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 4 Nov 2025 10:08:34 -0500 Subject: [PATCH 26/46] Add sending/storage limits, ChunksPresent command, stats page --- README.md | 2 + config.nims | 1 + nim.cfg | 1 + pkger/deps.json | 9 ++ src/objs.nim | 84 ++++++++--- src/proto2.nim | 306 ++++++++++++++++++++++++++++++-------- src/sampleclient.nim | 55 +++++-- src/server2.nim | 217 +++++++++++++++++++++++---- src/templates/index.nimja | 60 ++++++++ src/templates/stats.nimja | 119 +++++++++++++++ tests/tfunctional.nim | 20 ++- tests/tproto2.nim | 132 +++++++++++++++- tests/tserde2.nim | 6 + 13 files changed, 874 insertions(+), 138 deletions(-) create mode 100644 src/templates/stats.nimja diff --git a/README.md b/README.md index 1ef7ff8..383ac09 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ Clients send the following commands: | `SendData` | Store/forward bytes to other clients, addressed by relay-authenticated public keys | | `StoreChunk` | Store bytes for other clients to fetch addressed by key and public key. | | `GetChunk` | Request stored chunk | +| `ChunksPresent` | Ask which chunks exist | ### Server Events @@ -75,6 +76,7 @@ The relay server sends the following events: | `Note` | Data payload of a note requested by `FetchNote` | | `Data` | Data payload from another client, addressed by relay-authenticated public key | | `Chunk` | Data payload response to `GetChunk` request | +| `ChunkStatus` | Response to `ChunksPresent` indicating which chunks exist/don't | ### Authentication diff --git a/config.nims b/config.nims index 5a61f19..56f95fa 100644 --- a/config.nims +++ b/config.nims @@ -1,6 +1,7 @@ # See LICENSE.md for licensing switch("gc", "orc") switch("threads", "off") +switch("d", "useStdLib") # rather than httpBeast import os const ROOT = currentSourcePath.parentDir() diff --git a/nim.cfg b/nim.cfg index b66a710..804914f 100644 --- a/nim.cfg +++ b/nim.cfg @@ -6,4 +6,5 @@ --path:"pkger/lazy/db_connector/src" --path:"pkger/lazy/argparse/src" --path:"pkger/lazy/libsodium" +--path:"pkger/lazy/jester" ### PKGER END - DO NOT EDIT ABOVE ########### diff --git a/pkger/deps.json b/pkger/deps.json index 596e6d2..5dd831f 100644 --- a/pkger/deps.json +++ b/pkger/deps.json @@ -53,6 +53,15 @@ "kind": "git" }, "sha": "d3a116e1047b6b1363dfee08876a8f1c1bb1a128" + }, + "jester": { + "pkgname": "jester", + "parent": "", + "src": { + "url": "https://github.com/dom96/jester/", + "kind": "git" + }, + "sha": "ac9b8541dce64feff9b53b700cab8496c1816651" } } } \ No newline at end of file diff --git a/src/objs.nim b/src/objs.nim index db03f3a..84773f4 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -35,11 +35,14 @@ type Note Data Chunk + ChunkStatus ErrorCode* = enum Generic = 0 NotAllowed = 1 TooLarge = 2 + StorageLimitExceeded = 3 + TransferLimitExceeeded = 4 RelayMessage* = object case kind*: MessageKind @@ -61,6 +64,10 @@ type chunk_src*: PublicKey chunk_key*: string chunk_val*: Option[string] + of ChunkStatus: + status_src*: PublicKey + present*: seq[string] + absent*: seq[string] CommandKind* = enum Iam @@ -69,6 +76,7 @@ type SendData StoreChunk GetChunks + ChunksPresent RelayCommand* = object case kind*: CommandKind @@ -90,10 +98,14 @@ type of GetChunks: chunk_src*: PublicKey chunk_keys*: seq[string] + of ChunksPresent: + present_src*: PublicKey + present_keys*: seq[string] const RELAY_MAX_TOPIC_SIZE* = 512 RELAY_MAX_NOTE_SIZE* = 4096 + RELAY_MAX_NOTES* = 1000 RELAY_NOTE_DURATION* = 5 * 24 * 60 * 60 RELAY_MAX_MESSAGE_SIZE* = 4096 RELAY_MAX_CHUNK_KEY_SIZE* = 4096 @@ -135,6 +147,11 @@ proc hash*(p: PublicKey): Hash {.borrow.} proc `==`*(a,b: PublicKey): bool {.borrow.} proc abbr*(a: PublicKey): string = abbr(a.nice) +proc abbr*(a: Option[PublicKey]): string = + if a.isSome: + a.get.abbr + else: + "none" proc `$`*(ch: Challenge): string = result = &"({ch.bits} {ch.opslimit} {ch.memlimit} rand={ch.rand.nice})" @@ -157,6 +174,12 @@ proc `$`*(msg: RelayMessage): string = result.add &"{msg.data_src.nice.abbr} val={msg.data_val.nicelong}" of Chunk: result.add &"{msg.chunk_src.nice.abbr} {msg.chunk_key.nice.abbr}={msg.chunk_val.nicelong}" + of ChunkStatus: + result.add &"{msg.status_src.nice.abbr} present=[" + result.add msg.present.mapIt(it.nice.abbr).join(", ") + result.add "] absent=[" + result.add msg.absent.mapIt(it.nice.abbr).join(", ") + result.add "]" result.add ")" proc `==`*(a, b: RelayMessage): bool = @@ -176,6 +199,8 @@ proc `==`*(a, b: RelayMessage): bool = return a.data_src == b.data_src and a.data_val == b.data_val of Chunk: return a.chunk_src == b.chunk_src and a.chunk_key == b.chunk_key and a.chunk_val == b.chunk_val + of ChunkStatus: + return a.status_src == b.status_src and a.present == b.present and a.absent == b.absent proc `$`*(cmd: RelayCommand): string = result.add $cmd.kind & "(" @@ -196,6 +221,10 @@ proc `$`*(cmd: RelayCommand): string = result.add &"{cmd.chunk_src.nice.abbr} keys=[" result.add cmd.chunk_keys.mapIt(it.nice.abbr).join(", ") result.add "]" + of ChunksPresent: + result.add &"{cmd.present_src.nice.abbr} keys=[" + result.add cmd.present_keys.mapIt(it.nice.abbr).join(", ") + result.add "]" result.add ")" proc `==`*(a, b: RelayCommand): bool = @@ -215,6 +244,8 @@ proc `==`*(a, b: RelayCommand): bool = return a.chunk_dst == b.chunk_dst and a.chunk_key == b.chunk_key and a.chunk_val == b.chunk_val of GetChunks: return a.chunk_src == b.chunk_src and a.chunk_keys == b.chunk_keys + of ChunksPresent: + return a.present_src == b.present_src and a.present_keys == b.present_keys #-------------------------------------------------------------- # serialization @@ -295,6 +326,7 @@ proc serialize*(kind: MessageKind): char = of Note: 'n' of Data: 'd' of Chunk: 'k' + of ChunkStatus: 's' proc deserialize*(kind: typedesc[MessageKind], val: char): MessageKind = case val @@ -304,6 +336,7 @@ proc deserialize*(kind: typedesc[MessageKind], val: char): MessageKind = of 'n': Note of 'd': Data of 'k': Chunk + of 's': ChunkStatus else: raise ValueError.newException("Unknown MessageKind: " & val) proc serialize*(kind: CommandKind): char = @@ -314,6 +347,7 @@ proc serialize*(kind: CommandKind): char = of SendData: 's' of StoreChunk: 'c' of GetChunks: 'g' + of ChunksPresent: 't' proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = case val: @@ -323,6 +357,7 @@ proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = of 's': SendData of 'c': StoreChunk of 'g': GetChunks + of 't': ChunksPresent else: raise ValueError.newException("Unknown CommandKind: " & val) proc serialize*(err: ErrorCode): char = @@ -401,6 +436,10 @@ proc serialize*(msg: RelayMessage): string = result &= msg.chunk_key.nsencode if msg.chunk_val.isSome: result &= msg.chunk_val.get().nsencode + of ChunkStatus: + result &= msg.status_src.string.nsencode + result &= nsencode(msg.present.serialize()) + result &= nsencode(msg.absent.serialize()) proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = @@ -421,35 +460,36 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = ) of Note: var idx = 1 - let note_topic = s.nsdecode(idx) - let note_data = s.nsdecode(idx) return RelayMessage( kind: Note, - note_topic: note_topic, - note_data: note_data, + note_topic: s.nsdecode(idx), + note_data: s.nsdecode(idx), ) of Data: var idx = 1 - let data_src = s.nsdecode(idx).PublicKey - let data_val = s.nsdecode(idx) return RelayMessage( kind: Data, - data_src: data_src, - data_val: data_val, + data_src: s.nsdecode(idx).PublicKey, + data_val: s.nsdecode(idx), ) of Chunk: var idx = 1 - let chunk_src = s.nsdecode(idx).PublicKey - let chunk_key = s.nsdecode(idx) - let chunk_val = if idx >= s.len: - none[string]() - else: - some(s.nsdecode(idx)) return RelayMessage( kind: Chunk, - chunk_src: chunk_src, - chunk_key: chunk_key, - chunk_val: chunk_val, + chunk_src: s.nsdecode(idx).PublicKey, + chunk_key: s.nsdecode(idx), + chunk_val: if idx >= s.len: + none[string]() + else: + some(s.nsdecode(idx)), + ) + of ChunkStatus: + var idx = 1 + return RelayMessage( + kind: ChunkStatus, + status_src: s.nsdecode(idx).PublicKey, + present: deserialize(seq[string], s.nsdecode(idx)), + absent: deserialize(seq[string], s.nsdecode(idx)), ) proc serialize*(cmd: RelayCommand): string = @@ -473,6 +513,9 @@ proc serialize*(cmd: RelayCommand): string = of GetChunks: result &= cmd.chunk_src.string.nsencode result &= nsencode(cmd.chunk_keys.serialize()) + of ChunksPresent: + result &= cmd.present_src.string.nsencode + result &= nsencode(cmd.present_keys.serialize()) proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = if s.len == 0: @@ -521,3 +564,10 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = chunk_src: s.nsdecode(idx).PublicKey, chunk_keys: deserialize(seq[string], s.nsdecode(idx)), ) + of ChunksPresent: + var idx = 1 + return RelayCommand( + kind: ChunksPresent, + present_src: s.nsdecode(idx).PublicKey, + present_keys: deserialize(seq[string], s.nsdecode(idx)), + ) diff --git a/src/proto2.nim b/src/proto2.nim index ad205c3..f950f12 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -28,12 +28,15 @@ type Relay*[T] = object db*: DbConn clients: TableRef[PublicKey, RelayConnection[T]] + max_chunk_space*: int + max_transfer_rate*: int RelayConnection*[T] = ref object sender*: T - pubkey*: PublicKey ## The authenticated pubkey + pubkey*: Option[PublicKey] ## The authenticated pubkey challenge: Option[Challenge] relay*: Relay[T] + ip*: string when TESTMODE: var TIME_SKEW = 0 @@ -183,9 +186,11 @@ proc updateSchema*(db: DbConn) = db.exec(sql"""CREATE TABLE note ( topic TEXT PRIMARY KEY, created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + src TEXT NOT NULL, data BLOB DEFAULT '' )""") db.exec(sql"CREATE INDEX note_created ON note(created)") + db.exec(sql"CREATE INDEX note_src ON note(src)") # message db.exec(sql"""CREATE TABLE message ( @@ -214,6 +219,16 @@ proc updateSchema*(db: DbConn) = PRIMARY KEY (src, key, dst), FOREIGN KEY (src, key) REFERENCES chunk(src, key) ON DELETE CASCADE )""") + + # stats + db.exec(sql"""CREATE TABLE stats_transfer ( + period TEXT NOT NULL DEFAULT(strftime('%Y-%W')), + ip TEXT NOT NULL, + pubkey TEXT NOT NULL, + data_in INTEGER DEFAULT 0, + data_out INTEGER DEFAULT 0, + PRIMARY KEY (period, ip, pubkey) + )""") #----------- in-memory stuff db.exec(sql"""CREATE TEMPORARY TABLE note_sub ( @@ -274,10 +289,83 @@ proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = result.relay = relay proc disconnect*[T](relay: Relay[T], conn: RelayConnection[T]) = - relay.db.exec(sql"DELETE FROM note_sub WHERE pubkey=?", conn.pubkey) - relay.clients.del(conn.pubkey) + if conn.pubkey.isSome: + let pubkey = conn.pubkey.get() + relay.db.exec(sql"DELETE FROM note_sub WHERE pubkey=?", pubkey) + relay.clients.del(pubkey) info &"[{conn.pubkey.abbr}] disconnected" +#------------------------------------------------------------------- +# stats +#------------------------------------------------------------------- +type + TransferTotal* = tuple + data_in: int + data_out: int + ip: string + pubkey: PublicKey + period: string + + PeriodRange* = tuple + a: string + b: string + +proc record_transfer_stat*(db: DbConn, ip: string, pubkey = "".PublicKey, data_in = 0, data_out = 0) = + db.exec(sql""" + INSERT INTO stats_transfer (ip, pubkey, data_in, data_out) + VALUES (?, ?, ?, ?) + ON CONFLICT(period, ip, pubkey) DO UPDATE SET + data_in = data_in + excluded.data_in, + data_out = data_out + excluded.data_out; + """, ip, pubkey, data_in, data_out) + +when TESTMODE: + proc record_transfer_stat_period*(db: DbConn, ip: string, pubkey = "".PublicKey, period = "", data_in = 0, data_out = 0) = + db.exec(sql""" + INSERT INTO stats_transfer (ip, pubkey, period, data_in, data_out) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(period, ip, pubkey) DO UPDATE SET + data_in = data_in + excluded.data_in, + data_out = data_out + excluded.data_out; + """, ip, pubkey, period, data_in, data_out) + +proc chunk_space_used*(db: DbConn, pubkey: PublicKey): int = + ## Return the amount of space being used by the given public key + db.getRow(sql""" + SELECT coalesce(sum(length(val)), 0) FROM chunk WHERE src = ? + """, pubkey).get()[0].i.int + +proc current_data_in*(db: DbConn, pubkey: PublicKey): int = + ## Return the amount of data that has been transferred in by the given + ## public key for the current time period + db.getRow(sql""" + SELECT coalesce(sum(data_in), 0) FROM stats_transfer + WHERE + pubkey = ? + AND period = strftime('%Y-%W') + """, pubkey).get()[0].i.int + +proc stats_transfer_total*(db: DbConn, ip = "", pubkey = "".PublicKey, period = ""): TransferTotal = + var query = "SELECT sum(data_in), sum(data_out) FROM stats_transfer" + var whereparts: seq[string] + var params: seq[DbValue] + var groupby: seq[string] + if period != "": + whereparts.add "period=?" + params.add(period.dbValue()) + if ip != "": + whereparts.add "ip=?" + params.add(ip.dbValue()) + if pubkey.string != "": + whereparts.add "pubkey=?" + params.add(pubkey.dbValue()) + if whereparts.len > 0: + query &= " WHERE " & whereparts.join(" AND ") + let orow = db.getRow(sql(query), params) + if orow.isSome(): + let row = orow.get() + return (row[0].i.int, row[1].i.int, ip, pubkey, period) + #------------------------------------------------------------------- # pub/sub notes #------------------------------------------------------------------- @@ -327,6 +415,9 @@ proc delNoteSub(relay: Relay, topic: string) = relay.db.exec(sql"DELETE FROM note_sub WHERE topic = ?", topic.DbBlob) info &"[note] del {topic}" +proc noteCount(relay: Relay, pubkey: PublicKey): int = + ## Return the number of notes currently published by this ip + relay.db.getRow(sql"SELECT count(*) FROM note WHERE src = ?", pubkey).get()[0].i.int #------------------------------------------------------------------- # send/receive data @@ -375,7 +466,7 @@ proc delExpiredChunks(relay: Relay) = proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: RelayCommand) = when LOG_COMMS: info "[" & conn.pubkey.abbr & "] DO " & $cmd - if conn.pubkey.string == "" and cmd.kind != Iam: + if conn.pubkey.isNone and cmd.kind != Iam: conn.sendError("Not allowed", cmd.kind, NotAllowed) return @@ -396,17 +487,25 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay return # successful connection - conn.pubkey = cmd.iam_pubkey - relay.clients[conn.pubkey] = conn + let pubkey = cmd.iam_pubkey + conn.pubkey = some(pubkey) + relay.clients[pubkey] = conn info &"[{conn.pubkey.abbr}] connected" conn.sendOkay cmd.kind # send all queued messages relay.delExpiredMessages() while true: - let nexto = relay.nextMessage(conn.pubkey) + let nexto = relay.nextMessage(pubkey) if nexto.isSome: - conn.sendMessage(nexto.get()) + let msg = nexto.get() + conn.sendMessage(msg) + if msg.kind == Data: + relay.db.record_transfer_stat( + ip = conn.ip, + pubkey = pubkey, + data_out = msg.data_val.len, + ) else: break of PublishNote: @@ -415,27 +514,42 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay elif cmd.pub_data.len > RELAY_MAX_NOTE_SIZE: conn.sendError("Data too long", cmd.kind, TooLarge) else: - let opubkey = relay.getNoteSub(cmd.pub_topic) - if opubkey.isSome: - # someone is waiting - var other_conn = relay.clients[opubkey.get()] - conn.sendOkay cmd.kind - other_conn.sendMessage(RelayMessage( - kind: Note, - note_data: cmd.pub_data, - note_topic: cmd.pub_topic, - )) - relay.delNoteSub(cmd.pub_topic) + let pubkey = conn.pubkey.get() + if relay.noteCount(pubkey) >= RELAY_MAX_NOTES: + conn.sendError("Too many notes", cmd.kind, StorageLimitExceeded) else: - # no one is waiting - try: - relay.db.exec(sql"INSERT INTO note (topic, data) VALUES (?, ?)", - cmd.pub_topic.DbBlob, - cmd.pub_data.DbBlob, - ) + relay.db.record_transfer_stat( + ip = conn.ip, + pubkey = pubkey, + data_in = cmd.pub_data.len, + ) + let opubkey = relay.getNoteSub(cmd.pub_topic) + if opubkey.isSome: + # someone is waiting + var other_conn = relay.clients[opubkey.get()] conn.sendOkay cmd.kind - except: - conn.sendError("Duplicate topic", cmd.kind, Generic) + other_conn.sendMessage(RelayMessage( + kind: Note, + note_data: cmd.pub_data, + note_topic: cmd.pub_topic, + )) + relay.delNoteSub(cmd.pub_topic) + relay.db.record_transfer_stat( + ip = other_conn.ip, + pubkey = other_conn.pubkey.get(), + data_out = cmd.pub_data.len, + ) + else: + # no one is waiting + try: + relay.db.exec(sql"INSERT INTO note (topic, data, src) VALUES (?, ?, ?)", + cmd.pub_topic.DbBlob, + cmd.pub_data.DbBlob, + pubkey, + ) + conn.sendOkay cmd.kind + except: + conn.sendError("Duplicate topic", cmd.kind, Generic) of FetchNote: if cmd.fetch_topic.len > RELAY_MAX_TOPIC_SIZE: conn.sendError("Topic too long", cmd.kind, TooLarge) @@ -443,30 +557,50 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay let odata = relay.popNote(cmd.fetch_topic) if odata.isSome(): # the note is already here + let data = odata.get() conn.sendMessage(RelayMessage( kind: Note, - note_data: odata.get(), + note_data: data, note_topic: cmd.fetch_topic, )) + relay.db.record_transfer_stat( + ip = conn.ip, + pubkey = conn.pubkey.get(), + data_out = data.len, + ) else: # the note isn't here yet - relay.addNoteSub(cmd.fetch_topic, conn.pubkey) + relay.addNoteSub(cmd.fetch_topic, conn.pubkey.get()) of SendData: if cmd.send_val.len > RELAY_MAX_MESSAGE_SIZE: conn.sendError("Data too long", cmd.kind, TooLarge) else: - if relay.clients.hasKey(cmd.send_dst): - # dst is online - var other_conn = relay.clients[cmd.send_dst] - other_conn.sendMessage(RelayMessage( - kind: Data, - data_src: conn.pubkey, - data_val: cmd.send_val, - )) + let pubkey = conn.pubkey.get() + if relay.max_transfer_rate != 0 and relay.db.current_data_in(pubkey) > relay.max_transfer_rate: + conn.sendError("Rate limit exceeded", cmd.kind, TransferLimitExceeeded) else: - # dst is offline - relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?, ?, ?)", - conn.pubkey, cmd.send_dst, cmd.send_val.DbBlob) + relay.db.record_transfer_stat( + ip = conn.ip, + pubkey = pubkey, + data_in = cmd.send_val.len, + ) + if relay.clients.hasKey(cmd.send_dst): + # dst is online + var other_conn = relay.clients[cmd.send_dst] + other_conn.sendMessage(RelayMessage( + kind: Data, + data_src: pubkey, + data_val: cmd.send_val, + )) + relay.db.record_transfer_stat( + ip = other_conn.ip, + pubkey = other_conn.pubkey.get(), + data_in = cmd.send_val.len, + ) + else: + # dst is offline + relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?, ?, ?)", + pubkey, cmd.send_dst, cmd.send_val.DbBlob) of StoreChunk: if cmd.chunk_key.len > RELAY_MAX_CHUNK_KEY_SIZE: conn.sendError("Key too long", cmd.kind, TooLarge) @@ -475,33 +609,38 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay elif cmd.chunk_dst.len > RELAY_MAX_CHUNK_DSTS: conn.sendError("Too many recipients", cmd.kind, TooLarge) else: - relay.db.exec(sql"BEGIN") - try: - relay.db.exec(sql"DELETE FROM chunk_dst WHERE src=? AND key=?", conn.pubkey, cmd.chunk_key.DbBlob) - let offset = when TESTMODE: - $TIME_SKEW & " seconds" - else: - "0 seconds" - relay.db.exec(sql""" - INSERT OR REPLACE INTO chunk (last_used, src, key, val) - VALUES (datetime('now', ?), ?, ?, ?) - """, offset, conn.pubkey, cmd.chunk_key.DbBlob, cmd.chunk_val.DbBlob) - var dsts: seq[PublicKey] - dsts.add(cmd.chunk_dst) - if conn.pubkey notin dsts: - dsts.add(conn.pubkey) - for dst in dsts: - relay.db.exec(sql"INSERT INTO chunk_dst (src, key, dst) VALUES (?, ?, ?)", - conn.pubkey, cmd.chunk_key.DbBlob, dst) - relay.db.exec(sql"COMMIT") - except CatchableError: - relay.db.exec(sql"ROLLBACK") + let pubkey = conn.pubkey.get() + if relay.max_chunk_space > 0 and relay.db.chunk_space_used(pubkey) > relay.max_chunk_space: + conn.sendError("Too much chunk data", cmd.kind, StorageLimitExceeded) + else: + relay.db.exec(sql"BEGIN") + try: + relay.db.exec(sql"DELETE FROM chunk_dst WHERE src=? AND key=?", pubkey, cmd.chunk_key.DbBlob) + let offset = when TESTMODE: + $TIME_SKEW & " seconds" + else: + "0 seconds" + relay.db.exec(sql""" + INSERT OR REPLACE INTO chunk (last_used, src, key, val) + VALUES (datetime('now', ?), ?, ?, ?) + """, offset, pubkey, cmd.chunk_key.DbBlob, cmd.chunk_val.DbBlob) + var dsts: seq[PublicKey] + dsts.add(cmd.chunk_dst) + if pubkey notin dsts: + dsts.add(pubkey) + for dst in dsts: + relay.db.exec(sql"INSERT INTO chunk_dst (src, key, dst) VALUES (?, ?, ?)", + pubkey, cmd.chunk_key.DbBlob, dst) + relay.db.exec(sql"COMMIT") + except CatchableError: + relay.db.exec(sql"ROLLBACK") of GetChunks: for key in cmd.chunk_keys: if key.len > RELAY_MAX_CHUNK_KEY_SIZE: conn.sendError("Key too long", cmd.kind, TooLarge) return relay.delExpiredChunks() + let pubkey = conn.pubkey.get() for key in cmd.chunk_keys: let orow = relay.db.getRow(sql""" SELECT @@ -515,7 +654,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay d.src = ? AND d.key = ? AND d.dst = ? - """, cmd.chunk_src, key.DbBlob, conn.pubkey) + """, cmd.chunk_src, key.DbBlob, pubkey) if orow.isSome: let row = orow.get() conn.sendMessage(RelayMessage( @@ -531,3 +670,46 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay chunk_key: key, chunk_val: none[string](), )) + of ChunksPresent: + for key in cmd.present_keys: + if key.len > RELAY_MAX_CHUNK_KEY_SIZE: + conn.sendError("Key too long", cmd.kind, TooLarge) + return + relay.delExpiredChunks() + let pubkey = conn.pubkey.get() + var present: seq[string] + var absent: seq[string] + for key in cmd.present_keys: + let orow = relay.db.getRow(sql""" + SELECT + 1 + FROM + chunk_dst AS d + JOIN chunk AS c + ON d.src = c.src + AND d.key = c.key + WHERE + d.src = ? + AND d.key = ? + AND d.dst = ? + """, cmd.present_src, key.DbBlob, pubkey) + if orow.isSome: + present.add(key) + if cmd.present_src == pubkey: + # reset the expiration of the chunk, since the owner + # is touching it + let offset = when TESTMODE: + $TIME_SKEW & " seconds" + else: + "0 seconds" + relay.db.exec(sql""" + UPDATE chunk SET last_used = datetime('now', ?) WHERE src = ? AND key = ? + """, offset, cmd.present_src, key.DbBlob) + else: + absent.add(key) + conn.sendMessage(RelayMessage( + kind: ChunkStatus, + status_src: cmd.present_src, + present: present, + absent: absent, + )) diff --git a/src/sampleclient.nim b/src/sampleclient.nim index feb5914..268d69d 100644 --- a/src/sampleclient.nim +++ b/src/sampleclient.nim @@ -1,20 +1,51 @@ import std/asyncdispatch +import std/logging import std/options import ws import ./objs import ./proto2 -import ./server2 -export KeyPair, genkeys, NetstringSocket +type + NetstringClient* = ref object + buf: string + socket: WebSocket -proc newWS*(url: string): NetstringSocket = +proc newNetstringClient*(sock: WebSocket): NetstringClient = + new(result) + result.socket = sock + +proc receiveString*(ns: NetstringClient): Future[string] {.async.} = + while true: + try: + return ns.buf.nschop() + except IncompleteNetstring: + discard + let packet = await ns.socket.receiveStrPacket() + ns.buf &= packet + +proc sendString*(ns: NetstringClient, msg: string): Future[void] {.async.} = + await ns.socket.send(nsencode(msg)) + +proc sendCommand*(ns: NetstringClient, cmd: RelayCommand): Future[void] {.async.} = + when LOG_COMMS: + info "[client] -> " & $cmd + await ns.sendString(cmd.serialize()) + +proc receiveMessage*(ns: NetstringClient): Future[RelayMessage] {.async.} = + let s = await ns.receiveString() + let res = RelayMessage.deserialize(s) + when LOG_COMMS: + info "[client] <- " & $res + return res + +proc newNetstringClient*(url: string): NetstringClient = let ws = waitFor newWebSocket(url) - return newNetstringSocket(ws) + return newNetstringClient(ws) -proc newRelayClient*(url: string, keys: KeyPair): NetstringSocket = - var ns = newWS(url) +proc newRelayClient*(url: string, keys: KeyPair): NetstringClient = + var ns = newNetstringClient(url) let who = waitFor ns.receiveMessage() let answer = who.who_challenge.answer(keys.sk) waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: keys.pk)) @@ -22,7 +53,7 @@ proc newRelayClient*(url: string, keys: KeyPair): NetstringSocket = doAssert ok.kind == Okay, $ok return ns -proc publishNote*(ns: NetstringSocket, topic: string, data: string) {.async.} = +proc publishNote*(ns: NetstringClient, topic: string, data: string) {.async.} = await ns.sendCommand(RelayCommand( kind: PublishNote, pub_topic: topic, @@ -34,7 +65,7 @@ proc publishNote*(ns: NetstringSocket, topic: string, data: string) {.async.} = elif res.kind == Error: raise ValueError.newException("Error publishing note: " & $res.err_code & " " & res.err_message) -proc fetchNote*(ns: NetstringSocket, topic: string): Future[string] {.async.} = +proc fetchNote*(ns: NetstringClient, topic: string): Future[string] {.async.} = await ns.sendCommand(RelayCommand( kind: FetchNote, fetch_topic: topic, @@ -45,21 +76,21 @@ proc fetchNote*(ns: NetstringSocket, topic: string): Future[string] {.async.} = else: raise ValueError.newException("No such note: " & topic) -proc sendData*(ns: NetstringSocket, dst: PublicKey, val: string) {.async.} = +proc sendData*(ns: NetstringClient, dst: PublicKey, val: string) {.async.} = await ns.sendCommand(RelayCommand( kind: SendData, send_dst: dst, send_val: val, )) -proc getData*(ns: NetstringSocket): Future[string] {.async.} = +proc getData*(ns: NetstringClient): Future[string] {.async.} = let res = await ns.receiveMessage() if res.kind == Data: return res.data_val else: raise ValueError.newException("Expecting Data but got: " & $res) -proc storeChunk*(ns: NetstringSocket, dsts: seq[PublicKey], key: string, val: string) {.async.} = +proc storeChunk*(ns: NetstringClient, dsts: seq[PublicKey], key: string, val: string) {.async.} = await ns.sendCommand(RelayCommand( kind: StoreChunk, chunk_dst: dsts, @@ -67,7 +98,7 @@ proc storeChunk*(ns: NetstringSocket, dsts: seq[PublicKey], key: string, val: st chunk_val: val, )) -proc getChunk*(ns: NetstringSocket, src: PublicKey, key: string): Future[Option[string]] {.async.} = +proc getChunk*(ns: NetstringClient, src: PublicKey, key: string): Future[Option[string]] {.async.} = await ns.sendCommand(RelayCommand( kind: GetChunks, chunk_src: src, diff --git a/src/server2.nim b/src/server2.nim index 6cbe290..692c420 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -1,34 +1,59 @@ import std/asyncdispatch -import std/asynchttpserver import std/logging import std/strformat import std/strutils import std/deques +import jester import nimja import ws +import ws/jester_extra import lowdb/sqlite import ./proto2 import ./objs type - NetstringSocket* = ref object + NetstringSocket = ref object buf: string socket: WebSocket + ip: string + pubkey: Option[PublicKey] - QueuedMessage* = tuple + QueuedMessage = tuple socket: NetstringSocket msg: RelayMessage +const + VERSION = slurp"../CHANGELOG.md".split(" ")[1] + logo_png = slurp"static/logo.png" + favicon_png = slurp"static/favicon.png" + var relay: Relay[NetstringSocket] var message_queue = initDeque[QueuedMessage]() -proc newNetstringSocket*(sock: WebSocket): NetstringSocket = +proc trueClientIP(request: Request): string = + ## Return the true, originating client IP of a request + # CF-Connecting-IP (cloudflare) + result = request.headers.getOrDefault("cf-connecting-ip") + if result != "": + return result + # True-Client-IP (cloudflare) + result = request.headers.getOrDefault("true-client-ip") + if result != "": + return result + # X-Real-IP (nginx) + result = request.headers.getOrDefault("x-real-ip") + if result != "": + return result + result = request.ip + +proc newNetstringSocket(sock: WebSocket, ip: string): NetstringSocket = new(result) result.socket = sock + result.ip = ip -proc receiveString*(ns: NetstringSocket): Future[string] {.async.} = +proc receiveString(ns: NetstringSocket): Future[string] {.async.} = while true: try: return ns.buf.nschop() @@ -37,44 +62,34 @@ proc receiveString*(ns: NetstringSocket): Future[string] {.async.} = let packet = await ns.socket.receiveStrPacket() ns.buf &= packet -proc sendString*(ns: NetstringSocket, msg: string): Future[void] {.async.} = - await ns.socket.send(nsencode(msg)) +proc sendString(ns: NetstringSocket, msg: string): Future[void] {.async.} = + let tosend = nsencode(msg) + await ns.socket.send(tosend) -proc sendCommand*(ns: NetstringSocket, cmd: RelayCommand): Future[void] {.async.} = - when LOG_COMMS: - info "[client] -> " & $cmd - await ns.sendString(cmd.serialize()) - -proc receiveCommand*(ns: NetstringSocket): Future[RelayCommand] {.async.} = +proc receiveCommand(relay: Relay, ns: NetstringSocket): Future[RelayCommand] {.async.} = let s = await ns.receiveString() return RelayCommand.deserialize(s) -proc receiveMessage*(ns: NetstringSocket): Future[RelayMessage] {.async.} = - let s = await ns.receiveString() - let res = RelayMessage.deserialize(s) - when LOG_COMMS: - info "[client] <- " & $res - return res - -proc sendMessage*(ns: NetstringSocket, msg: RelayMessage) {.async.} = +proc sendMessage(ns: NetstringSocket, msg: RelayMessage) {.async.} = await ns.sendString(msg.serialize()) -proc sendMessage*(conn: RelayConnection[NetstringSocket], msg: RelayMessage) = +proc sendMessage(conn: RelayConnection[NetstringSocket], msg: RelayMessage) = message_queue.addLast((conn.sender, msg)) -proc sendQueuedMessages*() {.async.} = +proc sendQueuedMessages() {.async.} = while message_queue.len > 0: let (sock, msg) = message_queue.popFirst() await sock.sendMessage(msg) proc handleWebsocket(req: Request) {.async, gcsafe.} = var ws = await newWebSocket(req) - var ns = newNetstringSocket(ws) + var ns = newNetstringSocket(ws, req.trueClientIP()) var conn = relay.initAuth(ns) + conn.ip = req.trueClientIP() await sendQueuedMessages() while ns.socket.readyState == Open: let cmd = try: - await ns.receiveCommand() + await relay.receiveCommand(ns) except WebSocketClosedError: break except WebSocketProtocolMismatchError: @@ -87,15 +102,150 @@ proc handleWebsocket(req: Request) {.async, gcsafe.} = warn "CatchableError: ", getCurrentExceptionMsg() break relay.handleCommand(conn, cmd) + if cmd.kind == Iam: + ns.pubkey = conn.pubkey await sendQueuedMessages() relay.disconnect(conn) - await req.respond(Http200, "done") -proc cb(req: Request) {.async, gcsafe.} = - if req.url.path == "/ws": - await req.handleWebsocket() - else: - await req.respond(Http404, "Not found") +type + StorageStat = tuple + pubkey: PublicKey + message_size: int + chunk_size: int + total_size: int + +router myrouter: + get "/ws": + await request.handleWebsocket() + result[0] = TCActionRaw + + get "/": + var html = "" + compileTemplateFile("index.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") + resp html + + get "/static/logo.png": + resp logo_png + + get "/static/favicon.png": + resp favicon_png + + get "/stats": + when defined(release): + {.fatal: "Add protection to this endpoint".} + let days_back = "-28 days" + let datarange: PeriodRange = block: + let row = relay.db.getRow(sql"""SELECT + strftime('%Y-%W', datetime('now', ?)) AS a, + strftime('%Y-%W') AS b""", days_back).get() + (row[0].s, row[1].s) + + # total transfer + let row = relay.db.getRow(sql""" + SELECT + sum(data_in) AS din, + sum(data_out) AS dout + FROM + stats_transfer + WHERE + period >= ? + """, datarange.a).get() + let total_data_in = row[0].i.int + let total_data_out = row[1].i.int + + # total stored + let total_stored_note = relay.db.getRow(sql"SELECT coalesce(sum(length(data)), 0) FROM note").get()[0].i + let total_stored_message = relay.db.getRow(sql"SELECT coalesce(sum(length(data)), 0) FROM message").get()[0].i + let total_stored_chunk = relay.db.getRow(sql"SELECT coalesce(sum(length(val)), 0) FROM chunk").get()[0].i + let total_stored = total_stored_note + total_stored_message + total_stored_chunk + + let num_note = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM note").get()[0].i + let num_message = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM message").get()[0].i + let num_chunk = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM chunk").get()[0].i + + # top traffic by ip + var traffic_by_ip: seq[TransferTotal] + for row in relay.db.getAllRows(sql""" + SELECT + sum(data_in) AS din, + sum(data_out) AS dout, + sum(data_in) + sum(data_out) AS total, + ip + FROM + stats_transfer + WHERE + period >= ? + GROUP BY ip + ORDER BY total DESC + LIMIT 10 + """, datarange.a): + traffic_by_ip.add(( + data_in: row[0].i.int, + data_out: row[1].i.int, + ip: row[3].s, + pubkey: default(PublicKey), + period: "", + )) + + # top traffic by pubkey + var traffic_by_pubkey: seq[TransferTotal] + for row in relay.db.getAllRows(sql""" + SELECT + sum(data_in) AS din, + sum(data_out) AS dout, + sum(data_in) + sum(data_out) AS total, + pubkey + FROM + stats_transfer + WHERE + period >= ? + AND pubkey <> '' + GROUP BY pubkey + ORDER BY total DESC + LIMIT 10 + """, datarange.a): + traffic_by_pubkey.add(( + data_in: row[0].i.int, + data_out: row[1].i.int, + ip: "", + pubkey: PublicKey.fromDB(row[3].b), + period: "", + )) + + # top storage by pubkey + var storage_by_pubkey: seq[StorageStat] + for row in relay.db.getAllRows(sql""" + WITH msg AS ( + SELECT src, SUM(LENGTH(data)) AS msg_bytes + FROM message + GROUP BY src + ), + chunksize AS ( + SELECT src, SUM(LENGTH(val)) AS chunk_bytes + FROM chunk + GROUP BY src + ) + SELECT + COALESCE(m.src, c.src) AS src, + COALESCE(m.msg_bytes, 0) AS msg_bytes, + COALESCE(c.chunk_bytes, 0) AS chunk_bytes, + COALESCE(m.msg_bytes, 0) + COALESCE(c.chunk_bytes, 0) AS total_bytes + FROM msg AS m + FULL OUTER JOIN chunksize AS c + ON m.src = c.src + ORDER BY total_bytes DESC + LIMIT 10; + """): + storage_by_pubkey.add(( + pubkey: PublicKey.fromDb(row[0].b), + message_size: row[1].i.int, + chunk_size: row[2].i.int, + total_size: row[3].i.int, + )) + + var html = "" + compileTemplateFile("stats.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") + resp html proc main(database: string, port: Port, address = "127.0.0.1") = var L = newConsoleLogger() @@ -103,9 +253,10 @@ proc main(database: string, port: Port, address = "127.0.0.1") = info "Database: ", database var db = open(database, "", "", "") relay = newRelay[NetstringSocket](db) - var server = newAsyncHttpServer() info &"Serving on {address}:{port.int}" - waitFor server.serve(port, cb, address = address) + let settings = newSettings(port=port, bindAddr=address) + var jester = initJester(myrouter, settings=settings) + jester.serve() when isMainModule: import argparse diff --git a/src/templates/index.nimja b/src/templates/index.nimja index e69de29..56c949c 100644 --- a/src/templates/index.nimja +++ b/src/templates/index.nimja @@ -0,0 +1,60 @@ + + + Buckets Relay + + + + + +
+
+ +
+ +

+ Buckets Relay +

+
+ {{ VERSION }} +
+ +

+ If you use Buckets, this relay lets you securely share your budget among your devices. Data is stored on this relay for a time, but is removed after periods of inactivity. All data passing through this relay is encrypted end-to-end. +

+ +

+ Use of this service may be revoked at any time for any reason. Also, for now it's free to use, but that could change depending on the cost to run it. +

+ +

+ The code for this is open source if you'd like to run your own instance. +

+
+ + \ No newline at end of file diff --git a/src/templates/stats.nimja b/src/templates/stats.nimja new file mode 100644 index 0000000..bd90e98 --- /dev/null +++ b/src/templates/stats.nimja @@ -0,0 +1,119 @@ + + + Buckets Relay Stats + + + + + +

Stats [{{ datarange.a }}, {{ datarange.b }}]

+ +

Transfer

+ + + + + + + + + + + +
Data inData outTotal
{{ total_data_in }}{{ total_data_out }}{{ total_data_in + total_data_out }}
+ +

Storage total

+ + + + + + + + + + + + + + + + + + + + + + +
NotesMessagesChunksTotal
Size{{ total_stored_note }}{{ total_stored_message }}{{ total_stored_chunk }}{{ total_stored }}
Count{{ num_note }}{{ num_message }}{{ num_chunk }}
+ +

Top Traffic by IP

+ + + + + + + + {% for tot in traffic_by_ip %} + + + + + + + {% endfor %} +
IPData inData outTotal
{{ tot.ip }}{{ tot.data_in }}{{ tot.data_out }}{{ tot.data_in + tot.data_out }}
+ +

Top Traffic by Pubkey

+ + + + + + + + {% for tot in traffic_by_pubkey %} + + + + + + + {% endfor %} +
PubkeyData inData outTotal
{{ tot.pubkey.abbr }}{{ tot.data_in }}{{ tot.data_out }}{{ tot.data_in + tot.data_out }}
+ +

Top storage by Pubkey

+ + + + + + + {% for tot in storage_by_pubkey %} + + + + + + {% endfor %} +
PubkeyMessage bytesChunk bytes
{{ tot.pubkey.abbr }}{{ tot.message_size }}{{ tot.chunk_size }}
+ + \ No newline at end of file diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index 3699d1a..997c359 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -3,12 +3,12 @@ import std/net import std/options import std/os import std/osproc +import std/streams import std/unittest import ./util import sampleclient -import server2 import proto2 import ws @@ -24,13 +24,17 @@ proc startServer(port: Port): Process = let bin = absolutePath(currentSourcePath().parentDir() / "bin" / "server2") bin.parentDir.createDir() echo "compiling ", bin.relativePath(".") - echo execProcess("nim", + let compileProcess = startProcess( + "nim", workingDir = currentSourcePath().parentDir().parentDir(), - args = [ - "c", "-d:testmode", "-o:" & bin, "src"/"server2.nim", - ], + args = ["c", "-d:testmode", "-o:" & bin, "src/server2.nim"], options = {poStdErrToStdOut, poUsePath} ) + let output = compileProcess.outputStream.readAll() + let exitCode = compileProcess.waitForExit() + echo output + if exitCode != 0: + raise newException(OSError, "Compilation failed with exit code " & $exitCode) echo "compiled ", bin.relativePath(".") startProcess(bin, @@ -66,11 +70,11 @@ waitForPort(TESTPORT) proc serverURL(): string = "ws://127.0.0.1:" & $TESTPORT & "/ws" -proc testClient(keys: KeyPair): NetstringSocket = +proc testClient(keys: KeyPair): NetstringClient = let url = serverURL() newRelayClient(url, keys) -proc testClient(): NetstringSocket = +proc testClient(): NetstringClient = testClient(genkeys()) @@ -156,7 +160,7 @@ suite "invalid": test "RelayMessage": var keys = genkeys() let ws = waitFor newWebSocket(serverURL()) - let ns = newNetstringSocket(ws) + let ns = newNetstringClient(ws) let who = waitFor ns.receiveMessage() let answer = who.who_challenge.answer(keys.sk) waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: keys.pk)) diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 0ddad50..decc82c 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -95,7 +95,7 @@ proc authenticatedConn(relay: Relay): RelayConnection[TestClient] = # End of TestClient #--------------------------------- -suite "Auth": +suite "auth": test "basic": let relay = testRelay() let aclient = newTestClient(genkeys()) @@ -186,7 +186,7 @@ suite "Auth": let err = alice.pop(Error) check err.err_cmd == Iam -suite "PublishNote": +suite "notes": test "basic": let relay = testRelay() @@ -380,6 +380,26 @@ suite "PublishNote": let data = bob.pop(Note) check data.note_data == "c\x00d" check data.note_topic == "a\x00b" + + test "max notes": + var relay = testRelay() + var alice = relay.authenticatedConn() + for i in 0.. Date: Tue, 4 Nov 2025 12:06:03 -0500 Subject: [PATCH 27/46] Add password auth to stats page --- src/objs.nim | 4 +++- src/server2.nim | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/objs.nim b/src/objs.nim index 84773f4..b58e9d2 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -5,7 +5,9 @@ ## These are the objects used for the protocol. ## This file should be kept free of dependencies other than the stdlib -## as it's meant to be referenced by outside libraries. +## and should not include async stuff +## as it's meant to be referenced by outside libraries that may +## want to do things there own way. import std/hashes import std/options diff --git a/src/server2.nim b/src/server2.nim index 692c420..c08eafc 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -3,12 +3,15 @@ import std/logging import std/strformat import std/strutils import std/deques +import std/base64 +import std/httpcore import jester import nimja import ws import ws/jester_extra import lowdb/sqlite +import libsodium/sodium import ./proto2 import ./objs @@ -28,6 +31,11 @@ const VERSION = slurp"../CHANGELOG.md".split(" ")[1] logo_png = slurp"static/logo.png" favicon_png = slurp"static/favicon.png" + ADMIN_USERNAME {.strdefine.} = "admin" + ADMIN_PASSWORD {.strdefine.} = when not defined(release): + "admin" + else: + staticExec("uuidgen") var relay: Relay[NetstringSocket] var message_queue = initDeque[QueuedMessage]() @@ -48,6 +56,24 @@ proc trueClientIP(request: Request): string = return result result = request.ip +proc isAdmin(request: Request): bool = + if not request.headers.hasKey("Authorization"): + return false + let authHeader = request.headers["Authorization"] + let encodedCreds = authHeader[("Basic ".len)..^1] + + try: + let decodedCreds = base64.decode(encodedCreds) + let parts = decodedCreds.split(":", 1) + if parts.len == 2: + let username = parts[0] + let password = parts[1] + return sodium.memcmp(username, ADMIN_USERNAME) and sodium.memcmp(password, ADMIN_PASSWORD) + except CatchableError: + return false + + return false + proc newNetstringSocket(sock: WebSocket, ip: string): NetstringSocket = new(result) result.socket = sock @@ -131,8 +157,10 @@ router myrouter: resp favicon_png get "/stats": - when defined(release): - {.fatal: "Add protection to this endpoint".} + if not request.isAdmin(): + responseHeaders.setHeader("WWW-Authenticate", "Basic realm=\"Relay Admin\"") + resp Http401, "Unauthorized" + let days_back = "-28 days" let datarange: PeriodRange = block: let row = relay.db.getRow(sql"""SELECT From b883746142e4bd8bf31b190ec9362f2711c2373f Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 4 Nov 2025 12:08:24 -0500 Subject: [PATCH 28/46] Fix docker build --- docker/Dockerfile | 2 +- docker/config.nims | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5b92bab..5d603ae 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # -- Stage 1 -- # -FROM nimlang/nim:2.2.4-alpine-regular as builder +FROM nimlang/nim:2.2.6-alpine-regular as builder WORKDIR /app RUN apk update && apk add libsodium-static libsodium musl-dev RUN nimble refresh diff --git a/docker/config.nims b/docker/config.nims index f8dd00a..11792ee 100644 --- a/docker/config.nims +++ b/docker/config.nims @@ -3,3 +3,4 @@ switch("dynlibOverride", "libsodium") switch("cincludes", "/usr/include") switch("clibdir", "/usr/lib") switch("passL", "-lsodium") +switch("d", "useStdLib") # rather than httpBeast From fba5f2354d68c1080551809a5e5a5aacaf92bb1d Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 4 Nov 2025 15:57:32 -0500 Subject: [PATCH 29/46] Test same key --- tests/tfunctional.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index 997c359..195f81f 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -77,6 +77,10 @@ proc testClient(keys: KeyPair): NetstringClient = proc testClient(): NetstringClient = testClient(genkeys()) +suite "auth": + + test "same key auth": + check false suite "publishnote": From 45507e8a9d12a25cbb9c4990f45b08a46bae95dd Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 4 Nov 2025 16:01:40 -0500 Subject: [PATCH 30/46] cli --- src/cli.nim | 195 +++++++++++++++++++++++++++++++++++++++++++ src/objs.nim | 36 ++++---- src/proto2.nim | 14 ++-- src/sampleclient.nim | 14 +++- tests/tproto2.nim | 6 +- tests/tserde2.nim | 8 +- 6 files changed, 240 insertions(+), 33 deletions(-) create mode 100644 src/cli.nim diff --git a/src/cli.nim b/src/cli.nim new file mode 100644 index 0000000..aa5053a --- /dev/null +++ b/src/cli.nim @@ -0,0 +1,195 @@ +import std/asyncdispatch +import std/base64 +import std/json +import std/options +import std/os +import std/rdstdin +import std/strformat +import std/strutils + +import ./sampleclient +import ./proto2 + +type + CmdContext = object + dst: PublicKey + +proc serialize(pubkey: PublicKey): string = + base64.encode(pubkey.string) + +proc deserialize(pubkey: typedesc[PublicKey], x: string): PublicKey = + base64.decode(x).PublicKey + +proc serialize(keys: KeyPair): string = + base64.encode($(%* { + "pk": keys.pk.string, + "sk": keys.sk.string, + })) + +proc deserialize(pair: typedesc[KeyPair], x: string): KeyPair = + let j = base64.decode(x).parseJson() + (j["pk"].getStr().PublicKey, j["sk"].getStr().SecretKey) + +proc loadKeys(src: string): KeyPair = + let parts = src.split(":", 1) + case parts[0] + of "keys": + KeyPair.deserialize(parts[1]) + of "file": + KeyPair.deserialize(readFile(parts[1])) + else: + raise ValueError.newException("Error loading keys") + +proc saveKeys(keys: KeyPair, filename: string) = + writeFile(filename, keys.serialize()) + +proc parseLine(line: string): seq[string] = + result = @[] + var i = 0 + var current = "" + var inQuote = false + var quoteChar = '\0' + + while i < line.len: + let c = line[i] + + if inQuote: + if c == quoteChar: + inQuote = false + result.add(current) + current = "" + else: + current.add(c) + else: + if c in {'\'', '"'}: + inQuote = true + quoteChar = c + if current.len > 0: + result.add(current) + current = "" + elif c == ' ': + if current.len > 0: + result.add(current) + current = "" + elif c == '\\': + inc(i) + if i < line.len: + current.add(line[i]) + else: + current.add(c) + inc(i) + + if current.len > 0: + result.add(current) + + # Handle unclosed quote as literal + if inQuote and current.len > 0: + result.add(current) + +proc use(i: var int, s: seq[string]): string = + result = s[i] + i.inc() + +proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) = + let cmd = full[0] + let args = full[1..^1] + var i = 0 + case cmd + of "post": + let topic = i.use(args) + let data = i.use(args) + waitFor client.publishNote(topic, data) + echo "posted ", topic + of "fetch": + let topic = i.use(args) + let data = waitFor client.fetchNote(topic) + echo data + of "dst": + ctx.dst = PublicKey.deserialize(i.use(args)) + echo "dst for future commands set to ", ctx.dst.serialize() + of "send": + var dst = ctx.dst + if dst.string == "": + dst = PublicKey.deserialize(i.use(args)) + let val = i.use(args) + waitFor client.sendData(dst, val) + of "recv": + let data = waitFor client.getData() + echo data + of "store": + var dst = ctx.dst + if dst.string == "": + dst = PublicKey.deserialize(i.use(args)) + let key = i.use(args) + let val = i.use(args) + waitFor client.storeChunk(@[dst], key, val) + of "get": + var src = ctx.dst + if src.string == "": + src = PublicKey.deserialize(i.use(args)) + let key = i.use(args) + let odata = waitFor client.getChunk(src, key) + if odata.isSome: + echo odata.get() + else: + echo "(none)" + of "has": + var src = ctx.dst + if src.string == "": + src = PublicKey.deserialize(i.use(args)) + let key = i.use(args) + let res = waitFor client.hasChunk(src, key) + echo $res + else: + echo "Unknown command ", cmd, " ", args + +proc main(url: string, keys: KeyPair, dst = "".PublicKey) = + # authenticate + echo "...pubkey: ", keys.pk.serialize + echo "...connecting..." + var client = newRelayClient(url, keys) + var ctx: CmdContext + ctx.dst = dst + echo "...connected" + while true: + let line = readLineFromStdin(&"{ctx.dst.serialize}> ") + if line == nil or line.strip() in ["exit", "quit"]: + echo "...goodbye" + break + + if line.strip() == "": + continue + + let args = parseLine(line) + try: + client.doCommand(args, ctx) + except IndexDefect: + echo "ERROR: " & getCurrentExceptionMsg() + except CatchableError: + echo "ERROR: " & getCurrentExceptionMsg() + +when isMainModule: + import argparse + + var p = newParser: + option("-u", "--url", default = some("ws://127.0.0.1:9000/ws")) + option("-k", "--keys", default = some("file:client1.keys")) + command("genkeys"): + arg("filename") + run: + let keys = genkeys() + keys.saveKeys(opts.filename) + command("repl"): + option("-d", "--dst", help = "Default destination") + run: + var dst = if opts.dst != "": + PublicKey.deserialize(opts.dst) + else: + default(PublicKey) + main(opts.parentOpts.url, keys = loadKeys(opts.parentOpts.keys), dst = dst) + + try: + p.run(commandLineParams()) + except UsageError as e: + stderr.writeLine getCurrentExceptionMsg() + quit(1) \ No newline at end of file diff --git a/src/objs.nim b/src/objs.nim index b58e9d2..21dbce6 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -78,7 +78,7 @@ type SendData StoreChunk GetChunks - ChunksPresent + HasChunks RelayCommand* = object case kind*: CommandKind @@ -100,9 +100,9 @@ type of GetChunks: chunk_src*: PublicKey chunk_keys*: seq[string] - of ChunksPresent: - present_src*: PublicKey - present_keys*: seq[string] + of HasChunks: + has_src*: PublicKey + has_keys*: seq[string] const RELAY_MAX_TOPIC_SIZE* = 512 @@ -223,9 +223,9 @@ proc `$`*(cmd: RelayCommand): string = result.add &"{cmd.chunk_src.nice.abbr} keys=[" result.add cmd.chunk_keys.mapIt(it.nice.abbr).join(", ") result.add "]" - of ChunksPresent: - result.add &"{cmd.present_src.nice.abbr} keys=[" - result.add cmd.present_keys.mapIt(it.nice.abbr).join(", ") + of HasChunks: + result.add &"{cmd.has_src.nice.abbr} keys=[" + result.add cmd.has_keys.mapIt(it.nice.abbr).join(", ") result.add "]" result.add ")" @@ -246,8 +246,8 @@ proc `==`*(a, b: RelayCommand): bool = return a.chunk_dst == b.chunk_dst and a.chunk_key == b.chunk_key and a.chunk_val == b.chunk_val of GetChunks: return a.chunk_src == b.chunk_src and a.chunk_keys == b.chunk_keys - of ChunksPresent: - return a.present_src == b.present_src and a.present_keys == b.present_keys + of HasChunks: + return a.has_src == b.has_src and a.has_keys == b.has_keys #-------------------------------------------------------------- # serialization @@ -349,7 +349,7 @@ proc serialize*(kind: CommandKind): char = of SendData: 's' of StoreChunk: 'c' of GetChunks: 'g' - of ChunksPresent: 't' + of HasChunks: 't' proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = case val: @@ -359,7 +359,7 @@ proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = of 's': SendData of 'c': StoreChunk of 'g': GetChunks - of 't': ChunksPresent + of 't': HasChunks else: raise ValueError.newException("Unknown CommandKind: " & val) proc serialize*(err: ErrorCode): char = @@ -515,9 +515,9 @@ proc serialize*(cmd: RelayCommand): string = of GetChunks: result &= cmd.chunk_src.string.nsencode result &= nsencode(cmd.chunk_keys.serialize()) - of ChunksPresent: - result &= cmd.present_src.string.nsencode - result &= nsencode(cmd.present_keys.serialize()) + of HasChunks: + result &= cmd.has_src.string.nsencode + result &= nsencode(cmd.has_keys.serialize()) proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = if s.len == 0: @@ -566,10 +566,10 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = chunk_src: s.nsdecode(idx).PublicKey, chunk_keys: deserialize(seq[string], s.nsdecode(idx)), ) - of ChunksPresent: + of HasChunks: var idx = 1 return RelayCommand( - kind: ChunksPresent, - present_src: s.nsdecode(idx).PublicKey, - present_keys: deserialize(seq[string], s.nsdecode(idx)), + kind: HasChunks, + has_src: s.nsdecode(idx).PublicKey, + has_keys: deserialize(seq[string], s.nsdecode(idx)), ) diff --git a/src/proto2.nim b/src/proto2.nim index f950f12..04d6b9d 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -670,8 +670,8 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay chunk_key: key, chunk_val: none[string](), )) - of ChunksPresent: - for key in cmd.present_keys: + of HasChunks: + for key in cmd.has_keys: if key.len > RELAY_MAX_CHUNK_KEY_SIZE: conn.sendError("Key too long", cmd.kind, TooLarge) return @@ -679,7 +679,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay let pubkey = conn.pubkey.get() var present: seq[string] var absent: seq[string] - for key in cmd.present_keys: + for key in cmd.has_keys: let orow = relay.db.getRow(sql""" SELECT 1 @@ -692,10 +692,10 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay d.src = ? AND d.key = ? AND d.dst = ? - """, cmd.present_src, key.DbBlob, pubkey) + """, cmd.has_src, key.DbBlob, pubkey) if orow.isSome: present.add(key) - if cmd.present_src == pubkey: + if cmd.has_src == pubkey: # reset the expiration of the chunk, since the owner # is touching it let offset = when TESTMODE: @@ -704,12 +704,12 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay "0 seconds" relay.db.exec(sql""" UPDATE chunk SET last_used = datetime('now', ?) WHERE src = ? AND key = ? - """, offset, cmd.present_src, key.DbBlob) + """, offset, cmd.has_src, key.DbBlob) else: absent.add(key) conn.sendMessage(RelayMessage( kind: ChunkStatus, - status_src: cmd.present_src, + status_src: cmd.has_src, present: present, absent: absent, )) diff --git a/src/sampleclient.nim b/src/sampleclient.nim index 268d69d..cb3da9e 100644 --- a/src/sampleclient.nim +++ b/src/sampleclient.nim @@ -108,4 +108,16 @@ proc getChunk*(ns: NetstringClient, src: PublicKey, key: string): Future[Option[ if res.kind == Chunk: return res.chunk_val else: - raise ValueError.newException("Expecing Chunk but got: " & $res) + raise ValueError.newException("Expecting Chunk but got: " & $res) + +proc hasChunk*(ns: NetstringClient, src: PublicKey, key: string): Future[bool] {.async.} = + await ns.sendCommand(RelayCommand( + kind: HasChunks, + has_src: src, + has_keys: @[key], + )) + let res = await ns.receiveMessage() + if res.kind == ChunkStatus: + return key in res.present + else: + raise ValueError.newException("Expecting ChunkStatus but got: " & $res) diff --git a/tests/tproto2.nim b/tests/tproto2.nim index decc82c..ae2cba6 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -502,9 +502,9 @@ proc getChunk(conn: var RelayConnection[TestClient], src: var RelayConnection[Te proc chunkExists(conn: var RelayConnection[TestClient], src: var RelayConnection[TestClient], key: string): bool = conn.relay.handleCommand(conn, RelayCommand( - kind: ChunksPresent, - present_src: src.pk, - present_keys: @[key], + kind: HasChunks, + has_src: src.pk, + has_keys: @[key], )) let resp = conn.pop(ChunkStatus) return key in resp.present diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 9009ad5..0b0e56b 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -51,10 +51,10 @@ test "RelayCommand": chunk_val: "someval" ) of GetChunks: RelayCommand(kind: GetChunks, chunk_src: "hey".PublicKey, chunk_keys: @["foo", "bar"]) - of ChunksPresent: RelayCommand( - kind: ChunksPresent, - present_src: "hey".PublicKey, - present_keys: @["foo", "Bar"], + of HasChunks: RelayCommand( + kind: HasChunks, + has_src: "hey".PublicKey, + has_keys: @["foo", "Bar"], ) let serialized = example.serialize() info $example From 6811102c2e6b4822789297da56e739451b16f434 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 21 Nov 2025 09:04:44 -0500 Subject: [PATCH 31/46] ADMIN_USERNAME and ADMIN_PWHASH fully supported now --- src/cli.nim | 27 +++++++++++++++++++++++---- src/server2.nim | 26 +++++++++++++++++--------- tests/tfunctional.nim | 7 ++++++- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/cli.nim b/src/cli.nim index aa5053a..d8e6e8d 100644 --- a/src/cli.nim +++ b/src/cli.nim @@ -105,7 +105,10 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) let data = waitFor client.fetchNote(topic) echo data of "dst": - ctx.dst = PublicKey.deserialize(i.use(args)) + if args.len == 0: + ctx.dst = "".PublicKey + else: + ctx.dst = PublicKey.deserialize(i.use(args)) echo "dst for future commands set to ", ctx.dst.serialize() of "send": var dst = ctx.dst @@ -118,15 +121,17 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) echo data of "store": var dst = ctx.dst - if dst.string == "": + if dst.string == "" or args.len >= 3: dst = PublicKey.deserialize(i.use(args)) + echo "Using key=" & dst.nice let key = i.use(args) let val = i.use(args) waitFor client.storeChunk(@[dst], key, val) of "get": var src = ctx.dst - if src.string == "": + if src.string == "" or args.len >= 2: src = PublicKey.deserialize(i.use(args)) + echo "Using key=" & src.serialize let key = i.use(args) let odata = waitFor client.getChunk(src, key) if odata.isSome: @@ -135,11 +140,25 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) echo "(none)" of "has": var src = ctx.dst - if src.string == "": + if src.string == "" or args.len >= 2: src = PublicKey.deserialize(i.use(args)) + echo "Using key=" & src.serialize let key = i.use(args) let res = waitFor client.hasChunk(src, key) echo $res + of "help": + echo """ + post TOPIC DATA + fetch TOPIC + dst PUBKEY + Set the destination PUBKEY for future commands + send [PUBKEY] DATA + recv + store [PUBKEY] KEY VAL + get [PUBKEY] KEY + has [PUBKEY] KEY + help + """ else: echo "Unknown command ", cmd, " ", args diff --git a/src/server2.nim b/src/server2.nim index c08eafc..8d9a1df 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -1,10 +1,11 @@ import std/asyncdispatch +import std/base64 +import std/deques +import std/httpcore import std/logging +import std/os import std/strformat import std/strutils -import std/deques -import std/base64 -import std/httpcore import jester import nimja @@ -31,11 +32,13 @@ const VERSION = slurp"../CHANGELOG.md".split(" ")[1] logo_png = slurp"static/logo.png" favicon_png = slurp"static/favicon.png" - ADMIN_USERNAME {.strdefine.} = "admin" - ADMIN_PASSWORD {.strdefine.} = when not defined(release): - "admin" - else: - staticExec("uuidgen") + +let ADMIN_USERNAME = getEnv("ADMIN_USERNAME", "admin") +let ADMIN_PWHASH = when defined(release): + getEnv("ADMIN_PWHASH", "") + else: + # the password is 'admin' + getEnv("ADMIN_PWHASH", "$argon2id$v=19$m=262144,t=3,p=1$AxXWW9mRuyJjWWbxa4WYoQ$xHAyhzWgKGFH+amM4D1GMuNsPSjGNNp40MueB9dJkgA") var relay: Relay[NetstringSocket] var message_queue = initDeque[QueuedMessage]() @@ -68,7 +71,7 @@ proc isAdmin(request: Request): bool = if parts.len == 2: let username = parts[0] let password = parts[1] - return sodium.memcmp(username, ADMIN_USERNAME) and sodium.memcmp(password, ADMIN_PASSWORD) + return sodium.memcmp(username, ADMIN_USERNAME) and crypto_pwhash_str_verify(ADMIN_PWHASH, password) except CatchableError: return false @@ -290,6 +293,11 @@ when isMainModule: import argparse var p = newParser: option("-d", "--database", default=some("brelay.sqlite"), help="Database") + command("hashpassword"): + help("Generate a hash for a password given on stdin") + run: + let password = stdin.readAll().strip() + echo crypto_pwhash_str(password) command("server"): option("-p", "--port", default=some("9000")) option("-a", "--address", default=some("127.0.0.1")) diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index 195f81f..7c6230a 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -80,7 +80,12 @@ proc testClient(): NetstringClient = suite "auth": test "same key auth": - check false + var keys = genkeys() + var alice = testClient(keys) + var alice2 = testClient(keys) + var bob = testClient() + waitFor bob.sendData(keys.pk, "this is bob") + check (waitFor alice2.getData()) == "this is bob" suite "publishnote": From a165d02c7a99f8099d0ff1f5b2e1b25fad688a53 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 21 Nov 2025 09:10:23 -0500 Subject: [PATCH 32/46] Format stats numbers with commas --- src/server2.nim | 2 ++ src/templates/stats.nimja | 36 ++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/server2.nim b/src/server2.nim index 8d9a1df..83f6223 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -77,6 +77,8 @@ proc isAdmin(request: Request): bool = return false +proc wcommas(x: int): string = insertSep($x, sep = ',') + proc newNetstringSocket(sock: WebSocket, ip: string): NetstringSocket = new(result) result.socket = sock diff --git a/src/templates/stats.nimja b/src/templates/stats.nimja index bd90e98..04aa5c5 100644 --- a/src/templates/stats.nimja +++ b/src/templates/stats.nimja @@ -33,9 +33,9 @@ Total - {{ total_data_in }} - {{ total_data_out }} - {{ total_data_in + total_data_out }} + {{ total_data_in.wcommas }} + {{ total_data_out.wcommas }} + {{ (total_data_in + total_data_out).wcommas }} @@ -50,16 +50,16 @@ Size - {{ total_stored_note }} - {{ total_stored_message }} - {{ total_stored_chunk }} - {{ total_stored }} + {{ total_stored_note.wcommas }} + {{ total_stored_message.wcommas }} + {{ total_stored_chunk.wcommas }} + {{ total_stored.wcommas }} Count - {{ num_note }} - {{ num_message }} - {{ num_chunk }} + {{ num_note.wcommas }} + {{ num_message.wcommas }} + {{ num_chunk.wcommas }} @@ -75,9 +75,9 @@ {% for tot in traffic_by_ip %} {{ tot.ip }} - {{ tot.data_in }} - {{ tot.data_out }} - {{ tot.data_in + tot.data_out }} + {{ tot.data_in.wcommas }} + {{ tot.data_out.wcommas }} + {{ (tot.data_in + tot.data_out).wcommas }} {% endfor %} @@ -93,9 +93,9 @@ {% for tot in traffic_by_pubkey %} {{ tot.pubkey.abbr }} - {{ tot.data_in }} - {{ tot.data_out }} - {{ tot.data_in + tot.data_out }} + {{ tot.data_in.wcommas }} + {{ tot.data_out.wcommas }} + {{ (tot.data_in + tot.data_out).wcommas }} {% endfor %} @@ -110,8 +110,8 @@ {% for tot in storage_by_pubkey %} {{ tot.pubkey.abbr }} - {{ tot.message_size }} - {{ tot.chunk_size }} + {{ tot.message_size.wcommas }} + {{ tot.chunk_size.wcommas }} {% endfor %} From 7272d367954226ba1fb24d4f3a88900811f4f252 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 21 Nov 2025 09:21:30 -0500 Subject: [PATCH 33/46] Fix error on empty stats --- src/server2.nim | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/server2.nim b/src/server2.nim index 83f6223..bcc5451 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -176,8 +176,8 @@ router myrouter: # total transfer let row = relay.db.getRow(sql""" SELECT - sum(data_in) AS din, - sum(data_out) AS dout + coalesce(sum(data_in), 0) AS din, + coalesce(sum(data_out), 0) AS dout FROM stats_transfer WHERE @@ -191,7 +191,6 @@ router myrouter: let total_stored_message = relay.db.getRow(sql"SELECT coalesce(sum(length(data)), 0) FROM message").get()[0].i let total_stored_chunk = relay.db.getRow(sql"SELECT coalesce(sum(length(val)), 0) FROM chunk").get()[0].i let total_stored = total_stored_note + total_stored_message + total_stored_chunk - let num_note = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM note").get()[0].i let num_message = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM message").get()[0].i let num_chunk = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM chunk").get()[0].i @@ -200,9 +199,9 @@ router myrouter: var traffic_by_ip: seq[TransferTotal] for row in relay.db.getAllRows(sql""" SELECT - sum(data_in) AS din, - sum(data_out) AS dout, - sum(data_in) + sum(data_out) AS total, + sum(coalesce(data_in, 0)) AS din, + sum(coalesce(data_out, 0)) AS dout, + sum(coalesce(data_in, 0)) + sum(coalesce(data_out, 0)) AS total, ip FROM stats_transfer @@ -224,9 +223,9 @@ router myrouter: var traffic_by_pubkey: seq[TransferTotal] for row in relay.db.getAllRows(sql""" SELECT - sum(data_in) AS din, - sum(data_out) AS dout, - sum(data_in) + sum(data_out) AS total, + sum(coalesce(data_in, 0)) AS din, + sum(coalesce(data_out, 0)) AS dout, + sum(coalesce(data_in, 0)) + sum(coalesce(data_out, 0)) AS total, pubkey FROM stats_transfer @@ -249,12 +248,12 @@ router myrouter: var storage_by_pubkey: seq[StorageStat] for row in relay.db.getAllRows(sql""" WITH msg AS ( - SELECT src, SUM(LENGTH(data)) AS msg_bytes + SELECT src, SUM(coalesce(LENGTH(data), 0)) AS msg_bytes FROM message GROUP BY src ), chunksize AS ( - SELECT src, SUM(LENGTH(val)) AS chunk_bytes + SELECT src, SUM(coalesce(LENGTH(val), 0)) AS chunk_bytes FROM chunk GROUP BY src ) From 33fcac4420eb692eac53317d63621a8199ee785c Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 21 Nov 2025 09:49:07 -0500 Subject: [PATCH 34/46] work on mounted volumes --- src/server2.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/server2.nim b/src/server2.nim index bcc5451..ce336aa 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -284,6 +284,9 @@ proc main(database: string, port: Port, address = "127.0.0.1") = addHandler(L) info "Database: ", database var db = open(database, "", "", "") + db.exec(sql"PRAGMA journal_mode=PERSIST") + db.exec(sql"PRAGMA busy_timeout = 5000") + db.exec(sql"PRAGMA synchronous = FULL") relay = newRelay[NetstringSocket](db) info &"Serving on {address}:{port.int}" let settings = newSettings(port=port, bindAddr=address) From e7a4db2c64f353cb4fde881d219e3762926ab759 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 21 Nov 2025 14:02:33 -0500 Subject: [PATCH 35/46] More stats --- .gitignore | 1 + src/proto2.nim | 42 ++++++++++++++++++ src/server2.nim | 89 +++++++++++++++++++++++++++++++++++++ src/templates/stats.nimja | 92 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 217 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index e2750f3..2bbbdf9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ libs *.keys *.sqlite tests/bin +*.sqlite-journal diff --git a/src/proto2.nim b/src/proto2.nim index 04d6b9d..4e886ba 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -229,6 +229,17 @@ proc updateSchema*(db: DbConn) = data_out INTEGER DEFAULT 0, PRIMARY KEY (period, ip, pubkey) )""") + db.patch(applied, "connects"): + db.exec(sql"""CREATE TABLE stats_event ( + period TEXT NOT NULL DEFAULT(strftime('%Y-%W')), + ip TEXT NOT NULL, + pubkey TEXT NOT NULL, + connect INTEGER DEFAULT 0, + publish INTEGER DEFAULT 0, + send INTEGER DEFAULT 0, + store INTEGER DEFAULT 0, + PRIMARY KEY (period, ip, pubkey) + )""") #----------- in-memory stuff db.exec(sql"""CREATE TEMPORARY TABLE note_sub ( @@ -329,6 +340,17 @@ when TESTMODE: data_out = data_out + excluded.data_out; """, ip, pubkey, period, data_in, data_out) +proc record_event_stat*(db: DbConn, ip: string, pubkey: PublicKey, connect = 0, publish = 0, send = 0, store = 0) = + db.exec(sql""" + INSERT INTO stats_event (ip, pubkey, connect, publish, send, store) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(period, ip, pubkey) DO UPDATE SET + connect = connect + excluded.connect, + publish = publish + excluded.publish, + send = send + excluded.send, + store = store + excluded.store + """, ip, pubkey, connect, publish, send, store) + proc chunk_space_used*(db: DbConn, pubkey: PublicKey): int = ## Return the amount of space being used by the given public key db.getRow(sql""" @@ -492,6 +514,11 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay relay.clients[pubkey] = conn info &"[{conn.pubkey.abbr}] connected" conn.sendOkay cmd.kind + relay.db.record_event_stat( + ip = conn.ip, + pubkey = pubkey, + connect = 1, + ) # send all queued messages relay.delExpiredMessages() @@ -523,6 +550,11 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay pubkey = pubkey, data_in = cmd.pub_data.len, ) + relay.db.record_event_stat( + ip = conn.ip, + pubkey = pubkey, + publish = 1, + ) let opubkey = relay.getNoteSub(cmd.pub_topic) if opubkey.isSome: # someone is waiting @@ -584,6 +616,11 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay pubkey = pubkey, data_in = cmd.send_val.len, ) + relay.db.record_event_stat( + ip = conn.ip, + pubkey = pubkey, + send = 1, + ) if relay.clients.hasKey(cmd.send_dst): # dst is online var other_conn = relay.clients[cmd.send_dst] @@ -613,6 +650,11 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay if relay.max_chunk_space > 0 and relay.db.chunk_space_used(pubkey) > relay.max_chunk_space: conn.sendError("Too much chunk data", cmd.kind, StorageLimitExceeded) else: + relay.db.record_event_stat( + ip = conn.ip, + pubkey = pubkey, + store = 1, + ) relay.db.exec(sql"BEGIN") try: relay.db.exec(sql"DELETE FROM chunk_dst WHERE src=? AND key=?", pubkey, cmd.chunk_key.DbBlob) diff --git a/src/server2.nim b/src/server2.nim index ce336aa..e841a0d 100644 --- a/src/server2.nim +++ b/src/server2.nim @@ -144,6 +144,14 @@ type message_size: int chunk_size: int total_size: int + + PubkeyEventStat = tuple + pubkey: PublicKey + count: int + + IPEventStat = tuple + ip: string + count: int router myrouter: get "/ws": @@ -173,6 +181,10 @@ router myrouter: strftime('%Y-%W') AS b""", days_back).get() (row[0].s, row[1].s) + # total users + let total_ips = relay.db.getRow(sql"SELECT coalesce(count(distinct ip), 0) FROM stats_event").get()[0].i + let total_pubkeys = relay.db.getRow(sql"SELECT coalesce(count(distinct pubkey), 0) FROM stats_event").get()[0].i + # total transfer let row = relay.db.getRow(sql""" SELECT @@ -274,6 +286,83 @@ router myrouter: chunk_size: row[2].i.int, total_size: row[3].i.int, )) + + # top events by pubkey + var connects_by_pubkey: seq[PubkeyEventStat] + for row in relay.db.getAllRows(sql""" + SELECT + pubkey, + COALESCE(SUM(connect), 0) + FROM + stats_event + WHERE + period >= ? + AND pubkey <> '' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 10 + """, datarange.a): + connects_by_pubkey.add(( + pubkey: PublicKey.fromDb(row[0].b), + count: row[1].i.int, + )) + + var publish_by_pubkey: seq[PubkeyEventStat] + for row in relay.db.getAllRows(sql""" + SELECT + pubkey, + COALESCE(SUM(publish), 0) + FROM + stats_event + WHERE + period >= ? + AND pubkey <> '' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 10 + """, datarange.a): + publish_by_pubkey.add(( + pubkey: PublicKey.fromDb(row[0].b), + count: row[1].i.int, + )) + + var send_by_pubkey: seq[PubkeyEventStat] + for row in relay.db.getAllRows(sql""" + SELECT + pubkey, + COALESCE(SUM(send), 0) + FROM + stats_event + WHERE + period >= ? + AND pubkey <> '' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 10 + """, datarange.a): + send_by_pubkey.add(( + pubkey: PublicKey.fromDb(row[0].b), + count: row[1].i.int, + )) + + var store_by_pubkey: seq[PubkeyEventStat] + for row in relay.db.getAllRows(sql""" + SELECT + pubkey, + COALESCE(SUM(store), 0) + FROM + stats_event + WHERE + period >= ? + AND pubkey <> '' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 10 + """, datarange.a): + store_by_pubkey.add(( + pubkey: PublicKey.fromDb(row[0].b), + count: row[1].i.int, + )) var html = "" compileTemplateFile("stats.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") diff --git a/src/templates/stats.nimja b/src/templates/stats.nimja index 04aa5c5..ddde498 100644 --- a/src/templates/stats.nimja +++ b/src/templates/stats.nimja @@ -20,16 +20,31 @@ border: 1px solid #ddd; padding: 0 3px; } + .rows { + display: flex; + }

Stats [{{ datarange.a }}, {{ datarange.b }}]

+

Users

+ + + + + + + + + +
IPsPubkeys
{{ total_ips.wcommas }}{{ total_pubkeys.wcommas }}
+

Transfer

- - + + @@ -49,7 +64,7 @@ - + @@ -68,8 +83,8 @@
Data inData outBytes inBytes out Total
Total
SizeBytes {{ total_stored_note.wcommas }} {{ total_stored_message.wcommas }} {{ total_stored_chunk.wcommas }}
- - + + {% for tot in traffic_by_ip %} @@ -86,8 +101,8 @@
IPData inData outBytes inBytes out Total
- - + + {% for tot in traffic_by_pubkey %} @@ -115,5 +130,68 @@ {% endfor %}
PubkeyData inData outBytes inBytes out Total
+ +

Top events by Pubkey

+
+
+ + + + + + {% for st in connects_by_pubkey %} + + + + + {% endfor %} +
PubkeyConnect
{{ st.pubkey.abbr }}{{ st.count.wcommas }}
+
+ +
+ + + + + + {% for st in publish_by_pubkey %} + + + + + {% endfor %} +
PubkeyPublish
{{ st.pubkey.abbr }}{{ st.count.wcommas }}
+
+ +
+ + + + + + {% for st in send_by_pubkey %} + + + + + {% endfor %} +
PubkeySend
{{ st.pubkey.abbr }}{{ st.count.wcommas }}
+
+ +
+ + + + + + {% for st in store_by_pubkey %} + + + + + {% endfor %} +
PubkeyStore
{{ st.pubkey.abbr }}{{ st.count.wcommas }}
+
+
\ No newline at end of file From 19192e9f94c24958df6ecdead1c860a5d584b3a4 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 21 Nov 2025 14:32:46 -0500 Subject: [PATCH 36/46] A little effort given to refusing invalid public keys --- src/objs.nim | 1 + src/proto2.nim | 17 +++++++++++++++++ tests/tproto2.nim | 31 ++++++++++++++++++++++++++++--- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/objs.nim b/src/objs.nim index 21dbce6..1209d87 100644 --- a/src/objs.nim +++ b/src/objs.nim @@ -45,6 +45,7 @@ type TooLarge = 2 StorageLimitExceeded = 3 TransferLimitExceeeded = 4 + InvalidParams = 5 RelayMessage* = object case kind*: MessageKind diff --git a/src/proto2.nim b/src/proto2.nim index 4e886ba..3042135 100644 --- a/src/proto2.nim +++ b/src/proto2.nim @@ -289,6 +289,19 @@ template sendOkay*[T](conn: RelayConnection[T], cmd: CommandKind) = ok_cmd: cmd, )) +proc is_valid*(x: PublicKey): bool = + ## Return true if it looks like a valid public key + if x.string.len == 32: + return true + return false + +proc any_invalid(x: seq[PublicKey]): bool = + ## Return true if any of the public keys are invalid + for pk in x: + if not pk.is_valid(): + return true + return false + proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = new(result) result.sender = client @@ -606,6 +619,8 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay of SendData: if cmd.send_val.len > RELAY_MAX_MESSAGE_SIZE: conn.sendError("Data too long", cmd.kind, TooLarge) + elif not cmd.send_dst.is_valid(): + conn.sendError("Invalid pubkey", cmd.kind, InvalidParams) else: let pubkey = conn.pubkey.get() if relay.max_transfer_rate != 0 and relay.db.current_data_in(pubkey) > relay.max_transfer_rate: @@ -645,6 +660,8 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay conn.sendError("Value too long", cmd.kind, TooLarge) elif cmd.chunk_dst.len > RELAY_MAX_CHUNK_DSTS: conn.sendError("Too many recipients", cmd.kind, TooLarge) + elif cmd.chunk_dst.any_invalid(): + conn.sendError("Invalid pubkey", cmd.kind, InvalidParams) else: let pubkey = conn.pubkey.get() if relay.max_chunk_space > 0 and relay.db.chunk_space_used(pubkey) > relay.max_chunk_space: diff --git a/tests/tproto2.nim b/tests/tproto2.nim index ae2cba6..6e9be92 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -481,6 +481,20 @@ suite "data": let err = alice.pop(Error) check err.err_code == TransferLimitExceeeded check err.err_cmd == SendData + + test "invalid pubkey": + var relay = testRelay() + let chunksize = RELAY_MAX_MESSAGE_SIZE div 2 + relay.max_transfer_rate = chunksize * 10 + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_dst: "invalid".PublicKey, + send_val: "a", + )) + let err = alice.pop(Error) + check err.err_code == InvalidParams + check err.err_cmd == SendData proc storeChunk(conn: var RelayConnection[TestClient], key: string, val: string, dst = newSeq[PublicKey]()) = @@ -629,8 +643,7 @@ suite "chunks": checkpoint $relay.db.getAllRows(sql"SELECT src, key, last_used FROM chunk") skewTime(3) checkpoint $relay.db.getAllRows(sql"SELECT src, key, last_used FROM chunk") - check alice.chunkExists(alice, "key") - + check alice.chunkExists(alice, "key") test "remove dst": let relay = testRelay() @@ -724,7 +737,19 @@ suite "chunks": let err = alice.pop(Error) check err.err_cmd == StoreChunk check err.err_code == StorageLimitExceeded - + + test "invalid pubkey": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: @["fake".PublicKey], + chunk_key: "a", + chunk_val: "b", + )) + let err = alice.pop(Error) + check err.err_code == InvalidParams + check err.err_cmd == StoreChunk suite "anon": From e23623ea246ffa72a577952c9bb3f3317fd4e36e Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 25 Nov 2025 08:21:02 -0500 Subject: [PATCH 37/46] Move to subdir for better importing --- docker/Dockerfile | 2 +- src/{ => bucketsrelay}/cli.nim | 0 src/{ => bucketsrelay}/objs.nim | 0 src/{ => bucketsrelay}/proto2.nim | 0 src/{ => bucketsrelay}/sampleclient.nim | 0 src/{ => bucketsrelay}/server2.nim | 8 ++++---- tests/tfunctional.nim | 6 +++--- tests/tnetstring.nim | 2 +- tests/tproto2.nim | 2 +- tests/tserde2.nim | 4 ++-- 10 files changed, 12 insertions(+), 12 deletions(-) rename src/{ => bucketsrelay}/cli.nim (100%) rename src/{ => bucketsrelay}/objs.nim (100%) rename src/{ => bucketsrelay}/proto2.nim (100%) rename src/{ => bucketsrelay}/sampleclient.nim (100%) rename src/{ => bucketsrelay}/server2.nim (98%) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5d603ae..44fb281 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ RUN pkger fetch COPY ./ ./ COPY docker/config.nims . RUN find . -RUN nim c -d:release -o:brelay src/server2.nim +RUN nim c -d:release -o:brelay src/bucketsrelay/server2.nim RUN strip brelay # -- Stage 2 -- # diff --git a/src/cli.nim b/src/bucketsrelay/cli.nim similarity index 100% rename from src/cli.nim rename to src/bucketsrelay/cli.nim diff --git a/src/objs.nim b/src/bucketsrelay/objs.nim similarity index 100% rename from src/objs.nim rename to src/bucketsrelay/objs.nim diff --git a/src/proto2.nim b/src/bucketsrelay/proto2.nim similarity index 100% rename from src/proto2.nim rename to src/bucketsrelay/proto2.nim diff --git a/src/sampleclient.nim b/src/bucketsrelay/sampleclient.nim similarity index 100% rename from src/sampleclient.nim rename to src/bucketsrelay/sampleclient.nim diff --git a/src/server2.nim b/src/bucketsrelay/server2.nim similarity index 98% rename from src/server2.nim rename to src/bucketsrelay/server2.nim index e841a0d..edef9cd 100644 --- a/src/server2.nim +++ b/src/bucketsrelay/server2.nim @@ -30,8 +30,8 @@ type const VERSION = slurp"../CHANGELOG.md".split(" ")[1] - logo_png = slurp"static/logo.png" - favicon_png = slurp"static/favicon.png" + logo_png = slurp"../static/logo.png" + favicon_png = slurp"../static/favicon.png" let ADMIN_USERNAME = getEnv("ADMIN_USERNAME", "admin") let ADMIN_PWHASH = when defined(release): @@ -160,7 +160,7 @@ router myrouter: get "/": var html = "" - compileTemplateFile("index.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") + compileTemplateFile("index.nimja", baseDir = getScriptDir() / ".." / "templates", autoEscape = true, varname = "html") resp html get "/static/logo.png": @@ -365,7 +365,7 @@ router myrouter: )) var html = "" - compileTemplateFile("stats.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") + compileTemplateFile("stats.nimja", baseDir = getScriptDir() / ".." / "templates", autoEscape = true, varname = "html") resp html proc main(database: string, port: Port, address = "127.0.0.1") = diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index 7c6230a..0400be9 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -8,8 +8,8 @@ import std/unittest import ./util -import sampleclient -import proto2 +import bucketsrelay/sampleclient +import bucketsrelay/proto2 import ws @@ -27,7 +27,7 @@ proc startServer(port: Port): Process = let compileProcess = startProcess( "nim", workingDir = currentSourcePath().parentDir().parentDir(), - args = ["c", "-d:testmode", "-o:" & bin, "src/server2.nim"], + args = ["c", "-d:testmode", "-o:" & bin, "src/bucketsrelay/server2.nim"], options = {poStdErrToStdOut, poUsePath} ) let output = compileProcess.outputStream.readAll() diff --git a/tests/tnetstring.nim b/tests/tnetstring.nim index 9b2cd54..2f31298 100644 --- a/tests/tnetstring.nim +++ b/tests/tnetstring.nim @@ -1,6 +1,6 @@ import std/unittest -import objs +import bucketsrelay/objs suite "encode": test "nsencode": diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 6e9be92..3a19eaa 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -6,7 +6,7 @@ import std/strutils import std/unittest import lowdb/sqlite -import proto2 +import bucketsrelay/proto2 if getEnv("SHOW_LOGS") != "": var L = newConsoleLogger() diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 0b0e56b..19167ba 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -3,8 +3,8 @@ import std/logging import std/options import ./util -import proto2 -import objs +import bucketsrelay/proto2 +import bucketsrelay/objs test "MessageKind": for kind in low(MessageKind)..high(MessageKind): From b753c0cdb25f4124f1f72b440f15900000794976 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Tue, 25 Nov 2025 10:16:43 -0500 Subject: [PATCH 38/46] Bring v1 back --- docker/Dockerfile | 2 +- src/bucketsrelay/v1/asyncstdin.nim | 84 ++ src/bucketsrelay/v1/bclient.nim | 201 ++++ src/bucketsrelay/v1/brelay.nim | 178 ++++ src/bucketsrelay/v1/client.nim | 246 +++++ src/bucketsrelay/v1/common.nim | 59 ++ src/bucketsrelay/v1/dbschema.nim | 41 + src/bucketsrelay/v1/httpreq.nim | 88 ++ src/bucketsrelay/v1/jwtrsaonly.nim | 253 +++++ src/bucketsrelay/v1/licenses.nim | 80 ++ src/bucketsrelay/v1/mailer.nim | 56 ++ src/bucketsrelay/v1/netstring.nim | 116 +++ src/bucketsrelay/v1/proto.nim | 403 ++++++++ src/bucketsrelay/v1/server.nim | 922 ++++++++++++++++++ src/bucketsrelay/v1/stringproto.nim | 108 ++ src/bucketsrelay/{ => v2}/cli.nim | 0 src/bucketsrelay/{ => v2}/objs.nim | 0 src/bucketsrelay/{ => v2}/proto2.nim | 0 src/bucketsrelay/{ => v2}/sampleclient.nim | 0 src/bucketsrelay/{ => v2}/server2.nim | 8 +- src/{ => bucketsrelay/v2}/static/favicon.png | Bin src/{ => bucketsrelay/v2}/static/logo.png | Bin .../v2}/templates/index.nimja | 0 .../v2}/templates/stats.nimja | 0 tests/tfunctional.nim | 4 +- tests/tnetstring.nim | 2 +- tests/tproto2.nim | 2 +- tests/tserde2.nim | 4 +- 28 files changed, 2846 insertions(+), 11 deletions(-) create mode 100644 src/bucketsrelay/v1/asyncstdin.nim create mode 100644 src/bucketsrelay/v1/bclient.nim create mode 100644 src/bucketsrelay/v1/brelay.nim create mode 100644 src/bucketsrelay/v1/client.nim create mode 100644 src/bucketsrelay/v1/common.nim create mode 100644 src/bucketsrelay/v1/dbschema.nim create mode 100644 src/bucketsrelay/v1/httpreq.nim create mode 100644 src/bucketsrelay/v1/jwtrsaonly.nim create mode 100644 src/bucketsrelay/v1/licenses.nim create mode 100644 src/bucketsrelay/v1/mailer.nim create mode 100644 src/bucketsrelay/v1/netstring.nim create mode 100644 src/bucketsrelay/v1/proto.nim create mode 100644 src/bucketsrelay/v1/server.nim create mode 100644 src/bucketsrelay/v1/stringproto.nim rename src/bucketsrelay/{ => v2}/cli.nim (100%) rename src/bucketsrelay/{ => v2}/objs.nim (100%) rename src/bucketsrelay/{ => v2}/proto2.nim (100%) rename src/bucketsrelay/{ => v2}/sampleclient.nim (100%) rename src/bucketsrelay/{ => v2}/server2.nim (98%) rename src/{ => bucketsrelay/v2}/static/favicon.png (100%) rename src/{ => bucketsrelay/v2}/static/logo.png (100%) rename src/{ => bucketsrelay/v2}/templates/index.nimja (100%) rename src/{ => bucketsrelay/v2}/templates/stats.nimja (100%) diff --git a/docker/Dockerfile b/docker/Dockerfile index 44fb281..6135e7c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ RUN pkger fetch COPY ./ ./ COPY docker/config.nims . RUN find . -RUN nim c -d:release -o:brelay src/bucketsrelay/server2.nim +RUN nim c -d:release -o:brelay src/bucketsrelay/v2/server2.nim RUN strip brelay # -- Stage 2 -- # diff --git a/src/bucketsrelay/v1/asyncstdin.nim b/src/bucketsrelay/v1/asyncstdin.nim new file mode 100644 index 0000000..b909bbe --- /dev/null +++ b/src/bucketsrelay/v1/asyncstdin.nim @@ -0,0 +1,84 @@ +## Asynchronous reading from stdin +## +## The implementation may change. The important thing is that this works: +## +## var reader = asyncStdinReader() +## let res = waitFor reader.read(10) +import std/deques +import chronos + +const BUFSIZE = 4096.uint + +type + ReadResponse = uint + + AsyncStdinReader* = ref object + outQ: AsyncQueue[string] + inQ: AsyncQueue[uint] + closed: bool + thread: Thread[AsyncFD] + +var requestCh: Channel[uint] +requestCh.open(0) + +proc workerReadLoop(wfd: AsyncFD) {.thread.} = + ## Run this in a thread other than the main one + ## to get somewhat asynchronous IO + let transp = fromPipe(wfd) + var buf: array[BUFSIZE, char] + var closed = false + while not closed: + let req = requestCh.recv() + var toRead = req + var didRead: uint = 0 + while toRead > 0: + let toReadThisTime = min(BUFSIZE, toRead) + let n = stdin.readBuffer(addr buf, toReadThisTime) + if n == 0: + closed = true + break + didRead.inc(n) + toRead.dec(n) + discard waitFor transp.write(addr buf, n) + waitFor transp.closeWait() + +proc mainReadLoop(reader: AsyncStdinReader, transp: StreamTransport) {.async.} = + ## Run this companion loop of workerReadLoop in the main thread + while true: + let size = await reader.inQ.get() + var ret = "" + if not reader.closed: + var toRead = size + while toRead > 0 and not reader.closed: + var data: seq[byte] + try: + data = await transp.read(toRead.int) + except: + discard + if data.len == 0: + reader.closed = true + break + for c in data: + ret.add(chr(c)) + toRead.dec(data.len) + reader.outQ.putNoWait(ret) + +#--------------------------------------------------------------- +# Public API +#--------------------------------------------------------------- +proc asyncStdinReader*(): AsyncStdinReader = + new(result) + result.inQ = newAsyncQueue[uint]() + result.outQ = newAsyncQueue[string]() + let (rfd, wfd) = createAsyncPipe() + let readTransp = fromPipe(rfd) + result.thread.createThread(workerReadLoop, wfd) + asyncSpawn result.mainReadLoop(readTransp) + +proc read*(reader: AsyncStdinReader, size: uint): Future[string] {.async.} = + requestCh.send(size) + reader.inQ.putNoWait(size) + return await reader.outQ.get() + +template read*(reader: AsyncStdinReader, size: int): untyped = + reader.read(size.uint) diff --git a/src/bucketsrelay/v1/bclient.nim b/src/bucketsrelay/v1/bclient.nim new file mode 100644 index 0000000..05af65b --- /dev/null +++ b/src/bucketsrelay/v1/bclient.nim @@ -0,0 +1,201 @@ +# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +import std/logging +import std/strformat +import std/strutils +import std/base64 +import std/os + +import chronos except debug, info, warn, error + +import ./client +import ./proto +import ./asyncstdin + +type + SendHandler = ref object + data: string + sent: Future[void] + +proc newSendHandler(data: string): SendHandler = + new(result) + result.data = data + result.sent = newFuture[void]("newSendHandler") + +proc handleEvent(handler: SendHandler, ev: RelayEvent, remote: RelayClient) {.async.} = + try: + case ev.kind + of Connected: + await remote.sendData(ev.conn_pubkey, handler.data) + callSoon(proc(udata: pointer) = + handler.sent.complete()) + else: + discard + except CancelledError: + warn "SendHandler cancelled during event handling" + raise + +proc handleLifeEvent(handler: SendHandler, ev: ClientLifeEvent, remote: RelayClient) {.async.} = + discard + + +type + RecvHandler = ref object + buf: string + data: Future[string] + +proc newRecvHandler(): RecvHandler = + new(result) + result.data = newFuture[string]("newRecvHandler") + +proc handleEvent(handler: RecvHandler, ev: RelayEvent, remote: RelayClient) {.async.} = + try: + case ev.kind + of Data: + handler.buf.add(ev.data) + of Disconnected: + handler.data.complete(handler.buf) + else: + discard + except CancelledError: + warn "RecvHandler cancelled during event handling" + raise + +proc handleLifeEvent(handler: RecvHandler, ev: ClientLifeEvent, remote: RelayClient) {.async.} = + discard + +proc relaySend*(data: string, topubkey: PublicKey, relayurl: string, mykeys: KeyPair, username: string, password: string, verify = true): Future[void] {.async.} = + debug &"Sending {data.len} bytes to {topubkey} via {relayurl} ..." + var sh = newSendHandler(data) + var client = newRelayClient(mykeys, sh, username, password, verifyHostname = verify) + await client.connect(relayurl) + await client.connect(topubkey) + await sh.sent + await client.disconnect() + await client.done + +proc relayReceive*(frompubkey: PublicKey, relayurl: string, mykeys: KeyPair, username: string, password: string, verify = true): Future[string] {.async.} = + debug &"Receiving from {frompubkey} via {relayurl} ..." + var rh = newRecvHandler() + var client = newRelayClient(mykeys, rh, username, password, verifyHostname = verify) + await client.connect(relayurl) + await client.connect(frompubkey) + result = await rh.data + await client.disconnect() + await client.done + +type + ChatHandler = ref object + done: Future[void] + +proc handleEvent(handler: ChatHandler, ev: RelayEvent, remote: RelayClient) {.async.} = + case ev.kind + of Data: + stdout.write(ev.data) + stdout.flushFile() + of Disconnected: + handler.done.complete() + else: + discard + +proc handleLifeEvent(handler: ChatHandler, ev: ClientLifeEvent, remote: RelayClient) {.async.} = + discard + +proc chat(ch: ChatHandler, remote: RelayClient, remotePubkey: PublicKey) {.async.} = + let reader = asyncStdinReader() + while true: + let inp = reader.read(1) + await ch.done or inp + if ch.done.completed: + await inp.cancelAndWait() + break + else: + let data = inp.read() + await remote.sendData(remotePubkey, data) + +proc relayChat*(otherpubkey: PublicKey, relayurl: string, mykeys: KeyPair, username: string, password: string, verify = true): Future[string] {.async.} = + debug &"Attempting to chat with {otherpubkey} via {relayurl} ..." + var ch = ChatHandler() + ch.done = newFuture[void]() + var client = newRelayClient(mykeys, ch, username, password, verifyHostname = verify) + await client.connect(relayurl) + await client.connect(otherpubkey) + await ch.chat(client, otherpubkey) + +when isMainModule: + import argparse + var p = newParser: + command("genkeys"): + help("Generate a keypair") + option("-p", "--public", help="Filename to save public key to", default=some("relay.key.public")) + option("-s", "--secret", help="Filename to save secret key to", default=some("relay.key.secret")) + run: + var keys = genkeys() + writeFile(opts.public, keys.pk.string.encode & "\n") + echo "Wrote ", opts.public + writeFile(opts.secret, keys.sk.string.encode & "\n") + echo "Wrote ", opts.secret + echo "Public key:" + echo keys.pk.string.encode() + command("send"): + help("Send stdin through the relay") + option("-u", "--username", help="Relay username", env = "RELAY_USERNAME") + option("-p", "--password", help="Relay password", env = "RELAY_PASSWORD") + flag("-k", "--no-ssl-verify", help="Disable SSL verification") + option("--local-secret", help="Path to local secret key", default=some("relay.key.secret")) + option("--local-public", help="Path to local public key", default=some("relay.key.public")) + arg("url", help="Relay URL to connect to. Should end in /relay") + arg("public_key", help="Public key of remote client to connect to") + run: + newConsoleLogger(lvlAll, useStderr = true).addHandler() + let keys = ( + readFile(opts.local_public).decode().PublicKey, + readFile(opts.local_secret).decode().SecretKey, + ) + let pubkey = opts.public_key.decode().PublicKey + let data = stdin.readAll() + waitFor relaySend(data, pubkey, relayurl = opts.url, mykeys = keys, username = opts.username, password = opts.password, verify = not opts.no_ssl_verify) + command("receive"): + help("Receive data through the relay to stdout") + option("-u", "--username", help="Relay username", env = "RELAY_USERNAME") + option("-p", "--password", help="Relay password", env = "RELAY_PASSWORD") + flag("-k", "--no-ssl-verify", help="Disable SSL verification") + option("--local-secret", help="Path to local secret key", default=some("relay.key.secret")) + option("--local-public", help="Path to local public key", default=some("relay.key.public")) + arg("url", help="Relay URL to connect to. Should end in /relay") + arg("public_key", help="Public key of remote client to connect to") + run: + newConsoleLogger(lvlAll, useStderr = true).addHandler() + let keys = ( + readFile(opts.local_public).decode().PublicKey, + readFile(opts.local_secret).decode().SecretKey, + ) + let pubkey = opts.public_key.decode().PublicKey + echo waitFor relayReceive(pubkey, relayurl = opts.url, mykeys = keys, username = opts.username, password = opts.password, verify = not opts.no_ssl_verify) + command("chat"): + help("Open a symmetric chat stream with another client") + option("-u", "--username", help="Relay username", env = "RELAY_USERNAME") + option("-p", "--password", help="Relay password", env = "RELAY_PASSWORD") + flag("-k", "--no-ssl-verify", help="Disable SSL verification") + option("--local-secret", help="Path to local secret key", default=some("relay.key.secret")) + option("--local-public", help="Path to local public key", default=some("relay.key.public")) + flag("-v", "--verbose", help="Verbose logging") + arg("url", help="Relay URL to connect to. Should end in /relay") + arg("public_key", help="Public key of remote client to connect to") + run: + if opts.verbose: + newConsoleLogger(lvlAll, useStderr = true).addHandler() + let keys = ( + readFile(opts.local_public).decode().PublicKey, + readFile(opts.local_secret).decode().SecretKey, + ) + let pubkey = opts.public_key.decode().PublicKey + echo waitFor relayChat(pubkey, relayurl = opts.url, mykeys = keys, username = opts.username, password = opts.password, verify = not opts.no_ssl_verify) + try: + p.run() + except UsageError: + stderr.writeLine getCurrentExceptionMsg() + quit(1) diff --git a/src/bucketsrelay/v1/brelay.nim b/src/bucketsrelay/v1/brelay.nim new file mode 100644 index 0000000..dfd95ec --- /dev/null +++ b/src/bucketsrelay/v1/brelay.nim @@ -0,0 +1,178 @@ +# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +import std/logging +import std/strformat +import std/json + +import chronos + +import ./common +import ./server + +proc monitorMemory() {.async.} = + var + lastTotal = 0 + lastOccupied = 0 + lastFree = 0 + while true: + let + newTotal = getTotalMem() + newOccupied = getOccupiedMem() + newFree = getFreeMem() + diffTotal = newTotal - lastTotal + diffOccupied = newOccupied - lastOccupied + diffFree = newFree - lastFree + debug "--- Memory report ---" + debug &"Total memory: {newTotal:>10} <- {lastTotal:>10} diff {diffTotal:>10}" + debug &"Occupied memory: {newOccupied:>10} <- {lastOccupied:>10} diff {diffOccupied:>10}" + debug &"Free memory: {newFree:>10} <- {lastFree:>10} diff {diffFree:>10}" + lastTotal = newTotal + lastOccupied = newOccupied + lastFree = newFree + await sleepAsync(10.seconds) + +proc startRelaySingleUser*(username, password: string, port = 9001.Port, address = "127.0.0.1"): RelayServer {.singleuseronly.} = + ## Start the relay server on the given port. + result = newRelayServer(username, password) + let taddress = initTAddress(address, port.int) + info &"Starting Single-User Buckets Relay on {taddress} ..." + stderr.flushFile + result.start(taddress) + +proc getRelayServer(dbfilename: string): RelayServer {.multiuseronly.} = + newRelayServer(dbfilename, pubkey = AUTH_LICENSE_PUBKEY) + +proc startRelay*(dbfilename: string, port = 9001.Port, address = "127.0.0.1"): RelayServer {.multiuseronly.} = + ## Start the relay server on the given port. + result = getRelayServer(dbfilename) + let taddress = initTAddress(address, port.int) + info &"Starting Buckets Relay on {taddress} ..." + if result.pubkey == "": + info &"[config] License auth: DISABLED" + else: + info &"[config] License auth: on" + stderr.flushFile + result.start(taddress) + result.periodically_delete_old_stats() + +proc addverifieduser*(dbfilename, username, password: string) {.multiuseronly.} = + var rs = getRelayServer(dbfilename) + let userid = rs.register_user(username, password) + let token = rs.generate_email_verification_token(userid) + doAssert rs.use_email_verification_token(userid, token) == true + +proc blockuser*(dbfilename, email: string) {.multiuseronly.} = + var rs = getRelayServer(dbfilename) + let uid = rs.get_user_id(email) + rs.block_user(uid) + +proc unblockuser*(dbfilename, email: string) {.multiuseronly.} = + var rs = getRelayServer(dbfilename) + let uid = rs.get_user_id(email) + rs.unblock_user(uid) + +proc blocklicense*(dbfilename, email: string) {.multiuseronly.} = + var rs = getRelayServer(dbfilename) + let uid = rs.get_user_id(email) + rs.disable_most_recently_used_license(uid) + +proc stats(dbfilename: string, days = 30): JsonNode {.multiuseronly.} = + result = %* { + "days": days, + "users": [], + "ips": [], + } + var rs = newRelayServer(dbfilename, updateSchema = false, pubkey = AUTH_LICENSE_PUBKEY) + for row in rs.top_data_users(20, days = days): + result["users"].add(%* { + "sent": row.data.sent, + "recv": row.data.recv, + "user": row.user, + }) + for row in rs.top_data_ips(20, days = days): + result["ips"].add(%* { + "sent": row.data.sent, + "recv": row.data.recv, + "ip": row.ip, + }) + +proc showStats(dbfilename: string, days = 30): string {.multiuseronly.} = + ## Show some usage stats + return stats(dbfilename, days).pretty + +when defined(posix): + proc getpass(prompt: cstring) : cstring {.header: "", importc: "getpass".} +else: + proc getpass(prompt: cstring): cstring = + stdout.write(prompt) + stdout.flushFile() + stdin.readLine() + +when isMainModule: + import argparse + newConsoleLogger(lvlAll, useStderr = true).addHandler() + when multiusermode: + var p = newParser: + option("-d", "--database", help="User/stats database filename", default=some("bucketsrelay.sqlite")) + command("adduser"): + help("Add a user") + arg("email", help="Email address of user") + flag("--password-stdin", help="If given, read the password from stdin rather than from the terminal") + run: + var password = if opts.password_stdin: + stdout.write("Password? ") + stdout.flushFile + stdin.readLine() + else: + $getpass("Password? ".cstring) + addverifieduser(opts.parentOpts.database, opts.email, password) + echo "added user ", opts.email + command("blockuser"): + help("Block a user from using the relay") + arg("email", help="Email address of user to block") + run: + blockuser(opts.parentOpts.database, opts.email) + echo "User blocked" + command("unblockuser"): + help("Unblock a previously blocked user") + arg("email", help="Email address of user to block") + run: + unblockuser(opts.parentOpts.database, opts.email) + echo "User unblocked" + command("disablelicense"): + help("Disable a user's most recently-used license") + arg("email", help="Email address of user") + run: + blocklicense(opts.parentOpts.database, opts.email) + echo "License disabled" + command("stats"): + help("Show some statistics") + option("--days", help = "Show data for this number of days", default=some("30")) + run: + echo showStats(opts.parentOpts.database, days=opts.days.parseInt) + command("server"): + help("Start the relay server") + option("-p", "--port", help="Port to run server on", default=some("9001")) + option("-a", "--address", help="Address to run on", default=some("127.0.0.1")) + run: + var server = startRelay(opts.parentOpts.database, opts.port.parseInt.Port, opts.address) + runForever() + elif singleusermode: + var p = newParser: + command("server"): + help("Start a single user relay server. Set RELAY_USERNAME and RELAY_PASSWORD environment variables") + option("-p", "--port", help="Port to run server on", default=some("9001")) + option("-a", "--address", help="Address to run on", default=some("127.0.0.1")) + option("-u", "--username", help="Username", env = "RELAY_USERNAME") + option("-P", "--password", help="Password", env = "RELAY_PASSWORD") + run: + var server = startRelaySingleUser(opts.username, opts.password, opts.port.parseInt.Port, opts.address) + runForever() + try: + p.run() + except UsageError: + stderr.writeLine getCurrentExceptionMsg() + quit(1) diff --git a/src/bucketsrelay/v1/client.nim b/src/bucketsrelay/v1/client.nim new file mode 100644 index 0000000..586e161 --- /dev/null +++ b/src/bucketsrelay/v1/client.nim @@ -0,0 +1,246 @@ +# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. +import std/base64 +import std/logging +import std/options +import std/random +import std/strformat + +import chronos; export chronos +import chronicles except debug, info, warn, error +export activeChroniclesStream, Record, activeChroniclesScope +import stew/byteutils +import websock/websock + +import ./common +import ./netstring +import ./proto; export proto +import ./stringproto + +const HEARTBEAT_INTERVAL = 50.seconds +const HEARTBEAT_JITTER = 1000 + +type + ## RelayClient wraps a single websockets connection + ## to a relay server. It will call things on + ## `handler: T` to interact with your code. + RelayClient*[T] = ref object + keys: KeyPair + wsopt: Option[WSSession] + handler*: T + username: string + password: string + verifyHostname: bool + done*: Future[void] + tasks*: seq[Future[void]] + debugname*: string + + ClientLifeEventKind* = enum + ConnectedToServer + DisconnectedFromServer + + ClientLifeEvent* = ref object + case kind*: ClientLifeEventKind + of ConnectedToServer: + discard + of DisconnectedFromServer: + discard + + RelayErrLoginFailed* = RelayErr + RelayNotConnected* = RelayErr + +proc newRelayClient*[T](keys: KeyPair, handler: T, username, password: string, verifyHostname = true): RelayClient[T] = + new(result) + result.keys = keys + result.handler = handler + result.username = username + result.password = password + result.verifyHostname = verifyHostname + # result.done = newFuture[void]("newRelayClient done") + result.debugname = "RelayClient" & nextDebugName() + +proc logname*(client: RelayClient): string = + "(" & client.debugname & ") " + +proc `$`*(client: RelayClient): string = client.debugname + +proc ws*(client: RelayClient): WSSession = + if client.wsopt.isSome: + client.wsopt.get() + else: + raise RelayNotConnected.newException("Not connected") + +proc send(ws: WSSession, cmd: RelayCommand) {.async.} = + ## Send a RelayCommand to the server + await ws.send(nsencode(dumps(cmd)).toBytes, Opcode.Binary) + +proc keepAliveLoop(client: RelayClient) {.async.} = + ## Start a loop that periodically issues a ping to keep the + ## connection alive + try: + while true: + await sleepAsync(HEARTBEAT_INTERVAL + rand(HEARTBEAT_JITTER).milliseconds) + if client.wsopt.isSome: + let ws = client.wsopt.get() + await ws.ping() + else: + break + except CancelledError: + discard + except: + error client.logname, "unexpected error in ws keepAliveLoop" + +proc loop(client: RelayClient, authenticated: Future[void]): Future[void] {.async.} = + var decoder = newNetstringDecoder() + if client.wsopt.isSome(): + let ws = client.ws + while ws.readyState != ReadyState.Closed: + try: + let buff = try: + await ws.recvMsg() + except Exception as exc: + break + if buff.len <= 0: + break + let data = string.fromBytes(buff) + decoder.consume(data) + while decoder.hasMessage(): + let ev = loadsRelayEvent(decoder.nextMessage()) + case ev.kind + of Who: + await ws.send(RelayCommand( + kind: Iam, + iam_signature: sign(client.keys.sk, ev.who_challenge), + iam_pubkey: client.keys.pk, + )) + of Authenticated: + authenticated.complete() + else: + discard + try: + await client.handler.handleEvent(ev, client) + except: + debug client.logname, "Error handling event ", $ev, " ", getCurrentExceptionMsg() + raise + except CancelledError: + break + debug client.logname, "closing..." + client.wsopt = none[WSSession]() + await ws.close() + await client.handler.handleLifeEvent(ClientLifeEvent( + kind: DisconnectedFromServer, + ), client) + +proc authHeaderHook*(username, password: string): Hook = + ## Create a websock connection hook that adds Basic HTTP authentication + ## to the websocket. + new(result) + result.append = proc(ctx: Hook, headers: var HttpTable): Result[void, string] = + headers.add("Authorization", "Basic " & base64.encode(username & ":" & password)) + ok() + +proc addHeadersHook(addheaders: HttpTable): Hook = + new(result) + result.append = proc(ctx: Hook, headers: var HttpTable): Result[void, string] = + for key, val in addheaders.stringItems: + headers.add(key, val) + ok() + +proc connect*(client: RelayClient, url: string) {.async.} = + ## Connect and authenticate with a relay server. Returns + ## after authentication succeeds. + var uri = parseUri(url) + if uri.scheme == "http": + uri.scheme = "ws" + elif uri.scheme == "https": + uri.scheme = "wss" + let + hostname = uri.hostname + port = if uri.port == "": "443" else: uri.port + addresses = resolveTAddress(uri.hostname, port.parseInt.Port) + hooks = @[ + authHeaderHook(client.username, client.password), + addHeadersHook(HttpTable.init({ + "User-Agent": "buckets-relay client 1.0", + })), + ] + tls = uri.scheme == "https" or uri.scheme == "wss" or port == "443" + if addresses.len == 0: + raise ValueError.newException(&"Unable to resolve {uri.hostname}") + let address = addresses[0] + try: + let ws = if tls: + var flags: set[TLSFlags] + if not client.verifyHostname: + flags.incl(TLSFlags.NoVerifyHost) + flags.incl(TLSFlags.NoVerifyServerName) + await WebSocket.connect( + uri, + protocols = @["proto"], + flags = flags, + hooks = hooks, + version = WSDefaultVersion, + frameSize = WSDefaultFrameSize, + onPing = nil, + onPong = nil, + onClose = nil, + rng = nil, + ) + else: + await WebSocket.connect( + address, + path = uri.path, + hooks = hooks, + hostName = hostname, + ) + client.wsopt = some(ws) + except WebSocketError as exc: + let msg = getCurrentExceptionMsg() + if "403" in msg and "Forbidden" in msg: + raise RelayErrLoginFailed.newException("Failed initial authentication") + else: + raise exc + await client.handler.handleLifeEvent(ClientLifeEvent( + kind: ConnectedToServer, + ), client) + var authenticated = newFuture[void]("relay.client.dial.authenticated") + client.done = client.loop(authenticated) + let fut = client.keepAliveLoop() + client.tasks.add(fut) + await authenticated + +proc connect*(client: RelayClient, pubkey: PublicKey) {.async, raises: [RelayNotConnected].} = + ## Initiate a connection through the relay to the given public key + await client.ws.send(RelayCommand( + kind: Connect, + conn_pubkey: pubkey, + )) + +proc sendData*(client: RelayClient, dest_pubkey: PublicKey, data: string) {.async, raises: [RelayNotConnected].} = + ## Send data to a connection through the relay + await client.ws.send(RelayCommand( + kind: SendData, + send_data: data, + dest_pubkey: dest_pubkey, + )) + +proc disconnect*(client: RelayClient) {.async.} = + ## Disconnect this client from the network + if not client.done.isNil: + await client.done.cancelAndWait() + if client.wsopt.isSome: + await client.wsopt.get().close() + client.wsopt = none[WSSession]() + for task in client.tasks: + await task.cancelAndWait() + +proc disconnect*(client: RelayClient, dest_pubkey: PublicKey) {.async.} = + ## Disconnect this client from a remote client + if client.wsopt.isSome: + await client.wsopt.get().send(RelayCommand( + kind: Disconnect, + dcon_pubkey: dest_pubkey, + )) + \ No newline at end of file diff --git a/src/bucketsrelay/v1/common.nim b/src/bucketsrelay/v1/common.nim new file mode 100644 index 0000000..0b75e5d --- /dev/null +++ b/src/bucketsrelay/v1/common.nim @@ -0,0 +1,59 @@ +import std/macros +import std/strformat +import std/random; export random + +import chronicles + +import libsodium/sodium +import libsodium/sodium_sizes + +const + singleusermode* = defined(relaysingleusermode) + multiusermode* = not singleusermode + relayverbose* = defined(relayverbose) + +template nextDebugName*(): untyped = + $rand(100000..999999) + +template vlog*(x: varargs[string, `$`]): untyped = + when relayverbose: + debug x + +proc hash_password*(password: string): string = + # We use a lower memlimit because a stolen password is + # easy to mitigate and doesn't cause immediate harm to users. + let memlimit = max(crypto_pwhash_memlimit_min(), 10_000_000) + crypto_pwhash_str(password, memlimit = memlimit) + +proc verify_password*(pwhash: string, password: string): bool {.inline.} = + crypto_pwhash_str_verify(pwhash, password) + +macro multiuseronly*(fn: untyped): untyped = + ## Add as a pragma to procs that are only available in multiusermode + when multiusermode: + fn + else: + newStmtList() + +macro singleuseronly*(fn: untyped): untyped = + ## Add as a pragma to procs that should only be available in singleusermode + when singleusermode: + fn + else: + newStmtList() + +#------------------------------------------------------------ +# Memory-checking helpers +#------------------------------------------------------------ +var lastMem = getOccupiedMem() + +proc checkmem*(name: string) = + let newMem = getOccupiedMem() + let diffMem = newMem - lastMem + debug "checkmem", res = &"{diffMem:>10} = {newMem:>10} <- {lastMem:>10} {name}" + lastMem = newMem + +template checkMemGrowth(name: string, body: untyped): untyped = + let occ {.genSym.} = getOccupiedMem() + body + echo "Mem growth during: " & name & " " & $(getOccupiedMem() - occ) diff --git a/src/bucketsrelay/v1/dbschema.nim b/src/bucketsrelay/v1/dbschema.nim new file mode 100644 index 0000000..a9c92d6 --- /dev/null +++ b/src/bucketsrelay/v1/dbschema.nim @@ -0,0 +1,41 @@ +import std/strformat +import std/strutils +import std/logging + +import ndb/sqlite + +type + Patch* = tuple + name: string + sqls: seq[string] + +proc upgradeSchema*(db:DbConn, patches:openArray[Patch]) = + ## Apply database patches to this file + # See what patches have already been applied + db.exec(sql""" + CREATE TABLE IF NOT EXISTS _schema_version ( + id INTEGER PRIMARY KEY, + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + name TEXT UNIQUE + )""") + var applied:seq[string] + for row in db.getAllRows(sql"SELECT name FROM _schema_version"): + applied.add(row[0].s) + if applied.len > 0: + logging.debug &"(dbpatch) existing patches: {applied}" + + # Apply patches + for patch in patches: + if patch.name in applied: + continue + logging.info &"(dbpatch) applying patch: {patch.name}" + db.exec(sql"BEGIN") + try: + for statement in patch.sqls: + db.exec(sql(statement)) + db.exec(sql"INSERT INTO _schema_version (name) VALUES (?)", patch.name) + db.exec(sql"COMMIT") + except: + logging.error &"(dbpatch) error applying patch {patch.name}: {getCurrentExceptionMsg()}" + db.exec(sql"ROLLBACK") + raise diff --git a/src/bucketsrelay/v1/httpreq.nim b/src/bucketsrelay/v1/httpreq.nim new file mode 100644 index 0000000..68be8b3 --- /dev/null +++ b/src/bucketsrelay/v1/httpreq.nim @@ -0,0 +1,88 @@ +## HTTP client that does SSL/TLS with BearSSL (so you don't need `-d:ssl`) +## +import std/strutils +import std/options +import std/uri + +import chronos +import chronos/apps/http/httpclient +import chronos/apps/http/httpcommon; export httpcommon +import chronos/apps/http/httptable; export httptable + +export waitFor + +type + HttpResponse* = tuple + code: int + body: string + + +proc fetch*(session: HttpSessionRef, req: HttpClientRequestRef): Future[HttpResponseTuple] {.async.} = + ## Copied from nim-chronos + var + request = req + response: HttpClientResponseRef = nil + redirect: HttpClientRequestRef = nil + + while true: + try: + response = await request.send() + if response.status >= 300 and response.status < 400: + redirect = + block: + if "location" in response.headers: + let location = response.headers.getString("location") + if len(location) > 0: + let res = request.redirect(parseUri(location)) + if res.isErr(): + raiseHttpRedirectError(res.error()) + res.get() + else: + raiseHttpRedirectError("Location header with an empty value") + else: + raiseHttpRedirectError("Location header missing") + discard await response.consumeBody() + await response.closeWait() + response = nil + await request.closeWait() + request = nil + request = redirect + request.headers.set(HostHeader, request.address.hostname) + redirect = nil + else: + let data = await response.getBodyBytes() + let code = response.status + await response.closeWait() + response = nil + await request.closeWait() + request = nil + return (code, data) + except CancelledError as exc: + if not(isNil(response)): await closeWait(response) + if not(isNil(request)): await closeWait(request) + if not(isNil(redirect)): await closeWait(redirect) + raise exc + except HttpError as exc: + if not(isNil(response)): await closeWait(response) + if not(isNil(request)): await closeWait(request) + if not(isNil(redirect)): await closeWait(redirect) + raise exc + +proc request*(url: string, meth: HttpMethod, body = "", headers = HttpTable.init()): Future[HttpResponse] {.async.} = + ## High level request + var session = HttpSessionRef.new() + let address = session.getAddress(url).tryGet() + var req = HttpClientRequestRef.new(session, address, meth, + body = body.toOpenArrayByte(0, body.len-1), + headers = headers.toList()) + let (code, bytes) = await session.fetch(req) + return (code, bytes.bytesToString) + +when isMainModule: + import std/os + import std/strformat + let url = paramStr(1) + echo &"requesting {url}" + let resp = waitFor request(url, MethodGet) + echo "resp: ", resp[0] + echo resp[1] diff --git a/src/bucketsrelay/v1/jwtrsaonly.nim b/src/bucketsrelay/v1/jwtrsaonly.nim new file mode 100644 index 0000000..dbfef21 --- /dev/null +++ b/src/bucketsrelay/v1/jwtrsaonly.nim @@ -0,0 +1,253 @@ +# This code comes from https://github.com/yglukhov/nim-jwt +# but with modifications to work with the version of BearSSL +# included with this project and only support RSA256 JWTs. + +# The MIT License (MIT) + +# Copyright (c) 2017 Yuriy Glukhov + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import std/base64 +import std/json +import std/strutils + +import bearssl +import bearssl_pkey_decoder + +#-------------------------------------- +# jwt/private/utils +#-------------------------------------- + +proc encodeUrlSafe(s: openarray[byte]): string = + when NimMajor >= 1 and (NimMinor >= 1 or NimPatch >= 2): + result = base64.encode(s) + else: + result = base64.encode(s, newLine="") + while result.endsWith("="): + result.setLen(result.len - 1) + result = result.replace('+', '-').replace('/', '_') + +proc encodeUrlSafe(s: openarray[char]): string {.inline.} = + encodeUrlSafe(s.toOpenArrayByte(s.low, s.high)) + +proc decodeUrlSafeAsString(s: string): string = + var s = s.replace('-', '+').replace('_', '/') + while s.len mod 4 > 0: + s &= "=" + base64.decode(s) + +proc decodeUrlSafe(s: string): seq[byte] = + cast[seq[byte]](decodeUrlSafeAsString(s)) + +#-------------------------------------- +# jwt/private/jose +#-------------------------------------- + +proc toBase64(j: JsonNode): string = + encodeUrlSafe($j) + +#-------------------------------------- +# jwt/crypto +#-------------------------------------- + +# This pragma should be the same as in nim-bearssl/decls.nim +{.pragma: bearSslFunc, cdecl, gcsafe, noSideEffect, raises: [].} + +#-------------------------------------- +# Custom PEM-decoding +#-------------------------------------- + +proc invalidPemKey() = + raise newException(ValueError, "Invalid PEM encoding") + +proc pemDecoderLoop(pem: string, prc: proc(ctx: pointer, pbytes: pointer, nbytes: uint) {.bearSslFunc.}, ctx: pointer) = + var pemCtx: PemDecoderContext + pemDecoderInit(pemCtx) + var length = len(pem) + var offset = 0 + var inobj = false + while length > 0: + var tlen = pemDecoderPush(pemCtx, + unsafeAddr pem[offset], length.uint).int + offset = offset + tlen + length = length - tlen + + let event = pemDecoderEvent(pemCtx) + if event == PEM_BEGIN_OBJ: + inobj = true + pemDecoderSetdest(pemCtx, prc, ctx) + elif event == PEM_END_OBJ: + if inobj: + inobj = false + else: + break + elif event == 0 and length == 0: + break + else: + invalidPemKey() + +proc decodeFromPem(skCtx: var SkeyDecoderContext, pem: string) = + skeyDecoderInit(skCtx) + pemDecoderLoop(pem, cast[proc(ctx: pointer, pbytes: pointer, nbytes: uint) {.bearSslFunc.}](skeyDecoderPush), addr skCtx) + if skeyDecoderLastError(skCtx) != 0: invalidPemKey() + +proc decodeFromPem(pkCtx: var PkeyDecoderContext, pem: string) = + pkeyDecoderInit(addr pkCtx) + pemDecoderLoop(pem, cast[proc(ctx: pointer, pbytes: pointer, nbytes: uint) {.bearSslFunc.}](pkeyDecoderPush), addr pkCtx) + if pkeyDecoderLastError(addr pkCtx) != 0: invalidPemKey() + +proc calcHash(alg: ptr HashClass, data: string, output: var array[64, byte]) = + var ctx: array[512, byte] + let pCtx = cast[ptr ptr HashClass](addr ctx[0]) + assert(alg.contextSize <= sizeof(ctx).uint) + alg.init(pCtx) + if data.len > 0: + alg.update(pCtx, unsafeAddr data[0], data.len.uint) + alg.`out`(pCtx, addr output[0]) + +proc bearSignRSPem(data, key: string, alg: ptr HashClass, hashOid: cstring, hashLen: int): seq[byte] = + # Step 1. Extract RSA key from `key` in PEM format + var skCtx: SkeyDecoderContext + decodeFromPem(skCtx, key) + if skeyDecoderKeyType(skCtx) != KEYTYPE_RSA: + invalidPemKey() + + template privateKey(): RsaPrivateKey = skCtx.key.rsa + + # Step 2. Hash! + var digest: array[64, byte] + calcHash(alg, data, digest) + + let sigLen = (privateKey.nBitlen + 7) div 8 + result = newSeqUninitialized[byte](sigLen) + let s = rsaPkcs1SignGetDefault() + assert(not s.isNil) + if s(cast[ptr byte](hashOid), addr digest[0], hashLen.uint, addr privateKey, addr result[0]) != 1: + raise newException(ValueError, "Could not sign") + +proc bearVerifyRSPem(data, key: string, sig: openarray[byte], alg: ptr HashClass, hashOid: cstring, hashLen: int): bool = + # Step 1. Extract RSA key from `key` in PEM format + var pkCtx: PkeyDecoderContext + decodeFromPem(pkCtx, key) + if pkeyDecoderKeyType(addr pkCtx) != KEYTYPE_RSA: + invalidPemKey() + template publicKey(): RsaPublicKey = pkCtx.key.rsa + + var digest: array[64, byte] + calcHash(alg, data, digest) + + let s = rsaPkcs1VrfyGetDefault() + var digest2: array[64, byte] + + if s(unsafeAddr sig[0], sig.len.uint, cast[ptr byte](hashOid), hashLen.uint, addr publicKey, addr digest2[0]) != 1: + return false + + digest == digest2 + + +#-------------------------------------- +# jwt main +#-------------------------------------- + +type + InvalidToken* = object of ValueError + + JWT* = object + headerB64: string + claimsB64: string + header*: JsonNode + claims*: JsonNode + signature*: seq[byte] + + +proc splitToken(s: string): seq[string] = + let parts = s.split(".") + if parts.len != 3: + raise newException(InvalidToken, "Invalid token") + result = parts + +proc initJWT*(header: JsonNode, claims: JsonNode, signature: seq[byte] = @[]): JWT = + JWT( + headerB64: header.toBase64, + claimsB64: claims.toBase64, + header: header, + claims: claims, + signature: signature + ) + +# Load up a b64url string to JWT +proc toJWT*(s: string): JWT = + var parts = splitToken(s) + let + headerB64 = parts[0] + claimsB64 = parts[1] + headerJson = parseJson(decodeUrlSafeAsString(headerB64)) + claimsJson = parseJson(decodeUrlSafeAsString(claimsB64)) + signature = decodeUrlSafe(parts[2]) + + JWT( + headerB64: headerB64, + claimsB64: claimsB64, + header: headerJson, + claims: claimsJson, + signature: signature + ) + +proc toJWT*(node: JsonNode): JWT = + initJWT(node["header"], node["claims"]) + +# Encodes the raw signature to b64url +proc signatureToB64(token: JWT): string = + assert token.signature.len != 0 + result = encodeUrlSafe(token.signature) + +proc loaded(token: JWT): string = + token.headerB64 & "." & token.claimsB64 + +proc parsed(token: JWT): string = + result = token.header.toBase64 & "." & token.claims.toBase64 + +# Signs a string with a secret +proc signString(toSign: string, secret: string): seq[byte] = + template rsSign(hc, oid: typed, hashLen: int): seq[byte] = + bearSignRSPem(toSign, secret, addr hc, oid, hashLen) + return rsSign(sha256Vtable, HASH_OID_SHA256, sha256SIZE) + +# Verify that the token is not tampered with +proc verifySignature(data: string, signature: seq[byte], secret: string): bool = + result = bearVerifyRSPem(data, secret, signature, addr sha256Vtable, HASH_OID_SHA256, sha256SIZE) + +proc sign*(token: var JWT, secret: string) = + assert token.signature.len == 0 + token.signature = signString(token.parsed, secret) + +# Verify a token typically an incoming request +proc verify*(token: JWT, secret: string): bool = + verifySignature(token.loaded, token.signature, secret) + +proc toString(token: JWT): string = + token.header.toBase64 & "." & token.claims.toBase64 & "." & token.signatureToB64 + +proc `$`*(token: JWT): string = + token.toString + +proc `%`*(token: JWT): JsonNode = + let s = $token + %s diff --git a/src/bucketsrelay/v1/licenses.nim b/src/bucketsrelay/v1/licenses.nim new file mode 100644 index 0000000..452849c --- /dev/null +++ b/src/bucketsrelay/v1/licenses.nim @@ -0,0 +1,80 @@ +import std/json +import std/strformat +import std/strutils +import std/times + +import ./jwtrsaonly + +proc formatForEmail*(x: string): string = + ## Format a base64-encoded string nicely for email delivery + for i,c in x: + result.add(c) + if (i+1) mod 40 == 0: + result.add "\n" + elif (i+1) mod 10 == 0: + result.add " " + if result[^1] != '\n': + result.add "\n" + +#------------------------------------------------------ +# V1 RSA License +#------------------------------------------------------ +const + rsaPrefix = "-----BEGIN RSA PRIVATE KEY-----" + rsaSuffix = "-----END RSA PRIVATE KEY-----" + licensePrefix = "------------- START LICENSE ---------------" + licenseSuffix = "------------- END LICENSE -----------------" + +type + BucketsV1License* = distinct string + +proc unformatLicense*(x: string): BucketsV1License = + var tmp = x.replace(licensePrefix, "").replace(licenseSuffix, "") + var res: string + for c in tmp: + case c + of 'a'..'z','A'..'Z','0'..'9','+','=','_','-','/','.': + res.add c + else: + discard + return res.BucketsV1License + +proc createV1License*(privateKey: string, email: string): BucketsV1License = + ## Generate a new license + var privateKey = privateKey.replace(rsaPrefix, "") + privateKey = privateKey.replace(rsaSuffix, "") + privateKey = privateKey.strip().replace(" ", "\n") + privateKey = &"{rsaPrefix}\n{privateKey}\n{rsaSuffix}" + var token = toJWT( %* { + "header": { + "alg": "RS256", + "typ": "JWT" + }, + "claims": { + "email": email, + "iat": getTime().toUnix(), + } + }) + token.sign(privateKey) + return ($token).BucketsV1License + +proc `$`*(license: BucketsV1License): string = + ## Format a license for delivery in email + result.add licensePrefix & "\n" + result.add license.string.formatForEmail() + result.add licenseSuffix + +proc verify*(license: BucketsV1License, pubkey: string): bool = + ## Return true if the license is valid, raise an exception if not + result = false + let jwtToken = license.string.toJWT() + result = jwtToken.verify(pubkey) + +proc extractEmail*(license: BucketsV1License): string = + ## Extract the email address this license was issued to + try: + let jwt = license.string.toJWT() + return $jwt.claims["email"].getStr() + except: + discard + diff --git a/src/bucketsrelay/v1/mailer.nim b/src/bucketsrelay/v1/mailer.nim new file mode 100644 index 0000000..5fab560 --- /dev/null +++ b/src/bucketsrelay/v1/mailer.nim @@ -0,0 +1,56 @@ +import std/logging +import std/os +import std/strformat +import std/strutils + +import chronos + +import ./common + +const usepostmark = multiusermode and not defined(nopostmark) +const fromEmail {.strdefine.} = "relay@budgetwithbuckets.com" +when usepostmark: + const POSTMARK_API_KEY {.strdefine.} = "env:POSTMARK_API_KEY" + + import std/json + import ./httpreq + + +proc valueRef(location: string): string = + ## Get a value from the given location. `location` is a string + ## prefixed with one of the following, which determines where + ## the value comes from: + runnableExamples: + assert getValue("env:FOO") == getEnv("FOO") + assert getValue("embed:someval") == "someval" + if location.startsWith("env:"): + getEnv(location.substr("env:".len)) + elif location.startsWith("embed:"): + location.substr("embed:".len) + else: + raise ValueError.newException("Unknown variable ref type") + +proc sendEmail*(toEmail, subject, text: string) {.async, raises: [CatchableError].} = + when usepostmark: + let data = $(%* { + "From": fromEmail, + "To": toEmail, + "Subject": subject, + "MessageStream": "outbound", + "TextBody": text, + }) + var headers = HttpTable.init() + headers.add("Accept", "application/json") + headers.add("Content-Type", "application/json") + headers.add("X-Postmark-Server-Token", POSTMARK_API_KEY.valueRef) + let (code, res) = await request("https://api.postmarkapp.com/email", MethodPost, data, headers = headers) + if code != 200: + try: + error "Error sending email: " & $res + except: + discard + raise CatchableError.newException("Email sending failed") + else: + # logging only + info "EMAIL FAKE SENDER:\nFrom: " & fromEmail & "\nTo: " & toEmail & "\nSubject: " & subject & "\n\n" & text & "\n------------------------------------" + stderr.flushFile() diff --git a/src/bucketsrelay/v1/netstring.nim b/src/bucketsrelay/v1/netstring.nim new file mode 100644 index 0000000..a12b1f5 --- /dev/null +++ b/src/bucketsrelay/v1/netstring.nim @@ -0,0 +1,116 @@ +import std/deques +import std/strformat +import std/strutils + +type + NSDecoderState = enum + LookingForNumber, + ReadingData, + LookingForComma, + NetstringDecoder* = object + buf: string + expectedLen: int + state: NSDecoderState + maxlen: int + output: Deque[string] + terminalChar*: char + +const + COLONCHAR = ':' + TERMINALCHAR = ',' + DEFAULTMAXLEN = 1_000_000 + +proc nsencode*(msg:string, terminalChar = TERMINALCHAR):string {.inline.} = + $msg.len & COLONCHAR & msg & terminalChar + +proc newNetstringDecoder*(terminalChar = TERMINALCHAR):NetstringDecoder = + result.output = initDeque[string]() + result.terminalChar = terminalChar + result.maxlen = DEFAULTMAXLEN + +when defined(testmode): + proc reset*(p: var NetstringDecoder) = + ## Reset the parser. For testing only. + p.buf = "" + p.expectedLen = 0 + p.state = LookingForNumber + +proc `maxlen=`*(p: var NetstringDecoder, length:int) = + ## Set the maximum message length + p.maxlen = length + +proc `len`*(p: var NetstringDecoder):int = + p.output.len + +proc consume*(p: var NetstringDecoder, data:string) = + ## Send some netstring data (perhaps incomplete as yet) + var cursor:int = 0 + while cursor < data.len: + case p.state: + of LookingForNumber: + let ch = data[cursor] + cursor.inc() + case ch + of '0'..'9': + p.buf.add(ch) + if p.buf.len == 2 and p.buf[0] == '0': + raise newException(ValueError, &"Length may not start with 0") + if p.maxlen != 0: + if p.buf.parseInt() > p.maxlen: + raise newException(ValueError, &"Message too long") + of COLONCHAR: + p.expectedLen = p.buf.parseInt() + p.buf = "" + if p.expectedLen == 0: + p.state = LookingForComma + else: + p.state = ReadingData + else: + raise newException(ValueError, &"Invalid netstring length char: {ch.repr}") + + of ReadingData: + let toread = p.expectedLen - int(p.buf.len) + var sidx = cursor + var eidx = sidx + toread - 1 + if eidx >= int(data.len): + eidx = int(data.len-1) + let snippet = data[sidx..eidx] + + p.buf.add(snippet) + cursor += toread + if int(p.buf.len) == p.expectedLen: + # message possibly complete + p.state = LookingForComma + of LookingForComma: + let ch = data[cursor] + cursor.inc() + if ch == p.terminalChar: + # message complete! + # Is this a copy? I'd rather it be a move + let msg = p.buf + p.buf = "" + p.output.addLast(msg) + p.state = LookingForNumber + else: + raise newException(ValueError, &"Missing terminal comma") + +proc bytesToRead*(p: var NetstringDecoder): int = + ## Return how many bytes the decoder needs to read + case p.state + of LookingForNumber: + return 1 + of LookingForComma: + return 1 + of ReadingData: + return p.expectedLen - int(p.buf.len) + +proc hasMessage*(p: var NetstringDecoder): bool = + return p.output.len > 0 + +proc nextMessage*(p: var NetstringDecoder): string = + ## Get the next decoded message + if p.output.len > 0: + p.output.popFirst() + else: + raise newException(IndexError, &"No message available") + diff --git a/src/bucketsrelay/v1/proto.nim b/src/bucketsrelay/v1/proto.nim new file mode 100644 index 0000000..89cdaf6 --- /dev/null +++ b/src/bucketsrelay/v1/proto.nim @@ -0,0 +1,403 @@ +# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +import std/base64 +import std/hashes +import std/logging +import std/options +import std/sets; export sets +import std/strformat +import std/strutils +import std/tables + +import ./common + +import libsodium/sodium +import ndb/sqlite + +template TODO*(msg: string) = + when defined(release): + {.error: msg .} + +type + PublicKey* = distinct string + SecretKey* = distinct string + + KeyPair* = tuple + pk: PublicKey + sk: SecretKey + + ## Relay event types + EventKind* = enum + Who = "?" + Authenticated = "+" + Connected = "c" + Disconnected = "x" + Data = "d" + Entered = ">" + Exited = "^" + ErrorEvent = "E" + + ## RelayEvent error types + ErrorCode* = enum + Generic = 0 + DestNotPresent + + ## Relay events -- server to client message + RelayEvent* = object + case kind*: EventKind + of Who: + who_challenge*: string + of Authenticated: + discard + of Connected: + conn_pubkey*: PublicKey + of Disconnected: + dcon_pubkey*: PublicKey + of Data: + data*: string + sender_pubkey*: PublicKey + of Entered: + entered_pubkey*: PublicKey + of Exited: + exited_pubkey*: PublicKey + of ErrorEvent: + err_code*: ErrorCode + err_message*: string + + ## Relay command types + CommandKind* = enum + Iam = "i" + Connect = "c" + Disconnect = "x" + SendData = "d" + + ## Relay command - client to server message + RelayCommand* = object + case kind*: CommandKind + of Iam: + iam_signature*: string + iam_pubkey*: PublicKey + of Connect: + conn_pubkey*: PublicKey + of Disconnect: + dcon_pubkey*: PublicKey + of SendData: + send_data*: string + dest_pubkey*: PublicKey + + RelayConnection*[T] = ref object + challenge: string + pubkey*: PublicKey + channel*: string + peer_connections: HashSet[PublicKey] + sender*: T + + Relay*[T] = ref object + conns: TableRef[PublicKey, RelayConnection[T]] + channels: TableRef[string, HashSet[PublicKey]] + conn_requests: TableRef[PublicKey, HashSet[PublicKey]] + db: DbConn + + RelayErr* = object of CatchableError + +proc newRelay*[T](): Relay[T] = + new(result) + result.conns = newTable[PublicKey, RelayConnection[T]]() + result.channels = newTable[string, HashSet[PublicKey]]() + result.conn_requests = newTable[PublicKey, HashSet[PublicKey]]() + +proc `$`*(a: PublicKey): string = + a.string.encode() + +proc abbr*(s: string, size = 6): string = + if s.len > size: + result.add s.substr(0, size) & "..." + else: + result.add(s) + +proc abbr*(a: PublicKey): string = + a.string.encode().abbr + +proc `$`*(conn: RelayConnection): string = + result.add "[RConn " + if conn.pubkey.string == "": + result.add "----------" + else: + result.add conn.pubkey.abbr + result.add "]" + +proc `==`*(a, b: PublicKey): bool {.borrow.} + +proc hash*(p: PublicKey): Hash {.borrow.} + +proc `==`*(a, b: RelayEvent): bool = + if a.kind != b.kind: + return false + else: + case a.kind + of Who: + return a.who_challenge == b.who_challenge + of Authenticated: + return true + of Connected: + return a.conn_pubkey == b.conn_pubkey + of Disconnected: + return a.dcon_pubkey == b.dcon_pubkey + of Data: + return a.sender_pubkey == b.sender_pubkey and a.data == b.data + of Entered: + return a.entered_pubkey == b.entered_pubkey + of Exited: + return a.exited_pubkey == b.exited_pubkey + of ErrorEvent: + return a.err_message == b.err_message + +proc `==`*(a, b: RelayCommand): bool = + if a.kind != b.kind: + return false + else: + case a.kind: + of Iam: + return a.iam_signature == b.iam_signature and a.iam_pubkey == b.iam_pubkey + of Connect: + return a.conn_pubkey == b.conn_pubkey + of Disconnect: + return a.dcon_pubkey == b.dcon_pubkey + of SendData: + return a.send_data == b.send_data and a.dest_pubkey == b.dest_pubkey + +proc `$`*(ev: RelayEvent): string = + result.add "(" + case ev.kind + of Who: + result.add "Who challenge=" & ev.who_challenge.encode().abbr + of Authenticated: + result.add "Authenticated" + of Connected: + result.add "Connected " & ev.conn_pubkey.abbr + of Disconnected: + result.add "Disconnected " & ev.dcon_pubkey.abbr + of Data: + result.add "Data " & ev.sender_pubkey.abbr & " data=" & $ev.data.len + of Entered: + result.add "Entered " & ev.entered_pubkey.abbr + of Exited: + result.add "Exited " & ev.exited_pubkey.abbr + of ErrorEvent: + result.add "Error " & ev.err_message + result.add ")" + +template dbg*(ev: RelayEvent): string = $ev + +proc `$`*(cmd: RelayCommand): string = + result.add "(" + case cmd.kind + of Iam: + result.add &"Iam {cmd.iam_pubkey.abbr} sig={cmd.iam_signature.encode.abbr}" + of Connect: + result.add &"Connect {cmd.conn_pubkey.abbr}" + of Disconnect: + result.add &"Disconnect {cmd.dcon_pubkey.abbr}" + of SendData: + result.add &"SendData {cmd.dest_pubkey.abbr} data={cmd.send_data.len}" + result.add ")" + +template dbg*(cmd: RelayCommand): string = $cmd + +when defined(testmode): + # proc dump*(relay: Relay): string = + # for row in relay.db.getAllRows(sql"SELECT * FROM clients"): + # result.add $row & "\l" + # for row in relay.db.getAllRows(sql"SELECT * FROM pending_conns"): + # result.add $row & "\l" + + proc testmode_conns*[T](relay: Relay[T]): TableRef[PublicKey, RelayConnection[T]] = + relay.conns + + proc testmode_conns*(conn: RelayConnection): HashSet[PublicKey] = + conn.peer_connections + +proc newRelayConnection*[T](sender: T): RelayConnection[T] = + new(result) + result.sender = sender + result.peer_connections = initHashSet[PublicKey]() + +template sendEvent(conn: RelayConnection, ev: RelayEvent) = + case ev.kind + of Data: + when relayverbose: + debug $conn & "< " & ev.dbg + else: + discard + else: + debug $conn & "< " & ev.dbg + conn.sender.sendEvent(ev) + +template sendError(conn: RelayConnection, message: string) = + debug $conn & "< error: " & message + conn.sender.sendEvent(RelayEvent( + kind: ErrorEvent, + err_message: message, + )) + +proc initAuth*[T](relay: var Relay[T], client: T, channel = ""): RelayConnection[T] = + ## Ask the client to authenticate itself. After it succeeds, it will + ## be added as a connected client. + ## If channel is provided, this is the channel to which this client + ## will be subscribed for Entered/Exited events. + var conn = newRelayConnection[T](client) + conn.challenge = randombytes(32) + conn.channel = channel + conn.sendEvent(RelayEvent( + kind: Who, + who_challenge: conn.challenge, + )) + return conn + +proc connectPair[T](a, b: var RelayConnection[T]) = + ## Connect two clients together + a.peer_connections.incl(b.pubkey) + b.peer_connections.incl(a.pubkey) + a.sendEvent(RelayEvent(kind: Connected, conn_pubkey: b.pubkey)) + b.sendEvent(RelayEvent(kind: Connected, conn_pubkey: a.pubkey)) + +proc addConnRequest(relay: var Relay, alice_pubkey: PublicKey, bob_pubkey: PublicKey) = + ## Add or fulfil a connection request from alice to bob + var alice = relay.conns[alice_pubkey] + relay.conn_requests.mgetOrPut(alice_pubkey, initHashSet[PublicKey]()).incl(bob_pubkey) + if bob_pubkey in alice.peer_connections: + # They're already connected + return + var bob_requests = relay.conn_requests.getOrDefault(bob_pubkey, initHashSet[PublicKey]()) + if alice_pubkey in bob_requests: + # They both want to connect! + var bob = relay.conns[bob_pubkey] + connectPair(alice, bob) + +proc removeConnRequest(relay: var Relay, alice_pubkey: PublicKey, bob_pubkey: PublicKey) = + ## Remove a connection request from alice to bob + relay.conn_requests.mgetOrPut(alice_pubkey, initHashSet[PublicKey]()).excl(bob_pubkey) + +proc removeConnection*[T](relay: var Relay[T], conn: RelayConnection[T]) = + ## Remove a conn from the relay if it exists. + if conn.pubkey in relay.conn_requests: + relay.conn_requests.del(conn.pubkey) + # disconnect all peer connections + var commands: seq[RelayCommand] + for other_pubkey in conn.peer_connections: + commands.add(RelayCommand( + kind: Disconnect, + dcon_pubkey: other_pubkey, + )) + for command in commands: + relay.handleCommand(conn, command) + # notify the channel (if any) + if conn.channel != "": + relay.channels.mgetOrPut(conn.channel, initHashSet[PublicKey]()).excl(conn.pubkey) + for other in relay.channels[conn.channel].items: + if other in relay.conns: + relay.conns[other].sendEvent(RelayEvent( + kind: Exited, + exited_pubkey: conn.pubkey, + )) + # remove it from the registry + if conn.pubkey in relay.conns: + relay.conns.del(conn.pubkey) + debug &"{conn} gone" + +proc handleCommand*[T](relay: var Relay[T], conn: RelayConnection[T], command: RelayCommand) = + case command.kind + of SendData: + when defined(verbose): + debug &"{conn} > {command.dbg}" + else: + discard + else: + debug &"{conn} > {command.dbg}" + case command.kind + of Iam: + if conn.challenge == "": + conn.sendError "Authentication cannot proceed. Reconnect and try again." + try: + crypto_sign_verify_detached(command.iam_pubkey.string, conn.challenge, command.iam_signature) + except: + conn.challenge = "" # disable authentication + conn.sendError "Invalid signature" + return + conn.pubkey = command.iam_pubkey + if conn.pubkey in relay.conns: + # this pubkey is already connected; boot the old conn + relay.removeConnection(relay.conns[conn.pubkey]) + relay.conns[conn.pubkey] = conn + conn.sendEvent(RelayEvent( + kind: Authenticated, + )) + if conn.channel != "": + relay.channels.mgetOrPut(conn.channel, initHashSet[PublicKey]()).incl(conn.pubkey) + for other in relay.channels[conn.channel].items: + if other != conn.pubkey: + conn.sendEvent(RelayEvent( + kind: Entered, + entered_pubkey: other, + )) + if other in relay.conns: + relay.conns[other].sendEvent(RelayEvent( + kind: Entered, + entered_pubkey: conn.pubkey, + )) + of Connect: + if conn.pubkey.string == "": + conn.sendError "Connection forbidden" + elif command.conn_pubkey.string == conn.pubkey.string: + conn.sendError "Can't connect to self" + else: + relay.addConnRequest(conn.pubkey, command.conn_pubkey) + of Disconnect: + relay.removeConnRequest(conn.pubkey, command.dcon_pubkey) + if command.dcon_pubkey in conn.peer_connections: + if command.dcon_pubkey in relay.conns: + var other = relay.conns[command.dcon_pubkey] + # disassociate + other.peer_connections.excl(conn.pubkey) + conn.peer_connections.excl(other.pubkey) + # notify + conn.sendEvent(RelayEvent( + kind: Disconnected, + dcon_pubkey: other.pubkey, + )) + other.sendEvent(RelayEvent( + kind: Disconnected, + dcon_pubkey: conn.pubkey, + )) + of SendData: + if conn.pubkey.string == "": + conn.sendError "Sending forbidden" + elif command.dest_pubkey notin conn.peer_connections: + conn.sendError "No such connection" + else: + if command.dest_pubkey notin relay.conns: + conn.sendEvent(RelayEvent( + kind: ErrorEvent, + err_message: "Other side disconnected", + )) + else: + let remote = relay.conns[command.dest_pubkey] + remote.sendEvent(RelayEvent( + kind: Data, + sender_pubkey: conn.pubkey, + data: command.send_data, + )) + +#------------------------------------------------------------ +# utilities +#------------------------------------------------------------ +proc genkeys*(): KeyPair = + let (pk, sk) = crypto_sign_keypair() + result = (pk.PublicKey, sk.SecretKey) + +proc sign*(key: SecretKey, message: string): string = + ## Sign a message with the given secret key + result = crypto_sign_detached(key.string, message) diff --git a/src/bucketsrelay/v1/server.nim b/src/bucketsrelay/v1/server.nim new file mode 100644 index 0000000..35a9f9e --- /dev/null +++ b/src/bucketsrelay/v1/server.nim @@ -0,0 +1,922 @@ +# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +import std/base64 +import std/json +import std/logging +import std/mimetypes +import std/options; export options +import std/os +import std/sha1 +import std/sqlite3 +import std/strformat +import std/strutils +import std/tables + +import chronicles except debug, info, warn, error +import chronos +import httputils +import libsodium/sodium +import mustache +import ndb/sqlite +import stew/byteutils +import websock/extensions/compression/deflate +import websock/websock + +import ./common +import ./dbschema +import ./netstring +import ./proto +import ./stringproto +import ./mailer +import ./licenses + +type + WSClient = ref object + debugname*: string + ws: WSSession + user_id: int64 + ip: string + relayserver: RelayServer + eventQueue: AsyncQueue[RelayEvent] + + RelayHttpServer = ref object + case tls: bool + of true: + httpsServer: TlsHttpServer + of false: + httpServer: HttpServer + + RelayServer* = ref object + debugname*: string + nextid: int + relay: Relay[WSClient] + http: RelayHttpServer + mcontext*: proc(): mustache.Context + longrunservices: seq[Future[void]] + runningRequests: TableRef[int, HttpRequest] + when multiusermode: + pubkey*: string + dbfilename: string + updateSchema: bool + userdb: Option[DbConn] + elif singleusermode: + usernameHash: string + passwordHash: string + + NotFound* = object of CatchableError + WrongPassword* = object of CatchableError + DuplicateUser* = object of CatchableError + +const + partialsDir = currentSourcePath.parentDir.parentDir / "partials" + staticDir = currentSourcePath.parentDir.parentDir / "static" + +when multiusermode: + let + AUTH_LICENSE_PUBKEY* = getEnv("AUTH_LICENSE_PUBKEY", "") + LICENSE_HASH_SALT = getEnv("LICENSE_HASH_SALT", "yououghttochangethis") + +const versionSupport = static: + var jnode = %* { + "versions": [], + } + var authMethods = %* ["usernamepassword"] + when multiusermode: + authMethods.add newJString("v1license") + jnode["versions"].add(%* { + "version": "1", + "authMethods": authMethods, + }) + $jnode + +var mimedb = newMimetypes() + +when defined(release) or defined(embedassets): + # embed templates and static data + const partialsData = static: + var tab = initTable[string, string]() + echo "Embedding templates from ", partialsDir + for item in walkDir(partialsDir): + if item.kind == pcFile: + let + parts = item.path.splitFile + name = parts.name + echo " + ", name, ": ", item.path + tab[name] = slurp(item.path) + tab + proc addDefaultContext*(c: var Context) = + c.searchTable(partialsData) + + const staticData = static: + var tab = initTable[string, string]() + echo "Embedding static data from ", staticDir + for item in walkDir(staticDir): + if item.kind == pcFile: + let name = "/" & item.path.extractFilename + echo " + ", name, ": ", item.path + tab[name] = slurp(item.path) + tab + + template readStaticFile(path: string): string = + staticData[path] +else: + # read templates and static data from disk + proc addDefaultContext*(c: var Context) = + c.searchDirs(@[partialsDir]) + + proc readStaticFile(path: string): string = + let fullpath = normalizedPath(staticDir / path) + if fullpath.isRelativeTo(staticDir) and fullpath.fileExists(): + readFile(fullpath) + else: + raise NotFound.newException("No such file: " & path) + +template logname*(rs: RelayServer): string = + "(" & rs.debugname & ") " + +proc start*(rhs: RelayHttpServer) = + case rhs.tls + of true: + rhs.httpsServer.start() + of false: + rhs.httpServer.start() + +proc stop*(rhs: RelayHttpServer) = + case rhs.tls + of true: + rhs.httpsServer.stop() + of false: + rhs.httpServer.stop() + +proc close*(rhs: RelayHttpServer) = + case rhs.tls + of true: + rhs.httpsServer.close() + of false: + rhs.httpServer.close() + +proc join*(rhs: RelayHttpServer): Future[void] = + case rhs.tls + of true: + rhs.httpsServer.join() + of false: + rhs.httpServer.join() + +proc `handler=`*(rhs: RelayHttpServer, handler: HttpAsyncCallback) = + case rhs.tls + of true: + rhs.httpsServer.handler = handler + of false: + rhs.httpServer.handler = handler + +#------------------------------------------------------------- +# netstrings +#------------------------------------------------------------- +const + COLONCHAR = ':' + TERMINALCHAR = ',' + DEFAULTMAXLEN = 1_000_000 + +proc nsencode*(msg:string, terminalChar = TERMINALCHAR):string {.inline.} = + $msg.len & COLONCHAR & msg & terminalChar + +#------------------------------------------------------------- +# User management +#------------------------------------------------------------- +type + LowerString* = distinct string + +converter toLowercase*(s: string): LowerString = s.toLower().LowerString +converter toString*(s: LowerString): string = s.string + +const userdbSchema = [ + ("initial", @[ + """CREATE TABLE IF NOT EXISTS iplog ( + day TEXT NOT NULL, + ip TEXT NOT NULL, + bytes_sent INT DEFAULT 0, + bytes_recv INT DEFAULT 0, + PRIMARY KEY (day, ip) + )""", + """CREATE TABLE IF NOT EXISTS user ( + id INTEGER PRIMARY KEY, + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + email TEXT NOT NULL, + pwhash TEXT NOT NULL, + emailverified TINYINT DEFAULT 0, + blocked TINYINT DEFAULT 0, + recentlicensehash TEXT DEFAULT '', + UNIQUE(email) + )""", + """CREATE TABLE IF NOT EXISTS userlog ( + day TEXT NOT NULL, + user_id INTEGER, + bytes_sent INT DEFAULT 0, + bytes_recv INT DEFAULT 0, + PRIMARY KEY (day, user_id), + FOREIGN KEY (user_id) REFERENCES user(id) + )""", + """CREATE TABLE IF NOT EXISTS emailtoken ( + id INTEGER PRIMARY KEY, + expires TIMESTAMP DEFAULT (datetime('now', '+1 hour')), + user_id INTEGER NOT NULL, + token TEXT, + FOREIGN KEY (user_id) REFERENCES user(id) + )""", + """CREATE TABLE IF NOT EXISTS pwreset ( + id INTEGER PRIMARY KEY, + expires TIMESTAMP DEFAULT (datetime('now', '+1 hour')), + user_id INTEGER NOT NULL, + token TEXT, + FOREIGN KEY (user_id) REFERENCES user(id) + )""", + """CREATE TABLE IF NOT EXISTS disabledlicense ( + licensehash TEXT PRIMARY KEY + )""", + ]) +] + +template boolVal*(d: DbValue): bool = + d.i == 1 + +proc db*(rs: RelayServer): DbConn {.multiuseronly.} = + ## Get the user-data database for this server + if not rs.userdb.isSome: + var db = open(rs.dbfilename, "", "", "") + discard db.busy_timeout(1000) + db.exec(sql"PRAGMA foreign_keys = ON") + if rs.updateSchema: + db.upgradeSchema(userdbSchema) + rs.userdb = some(db) + rs.userdb.get() + +proc newRelayServer*(dbfilename: string, updateSchema = true, pubkey = ""): RelayServer {.multiuseronly.} = + ## Make a new multi-user relay server + new(result) + result.relay = newRelay[WSClient]() + result.mcontext = proc(): Context = + result = newContext() + result.addDefaultContext() + result.dbfilename = dbfilename + result.updateSchema = updateSchema + result.pubkey = pubkey + result.runningRequests = newTable[int, HttpRequest]() + discard result.db() + result.debugname = "RelayServer" & nextDebugName() + +proc stop*(rs: RelayServer) {.async.} = + for fut in rs.longrunservices: + await fut.cancelAndWait() + for req in rs.runningRequests.values(): + try: + await req.sendError(Http503) + except: + discard + +proc newRelayServer*(username, password: string): RelayServer {.singleuseronly.} = + ## Make a new single-user relay server + new(result) + result.relay = newRelay[WSClient]() + result.mcontext = proc(): Context = + result = newContext() + result.addDefaultContext() + result.usernameHash = hash_password(username) + result.passwordHash = hash_password(password) + result.runningRequests = newTable[int, HttpRequest]() + +proc get_user_id*(rs: RelayServer, email: LowerString): int64 {.multiuseronly.} = + ## Get a user's id from their email + try: + rs.db.getRow(sql"SELECT id FROM user WHERE email=?", email).get()[0].i + except: + raise NotFound.newException("No such user") + +proc register_user*(rs: RelayServer, email: LowerString, password: string): int64 {.multiuseronly.} = + ## Register a user with a password + let pwhash = try: + hash_password(password) + except: + logging.error "Error hashing password", getCurrentExceptionMsg() + raise CatchableError.newException("Crypto error") + try: + result = rs.db.insertID(sql"INSERT INTO user (email, pwhash) VALUES (?,?)", + email, pwhash) + except: + logging.error rs.logname, "failed registering", getCurrentExceptionMsg() + raise DuplicateUser.newException("Account already exists") + +proc password_auth*(rs: RelayServer, email: LowerString, password: string): int64 {.multiuseronly.} = + ## Return the userid if the password is correct, else raise an exception + let orow = rs.db.getRow(sql"SELECT id, pwhash FROM user WHERE email = ?", email) + if orow.isNone: + raise NotFound.newException("No such user") + else: + let row = orow.get() + let user_id = row[0].i + let pwhash = row[1].s + if verify_password(pwhash, password): + return user_id + raise WrongPassword.newException("Wrong password") + +proc password_auth*(rs: RelayServer, email: LowerString, password: string): int64 {.singleuseronly.} = + ## Return 1 if the password is correct, or else raise an exception + if not verify_password(rs.usernameHash, email): + raise NotFound.newException("No such user") + if verify_password(rs.passwordHash, password): + return 1 + raise WrongPassword.newException("Wrong password") + +proc strHash(lic: BucketsV1License, email: LowerString): string {.multiuseronly.} = + $secureHash( + $secureHash(LICENSE_HASH_SALT) & $secureHash(email.string) & $secureHash($lic) + ) + +proc license_auth*(rs: RelayServer, license: string): int64 {.multiuseronly.} = + ## Return the userid if the license is valid, else raise an error + if rs.pubkey == "": + raise WrongPassword.newException("License auth not supported") + let lic = license.unformatLicense() + if lic.verify(rs.pubkey) == false: + raise WrongPassword.newException("Invalid license") + let email = lic.extractEmail().toLowercase() + let lichash = strHash(lic, email) + let disabled = rs.db.getRow(sql"SELECT count(*) FROM disabledlicense WHERE licensehash=?", lichash).get()[0].i + if disabled != 0: + raise WrongPassword.newException("License disabled") + try: + result = rs.get_user_id(email) + except NotFound: + # create the user + result = rs.db.insertID(sql"INSERT INTO user (email, pwhash, emailverified) VALUES (?, '', 1)", email) + # disable former passwords + rs.db.exec(sql"UPDATE user SET pwhash='' WHERE recentlicensehash='' AND id=?", result) + # add license + rs.db.exec(sql"UPDATE user SET recentlicensehash=?, emailverified=1 WHERE id=?", lichash, result) + +proc is_email_verified*(rs: RelayServer, user_id: int64): bool {.multiuseronly.} = + ## Return true if the user has verified their email address + let row = rs.db.getRow(sql"SELECT emailverified FROM user WHERE id=?", user_id) + if row.isSome: + return row.get()[0].boolVal + +proc generate_email_verification_token*(rs: RelayServer, user_id: int64): string {.multiuseronly.} = + ## Generate a string to be emailed to a user that when returned + ## to `use_email_verification_token` will mark that user's email + ## as verified. + result = randombytes(16).toHex() + rs.db.exec(sql"INSERT INTO emailtoken (user_id, token) VALUES (?, ?)", + user_id, result) + rs.db.exec(sql"""DELETE FROM emailtoken WHERE id NOT IN + (SELECT id FROM emailtoken WHERE user_id=? ORDER BY id DESC LIMIT 3)""", + user_id) + +proc use_email_verification_token*(rs: RelayServer, user_id: int64, token: string): bool {.multiuseronly.} = + ## Verify a user's email address via token. Return `true` if they are now + ## verified and `false` if they are not. + try: + let row = rs.db.getRow(sql"SELECT count(*) FROM emailtoken WHERE user_id=? AND token=?", + user_id, token).get() + if row[0].i == 1: + rs.db.exec(sql"DELETE FROM emailtoken WHERE user_id = ?", user_id) + rs.db.exec(sql"UPDATE user SET emailverified=1 WHERE id=?", user_id) + except: + discard + return rs.is_email_verified(user_id) + +proc generate_password_reset_token*(rs: RelayServer, email: LowerString): string {.multiuseronly.} = + ## Generate a string token to be emailed to a user that can be used + ## to set their password. + result = randombytes(16).toHex() + let user_id = rs.get_user_id(email) + rs.db.exec(sql"INSERT INTO pwreset (user_id, token) VALUES (?, ?)", + user_id, result) + rs.db.exec(sql"""DELETE FROM pwreset WHERE id NOT IN + (SELECT id FROM pwreset WHERE user_id=? ORDER BY id DESC LIMIT 3)""", + user_id) + +proc delete_old_pwreset_tokens(rs: RelayServer) {.multiuseronly.} = + rs.db.exec(sql"DELETE FROM pwreset WHERE expires < datetime('now')") + +proc user_for_password_reset_token*(rs: RelayServer, token: string): Option[int64] {.multiuseronly.} = + ## Get the user associated with a password reset token, if one exists. + rs.delete_old_pwreset_tokens() + try: + let row = rs.db.getRow(sql"SELECT user_id FROM pwreset WHERE token = ?", token).get() + return some(row[0].i) + except: + discard + +proc update_password_with_token*(rs: RelayServer, token: string, newpassword: string) {.multiuseronly.} = + ## Update a user's password using a password-reset token + let o_user_id = rs.user_for_password_reset_token(token) + if o_user_id.isNone: + raise NotFound.newException("Invalid token") + let user_id = o_user_id.get() + let pwhash = hash_password(newpassword) + rs.db.exec(sql"DELETE FROM pwreset WHERE user_id = ?", user_id) + rs.db.exec(sql"UPDATE user SET pwhash=? WHERE id=?", pwhash, user_id) + +proc block_user*(rs: RelayServer, user_id: int64) {.multiuseronly.} = + ## Block a user's access to the relay + rs.db.exec(sql"UPDATE user SET blocked=1 WHERE id=?", user_id) + +proc block_user*(rs: RelayServer, email: LowerString) {.multiuseronly.} = + ## Block a user's access to the relay + rs.block_user(rs.get_user_id(email)) + +proc unblock_user*(rs: RelayServer, user_id: int64) {.multiuseronly.} = + ## Unblock a user's access to the relay + rs.db.exec(sql"UPDATE user SET blocked=0 WHERE id=?", user_id) + +proc unblock_user*(rs: RelayServer, email: LowerString) {.multiuseronly.} = + ## Unblock a user's access to the relay + rs.unblock_user(rs.get_user_id(email)) + +proc disable_most_recently_used_license*(rs: RelayServer, uid: int64) {.multiuseronly.} = + ## Block the most recently-used license for a user + let lichash = try: + rs.db.getRow(sql"SELECT recentlicensehash FROM user WHERE id=?", uid).get()[0].s + except: + raise NotFound.newException("No such user") + if lichash == "": + raise NotFound.newException("User has not authenticated via license") + rs.db.exec(sql"INSERT INTO disabledlicense (licensehash) VALUES (?) ON CONFLICT DO NOTHING", lichash) + +proc can_use_relay*(rs: RelayServer, user_id: int64): bool {.multiuseronly.} = + ## Return true if the user is allowed to use the relay + ## because their email is verified and they are not blocked + try: + return rs.db.getRow(sql"SELECT emailverified AND not(blocked) FROM user WHERE id=?", user_id).get()[0].boolVal + except: + discard + +type + DataSentRecv* = tuple + sent: int + recv: int + +proc log_user_data*(rs: RelayServer, user_id: int64, dlen: DataSentRecv) {.multiuseronly.} = + rs.db.exec(sql"""INSERT INTO userlog (day, user_id, bytes_sent, bytes_recv) + VALUES (date(), ?, ?, ?) + ON CONFLICT (day, user_id) DO + UPDATE SET + bytes_sent = bytes_sent + excluded.bytes_sent, + bytes_recv = bytes_recv + excluded.bytes_recv + """, user_id, dlen.sent, dlen.recv) + +when multiusermode: + template log_user_data_sent*(rs: RelayServer, user_id: int64, dlen: int) = + rs.log_user_data(user_id, (dlen, 0)) + + template log_user_data_recv*(rs: RelayServer, user_id: int64, dlen: int) = + rs.log_user_data(user_id, (0, dlen)) + +proc data_by_user*(rs: RelayServer, user_id: int64, days = 1): DataSentRecv {.multiuseronly.} = + let orow = rs.db.getRow(sql""" + SELECT + sum(bytes_sent), + sum(bytes_recv) + FROM + userlog + WHERE + user_id = ? + AND day >= date('now', '-' || ? || ' day') + """, user_id, days) + if orow.isSome: + let row = orow.get() + return (row[0].i.int, row[1].i.int) + +proc top_data_users*(rs: RelayServer, limit = 20, days = 7): seq[tuple[user: string, data: DataSentRecv]] {.multiuseronly.} = + let rows = rs.db.getAllRows(sql""" + SELECT + u.email, + sum(ll.bytes_sent), + sum(ll.bytes_recv), + sum(ll.bytes_sent + ll.bytes_recv) as total + FROM + userlog as ll + LEFT JOIN user AS u + ON ll.user_id = u.id + WHERE + ll.day >= date('now', '-' || ? || ' day') + GROUP BY 1 + ORDER BY total DESC + LIMIT ? + """, days, limit) + for row in rows: + result.add((row[0].s, (row[1].i.int, row[2].i.int))) + +proc log_ip_data*(rs: RelayServer, ip: string, dlen: DataSentRecv) {.multiuseronly.} = + rs.db.exec(sql"""INSERT INTO iplog (day, ip, bytes_sent, bytes_recv) + VALUES (date(), ?, ?, ?) + ON CONFLICT (day, ip) DO + UPDATE SET + bytes_sent = bytes_sent + excluded.bytes_sent, + bytes_recv = bytes_recv + excluded.bytes_recv + """, ip, dlen.sent, dlen.recv) + +when multiusermode: + template log_ip_data_sent*(rs: RelayServer, ip: string, dlen: int) = + rs.log_ip_data(ip, (dlen, 0)) + + template log_ip_data_recv*(rs: RelayServer, ip: string, dlen: int) = + rs.log_ip_data(ip, (0, dlen)) + +proc data_by_ip*(rs: RelayServer, ip: string, days = 1): DataSentRecv {.multiuseronly.} = + let orow = rs.db.getRow(sql""" + SELECT + sum(bytes_sent), + sum(bytes_recv) + FROM + iplog + WHERE + ip = ? + AND day >= date('now', '-' || ? || ' day') + """, ip, days) + if orow.isSome: + let row = orow.get() + return (row[0].i.int, row[1].i.int) + +proc top_data_ips*(rs: RelayServer, limit = 20, days = 7): seq[tuple[ip: string, data: DataSentRecv]] {.multiuseronly.} = + let rows = rs.db.getAllRows(sql""" + SELECT + ip, + sum(bytes_sent), + sum(bytes_recv), + sum(bytes_sent + bytes_recv) as total + FROM + iplog + WHERE + day >= date('now', '-' || ? || ' day') + GROUP BY 1 + ORDER BY total DESC + LIMIT ? + """, days, limit) + for row in rows: + result.add((row[0].s, (row[1].i.int, row[2].i.int))) + +proc delete_old_stats*(rs: RelayServer, keep_days = 90) {.gcsafe, multiuseronly.} = + ## Remote stats older than `keep_days` days + try: + info "Deleting stats older than " & $keep_days & "days" + except: + discard + {.gcsafe.}: + rs.db.exec(sql"DELETE FROM iplog WHERE day < date('now', '-' || ? || ' day')", keep_days) + rs.db.exec(sql"DELETE FROM userlog WHERE day < date('now', '-' || ? || ' day')", keep_days) + +proc clear_stat_loop(rs: RelayServer) {.async, multiuseronly.} = + while true: + await sleepAsync(24.hours) + rs.delete_old_stats() + +proc periodically_delete_old_stats*(rs: RelayServer) {.multiuseronly.} = + ## Delete old stats at a regular interval + rs.longrunservices.add(rs.clear_stat_loop()) + +#------------------------------------------------------------- +# Common HTTP helpers +#------------------------------------------------------------- + +proc ipAddress(request: HttpRequest): string = + ## Return the IP Address associated with this request + # # Forwarded (TODO) + # let forwarded = request.headers.getOrDefault("forwarded") + # if forwarded != "": + # return result + # True-Client-IP (cloudflare) + result = request.headers.getString("true-client-ip") + if result != "": + return result + # X-Real-IP (nginx) + result = request.headers.getString("x-real-ip") + if result != "": + return result + result = request.stream.writer.tsource.remoteAddress().host() + +proc sendHTML(req: HttpRequest, data: string) {.async.} = + var headers = HttpTable.init() + headers.add("Content-Type", "text/html") + await req.sendResponse(Http200, headers, data = data) + +#------------------------------------------------------------- +# Version 1 +#------------------------------------------------------------- + +template logname*(c: WSClient): string = + "(" & c.debugname & ") " + +proc sendEvent*(c: WSClient, ev: RelayEvent) = + ## Queue an event to a single ws client + c.eventQueue.addLastNoWait(ev) + +proc newWSClient(rs: RelayServer, ws: WSSession, user_id: int64, ip: string): WSClient = + new(result) + result.ws = ws + result.relayserver = rs + result.user_id = user_id + result.ip = ip + result.eventQueue = newAsyncQueue[RelayEvent]() + result.debugname = "WSClient" & nextDebugName() + +proc closeWait(c: WSClient) {.async.} = + c.ws = nil + +proc authenticate(rs: RelayServer, req: HttpRequest): int64 = + ## Perform HTTP basic authentication and return the + ## user id if correct. + let authorization = req.headers.getString("authorization") + let parts = authorization.strip().split(" ") + doAssert parts.len == 2, "Authorization header should have 2 items" + doAssert parts[0] == "Basic", "Only basic HTTP auth is supported" + let credentials = base64.decode(parts[1]).split(":", maxsplit = 1) + doAssert credentials.len == 2, "Must supply username and password" + let + username = credentials[0] + password = credentials[1] + try: + when multiusermode: + if rs.pubkey != "" and username == "_license": + return rs.license_auth(password) + else: + return rs.password_auth(username, password) + else: + return rs.password_auth(username, password) + except WrongPassword: + info rs.logname, "WrongPassword: " & getCurrentExceptionMsg() + raise + except: + logging.error rs.logname, "Error during authentication: " & getCurrentExceptionMsg() + raise + +when defined(testmode): + var allHttpRequests*: seq[HttpRequest] + +proc handleRequestRelayV1(rs: RelayServer, req: HttpRequest) {.async, gcsafe.} = + # Perform HTTP basic authenciation + {.gcsafe.}: + vlog rs.logname, "starting..." + let user_id = block: + try: + vlog "authenticating..." + rs.authenticate(req) + except: + logging.error "Error authenticating: " & getCurrentExceptionMsg() + await req.sendError(Http403) + return + vlog rs.logname, "auth ok" + when multiusermode: + if not rs.can_use_relay(user_id): + info rs.logname, "Blocked from relay: " & $user_id + await req.sendError(Http403) + return + let ip = req.ipAddress() + + # Upgrade protocol to websockets + var relayconn: RelayConnection[WSClient] + try: + let deflateFactory = deflateFactory() + let server = WSServer.new(factories = [deflateFactory]) + vlog rs.logname, "opening WS..." + var ws = await server.handleRequest(req) + if ws.readyState != Open: + raise ValueError.newException("Failed to open websocket connection") + + var wsclient = newWSClient(rs, ws, user_id, ip) + wsclient.debugname = rs.debugname & "." & wsclient.debugname + vlog wsclient.logname, "starting..." + try: + relayconn = rs.relay.initAuth(wsclient, channel = $user_id) + var decoder = newNetstringDecoder() + var msgfut: Future[seq[byte]] + var evfut: Future[RelayEvent] + while ws.readyState != ReadyState.Closed: + if msgfut.isNil: + msgfut = ws.recvMsg() + if evfut.isNil: + evfut = wsclient.eventQueue.get() + await (msgfut or evfut) + if evfut.finished: + let ev = await evfut + evfut = nil + let msg = nsencode(dumps(ev)) + when multiusermode: + rs.log_user_data_recv(wsclient.user_id, msg.len) + rs.log_ip_data_recv(wsclient.ip, msg.len) + await ws.send(msg.toBytes, Opcode.Binary) + if msgfut.finished: + let buff = try: + await msgfut + except: + vlog wsclient.logname, "error getting msg: ", getCurrentExceptionMsg() + break + msgfut = nil + when multiusermode: + rs.log_user_data_sent(user_id, buff.len) + rs.log_ip_data_sent(ip, buff.len) + decoder.consume(string.fromBytes(buff)) + while decoder.hasMessage(): + let cmd = loadsRelayCommand(decoder.nextMessage()) + rs.relay.handleCommand(relayconn, cmd) + finally: + await wsclient.closeWait() + except WSClosedError: + discard + except WebSocketError as exc: + error rs.logname, "WebSocketError: ", exc.msg + await req.sendError(Http400) + except Exception as exc: + error rs.logname, "connection failed: ", exc.msg + await req.sendError(Http400) + finally: + if not relayconn.isNil: + rs.relay.removeConnection(relayconn) + +proc handleRequestAuthV1(rs: RelayServer, req: HttpRequest) {.async, multiuseronly.} = + ## Handle user registration activities + # Upgrade protocol to websockets + {.gcsafe.}: + try: + vlog "[ws.auth] starting..." + let deflateFactory = deflateFactory() + let server = WSServer.new(factories = [deflateFactory]) + var ws = await server.handleRequest(req) + if ws.readyState != Open: + raise ValueError.newException("Failed to open websocket connection") + + while ws.readyState != ReadyState.Closed: + let buff = try: + await ws.recvMsg() + except: + break + let msg = string.fromBytes(buff) + let data = parseJson(msg) + var resp = newJObject() + resp["id"] = data["id"] + try: + let command = data["command"].getStr() + vlog "[ws.auth] command: " & command + let args = data["args"] + case command + of "register": + let email = args["email"].getStr() + let password = args["password"].getStr() + vlog "[ws.auth] register_user" + let user_id = rs.register_user(email, password) + vlog "[ws.auth] generate_email_verification_token" + let email_token = rs.generate_email_verification_token(user_id) + try: + vlog "[ws.auth] sendEmail" + await sendEmail(email, "Buckets Relay - Email Verification", + &"Use this code to verify your email address:\n\n{email_token}") + resp["response"] = newJBool(true) + except: + resp["error"] = newJString("Failed to send email") + of "sendVerify": + let email = args["email"].getStr() + let user_id = rs.get_user_id(email) + let email_token = rs.generate_email_verification_token(user_id) + try: + await sendEmail(email, "Buckets Relay - Email Verification", + &"Use this code to verify your email address:\n\n{email_token}") + resp["response"] = newJBool(true) + except: + resp["error"] = newJString("Failed to send email") + of "verify": + let email = args["email"].getStr() + let code = args["code"].getStr() + let user_id = rs.get_user_id(email) + resp["response"] = newJBool(rs.use_email_verification_token(user_id, code)) + of "resetPassword": + let email = args["email"].getStr() + let pw_token = rs.generate_password_reset_token(email) + try: + await sendEmail(email, "Buckets Relay - Password Reset", + &"Use this code to change your password:\n\n{pw_token}") + except: + resp["error"] = newJString("Failed to send email") + of "updatePassword": + let pw_token = args["token"].getStr() + let new_password = args["new_password"].getStr() + rs.update_password_with_token(pw_token, new_password) + else: + resp["error"] = newJString("Unknown command"); + except NotFound: + vlog "[ws.auth] not found" + resp["error"] = newJString("Not found") + except DuplicateUser: + vlog "[ws.auth] duplicate user" + resp["error"] = newJString("Account already exists") + except WrongPassword: + vlog "[ws.auth] wrong password" + resp["error"] = newJString("Wrong password") + except Exception as exc: + vlog "[ws.auth] unexpected error" + error exc.msg + resp["error"] = newJString("Unexpected error") + finally: + vlog "[ws.auth] sending response" + await ws.send($resp) + except WSClosedError: + vlog "[ws.auth] WSClosedError" + except WebSocketError as exc: + logging.error "relay/server: WebSocketError: " & exc.msg + await req.sendError(Http400) + except Exception as exc: + logging.error "relay/server: connection failed: " & exc.msg + await req.sendError(Http400) + +proc handleRequestV1(rs: RelayServer, req: HttpRequest, subpath: string) {.async, gcsafe.} = + ## Version 1 request handling + {.gcsafe.}: + var path = req.uri.path.substr(subpath.len) + if path == "": path = "/" + if path == "/relay": + await rs.handleRequestRelayV1(req) + elif path == "/auth": + when multiusermode: + await rs.handleRequestAuthV1(req) + else: + await req.sendError(Http404) + elif path == "/": + let ctx = rs.mcontext() + ctx["openregistration"] = multiusermode + let rendered = render("{{>index}}", ctx) + await req.sendHTML(rendered) + else: + await req.sendError(Http404) + +#------------------------------------------------------------- +# HTTP routing common to all versions +#------------------------------------------------------------- + +proc handleRequest*(rs: RelayServer, req: HttpRequest): Future[void] {.async, gcsafe.} = + ## Handle a relay server websocket request. + {.gcsafe.}: + let reqid = rs.nextid + rs.nextid.inc() + rs.runningRequests[reqid] = req + defer: rs.runningRequests.del(reqid) + when defined(testmode): + allHttpRequests.add(req) + defer: + allHttpRequests.delete(allHttpRequests.find(req)) + let path = req.uri.path + if path.startsWith("/v1/"): + await rs.handleRequestV1(req, "/v1") + elif path == "/versions": + await req.sendResponse(Http200, data = versionSupport) + elif path == "/": + var headers = HttpTable.init() + headers.add("Location", "/v1/") + await req.sendResponse(Http307, headers, "") + elif path.startsWith("/static"): + let subpath = path.substr("/static".len) + try: + var headers = HttpTable.init() + headers.add("Content-Type", mimedb.getMimetype(path.splitFile.ext)) + await req.sendResponse(Http200, headers, data = readStaticFile(subpath)) + except: + await req.sendError(Http404) + else: + await req.sendError(Http404) + +proc start*(rs: RelayServer, address: TransportAddress, tlsPrivateKey = "", tlsCertificate = "") = + ## Start the relay server at the given address. + let + socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} + if tlsPrivateKey != "" and tlsCertificate != "": + rs.http = RelayHttpServer( + tls: true, + httpsServer: TlsHttpServer.create( + address = address, + tlsPrivateKey = TLSPrivateKey.init(tlsPrivateKey), + tlsCertificate = TLSCertificate.init(tlsCertificate), + flags = socketFlags) + ) + else: + rs.http = RelayHttpServer( + tls: false, + httpServer: HttpServer.create(address, flags = socketFlags), + ) + + rs.http.handler = proc(request: HttpRequest) {.async.} = + try: + await rs.handleRequest(request) + except: + let msg = getCurrentExceptionMsg() + if "Stream is already closed" in msg: + discard + else: + logging.error rs.logname, "Error handling HTTP request: " & getCurrentExceptionMsg() + rs.http.start() + +proc finish*(rs: RelayServer) {.async.} = + ## Completely stop the running server + rs.http.stop() + rs.http.close() + await rs.http.join() + vlog rs.logname, "finished" diff --git a/src/bucketsrelay/v1/stringproto.nim b/src/bucketsrelay/v1/stringproto.nim new file mode 100644 index 0000000..eb75eaf --- /dev/null +++ b/src/bucketsrelay/v1/stringproto.nim @@ -0,0 +1,108 @@ +# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +import strutils +import ./proto +import ./netstring + +proc dumps*(ev: RelayEvent): string = + ## Serialize a RelayEvent to a string. Opposite of loadsRelayEvent + result = $ev.kind + case ev.kind + of Who: + result.add nsencode(ev.who_challenge) + of Authenticated: + discard + of Connected: + result.add nsencode(ev.conn_pubkey.string) + of Disconnected: + result.add nsencode(ev.dcon_pubkey.string) + of Data: + result.add nsencode($ev.sender_pubkey.string) + result.add nsencode(ev.data) + of Entered: + result.add nsencode(ev.entered_pubkey.string) + of Exited: + result.add nsencode(ev.exited_pubkey.string) + of ErrorEvent: + result.add nsencode($ev.err_code) + result.add nsencode(ev.err_message) + +proc loadsRelayEvent*(msg: string): RelayEvent = + ## Deserialize a RelayEvent from a string. Opposite of dumps + let kind = case $msg[0] + of $Who: Who + of $Authenticated: Authenticated + of $Connected: Connected + of $Disconnected: Disconnected + of $Data: Data + of $Entered: Entered + of $Exited: Exited + of $ErrorEvent: ErrorEvent + else: + raise ValueError.newException("Unknown event type: " & msg[0]) + let rest = msg[1..^1] + result = RelayEvent(kind: kind) + var decoder = newNetstringDecoder() + decoder.consume(rest) + case kind + of Who: + result.who_challenge = decoder.nextMessage() + of Authenticated: + discard + of Connected: + result.conn_pubkey = decoder.nextMessage().PublicKey + of Disconnected: + result.dcon_pubkey = decoder.nextMessage().PublicKey + of Data: + result.sender_pubkey = decoder.nextMessage().PublicKey + result.data = decoder.nextMessage() + of Entered: + result.entered_pubkey = decoder.nextMessage().PublicKey + of Exited: + result.exited_pubkey = decoder.nextMessage().PublicKey + of ErrorEvent: + result.err_code = parseEnum[ErrorCode](decoder.nextMessage()) + result.err_message = decoder.nextMessage() + +proc dumps*(cmd: RelayCommand): string = + ## Serialize a RelayCommand to a string. Opposite of loadsRelayCommand. + result = $cmd.kind + case cmd.kind + of Iam: + result.add nsencode(cmd.iam_signature) + result.add nsencode(cmd.iam_pubkey.string) + of Connect: + result.add nsencode(cmd.conn_pubkey.string) + of Disconnect: + result.add nsencode(cmd.dcon_pubkey.string) + of SendData: + result.add nsencode(cmd.dest_pubkey.string) + result.add nsencode(cmd.send_data) + +proc loadsRelayCommand*(msg: string): RelayCommand = + ## Deserialize a RelayCommand from a string. Opposite of dumps. + let kind = case $msg[0] + of $Iam: Iam + of $Connect: Connect + of $Disconnect: Disconnect + of $SendData: SendData + else: + raise ValueError.newException("Unknown command type: " & msg[0]) + let rest = msg[1..^1] + result = RelayCommand(kind: kind) + var decoder = newNetstringDecoder() + decoder.consume(rest) + case kind + of Iam: + result.iam_signature = decoder.nextMessage() + result.iam_pubkey = decoder.nextMessage().PublicKey + of Connect: + result.conn_pubkey = decoder.nextMessage().PublicKey + of Disconnect: + result.dcon_pubkey = decoder.nextMessage().PublicKey + of SendData: + result.dest_pubkey = decoder.nextMessage().PublicKey + result.send_data = decoder.nextMessage() diff --git a/src/bucketsrelay/cli.nim b/src/bucketsrelay/v2/cli.nim similarity index 100% rename from src/bucketsrelay/cli.nim rename to src/bucketsrelay/v2/cli.nim diff --git a/src/bucketsrelay/objs.nim b/src/bucketsrelay/v2/objs.nim similarity index 100% rename from src/bucketsrelay/objs.nim rename to src/bucketsrelay/v2/objs.nim diff --git a/src/bucketsrelay/proto2.nim b/src/bucketsrelay/v2/proto2.nim similarity index 100% rename from src/bucketsrelay/proto2.nim rename to src/bucketsrelay/v2/proto2.nim diff --git a/src/bucketsrelay/sampleclient.nim b/src/bucketsrelay/v2/sampleclient.nim similarity index 100% rename from src/bucketsrelay/sampleclient.nim rename to src/bucketsrelay/v2/sampleclient.nim diff --git a/src/bucketsrelay/server2.nim b/src/bucketsrelay/v2/server2.nim similarity index 98% rename from src/bucketsrelay/server2.nim rename to src/bucketsrelay/v2/server2.nim index edef9cd..20abb74 100644 --- a/src/bucketsrelay/server2.nim +++ b/src/bucketsrelay/v2/server2.nim @@ -30,8 +30,8 @@ type const VERSION = slurp"../CHANGELOG.md".split(" ")[1] - logo_png = slurp"../static/logo.png" - favicon_png = slurp"../static/favicon.png" + logo_png = slurp"./static/logo.png" + favicon_png = slurp"./static/favicon.png" let ADMIN_USERNAME = getEnv("ADMIN_USERNAME", "admin") let ADMIN_PWHASH = when defined(release): @@ -160,7 +160,7 @@ router myrouter: get "/": var html = "" - compileTemplateFile("index.nimja", baseDir = getScriptDir() / ".." / "templates", autoEscape = true, varname = "html") + compileTemplateFile("index.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") resp html get "/static/logo.png": @@ -365,7 +365,7 @@ router myrouter: )) var html = "" - compileTemplateFile("stats.nimja", baseDir = getScriptDir() / ".." / "templates", autoEscape = true, varname = "html") + compileTemplateFile("stats.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") resp html proc main(database: string, port: Port, address = "127.0.0.1") = diff --git a/src/static/favicon.png b/src/bucketsrelay/v2/static/favicon.png similarity index 100% rename from src/static/favicon.png rename to src/bucketsrelay/v2/static/favicon.png diff --git a/src/static/logo.png b/src/bucketsrelay/v2/static/logo.png similarity index 100% rename from src/static/logo.png rename to src/bucketsrelay/v2/static/logo.png diff --git a/src/templates/index.nimja b/src/bucketsrelay/v2/templates/index.nimja similarity index 100% rename from src/templates/index.nimja rename to src/bucketsrelay/v2/templates/index.nimja diff --git a/src/templates/stats.nimja b/src/bucketsrelay/v2/templates/stats.nimja similarity index 100% rename from src/templates/stats.nimja rename to src/bucketsrelay/v2/templates/stats.nimja diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index 0400be9..85f06f7 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -8,8 +8,8 @@ import std/unittest import ./util -import bucketsrelay/sampleclient -import bucketsrelay/proto2 +import bucketsrelay/v2/sampleclient +import bucketsrelay/v2/proto2 import ws diff --git a/tests/tnetstring.nim b/tests/tnetstring.nim index 2f31298..8e9136b 100644 --- a/tests/tnetstring.nim +++ b/tests/tnetstring.nim @@ -1,6 +1,6 @@ import std/unittest -import bucketsrelay/objs +import bucketsrelay/v2/objs suite "encode": test "nsencode": diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 3a19eaa..e33f63e 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -6,7 +6,7 @@ import std/strutils import std/unittest import lowdb/sqlite -import bucketsrelay/proto2 +import bucketsrelay/v2/proto2 if getEnv("SHOW_LOGS") != "": var L = newConsoleLogger() diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 19167ba..122362e 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -3,8 +3,8 @@ import std/logging import std/options import ./util -import bucketsrelay/proto2 -import bucketsrelay/objs +import bucketsrelay/v2/proto2 +import bucketsrelay/v2/objs test "MessageKind": for kind in low(MessageKind)..high(MessageKind): From 1f9b0650b9ecdb9b569f73f32ef17ee44f036b33 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 3 Dec 2025 10:36:14 -0500 Subject: [PATCH 39/46] RelayMessages have a resp_id to associate them with RelayCommand that caused the message --- ...ew-RelayMessages-have-a-20251203-103527.md | 1 + src/bucketsrelay/v2/objs.nim | 57 ++++-- src/bucketsrelay/v2/proto2.nim | 64 ++++--- tests/tfunctional.nim | 2 +- tests/tproto2.nim | 173 +++++++++++++++++- tests/tserde2.nim | 43 +++-- 6 files changed, 280 insertions(+), 60 deletions(-) create mode 100644 changes/new-RelayMessages-have-a-20251203-103527.md diff --git a/changes/new-RelayMessages-have-a-20251203-103527.md b/changes/new-RelayMessages-have-a-20251203-103527.md new file mode 100644 index 0000000..eb3335c --- /dev/null +++ b/changes/new-RelayMessages-have-a-20251203-103527.md @@ -0,0 +1 @@ +RelayMessages have a resp_id to associate them with RelayCommand that caused the message diff --git a/src/bucketsrelay/v2/objs.nim b/src/bucketsrelay/v2/objs.nim index 1209d87..f7714ec 100644 --- a/src/bucketsrelay/v2/objs.nim +++ b/src/bucketsrelay/v2/objs.nim @@ -48,6 +48,7 @@ type InvalidParams = 5 RelayMessage* = object + resp_id*: int case kind*: MessageKind of Who: who_challenge*: Challenge @@ -82,6 +83,7 @@ type HasChunks RelayCommand* = object + resp_id*: int case kind*: CommandKind of Iam: iam_pubkey*: PublicKey @@ -186,7 +188,7 @@ proc `$`*(msg: RelayMessage): string = result.add ")" proc `==`*(a, b: RelayMessage): bool = - if a.kind != b.kind: + if a.kind != b.kind or a.resp_id != b.resp_id: return false else: case a.kind @@ -231,7 +233,7 @@ proc `$`*(cmd: RelayCommand): string = result.add ")" proc `==`*(a, b: RelayCommand): bool = - if a.kind != b.kind: + if a.kind != b.kind or a.resp_id != b.resp_id: return false else: case a.kind @@ -419,6 +421,9 @@ proc deserialize*(typ: typedesc[seq[string]], val: string): seq[string] = proc serialize*(msg: RelayMessage): string = result &= msg.kind.serialize() + # For Who and Data messages, resp_id is always omitted (always 0) + if msg.kind notin {Who, Data}: + result &= nsencode($msg.resp_id) case msg.kind of Who: result &= msg.who_challenge.serialize() @@ -448,37 +453,45 @@ proc serialize*(msg: RelayMessage): string = proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = if s.len == 0: raise ValueError.newException("Empty RelayMessage") - let kind = MessageKind.deserialize(s[0]) + var idx = 0 + let kind = MessageKind.deserialize(s[idx]) + idx.inc() + # For Who and Data messages, resp_id is always 0 and not serialized + let resp_id = if kind in {Who, Data}: + 0 + else: + s.nsdecode(idx).parseInt() case kind of Who: - return RelayMessage(kind: Who, who_challenge: Challenge.deserialize(s[1..^1])) + return RelayMessage(kind: Who, resp_id: resp_id, who_challenge: Challenge.deserialize(s[idx..^1])) of Okay: - return RelayMessage(kind: Okay, ok_cmd: CommandKind.deserialize(s[1])) + return RelayMessage(kind: Okay, resp_id: resp_id, ok_cmd: CommandKind.deserialize(s[idx])) of Error: return RelayMessage( kind: Error, - err_cmd: CommandKind.deserialize(s[1]), - err_code: ErrorCode.deserialize(s[2]), - err_message: s[3..^1] + resp_id: resp_id, + err_cmd: CommandKind.deserialize(s[idx]), + err_code: ErrorCode.deserialize(s[idx+1]), + err_message: s[(idx+2)..^1] ) of Note: - var idx = 1 return RelayMessage( kind: Note, + resp_id: resp_id, note_topic: s.nsdecode(idx), note_data: s.nsdecode(idx), ) of Data: - var idx = 1 return RelayMessage( kind: Data, + resp_id: resp_id, data_src: s.nsdecode(idx).PublicKey, data_val: s.nsdecode(idx), ) of Chunk: - var idx = 1 return RelayMessage( kind: Chunk, + resp_id: resp_id, chunk_src: s.nsdecode(idx).PublicKey, chunk_key: s.nsdecode(idx), chunk_val: if idx >= s.len: @@ -487,9 +500,9 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = some(s.nsdecode(idx)), ) of ChunkStatus: - var idx = 1 return RelayMessage( kind: ChunkStatus, + resp_id: resp_id, status_src: s.nsdecode(idx).PublicKey, present: deserialize(seq[string], s.nsdecode(idx)), absent: deserialize(seq[string], s.nsdecode(idx)), @@ -497,6 +510,7 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = proc serialize*(cmd: RelayCommand): string = result &= cmd.kind.serialize + result &= nsencode($cmd.resp_id) case cmd.kind of Iam: result &= cmd.iam_pubkey.string.nsencode @@ -523,54 +537,57 @@ proc serialize*(cmd: RelayCommand): string = proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = if s.len == 0: raise ValueError.newException("Empty RelayCommand") - let kind = CommandKind.deserialize(s[0]) + var idx = 0 + let kind = CommandKind.deserialize(s[idx]) + idx.inc() + let resp_id = s.nsdecode(idx).parseInt() case kind of Iam: - var idx = 1 return RelayCommand( kind: Iam, + resp_id: resp_id, iam_pubkey: s.nsdecode(idx).PublicKey, iam_answer: ChallengeAnswer.deserialize(s.nsdecode(idx)), ) of PublishNote: - var idx = 1 return RelayCommand( kind: PublishNote, + resp_id: resp_id, pub_topic: s.nsdecode(idx), pub_data: s.nsdecode(idx), ) of FetchNote: - var idx = 1 return RelayCommand( kind: FetchNote, + resp_id: resp_id, fetch_topic: s.nsdecode(idx), ) of SendData: - var idx = 1 return RelayCommand( kind: SendData, + resp_id: resp_id, send_dst: s.nsdecode(idx).PublicKey, send_val: s.nsdecode(idx), ) of StoreChunk: - var idx = 1 return RelayCommand( kind: StoreChunk, + resp_id: resp_id, chunk_dst: deserializePubKeys(s.nsdecode(idx)), chunk_key: s.nsdecode(idx), chunk_val: s.nsdecode(idx), ) of GetChunks: - var idx = 1 return RelayCommand( kind: GetChunks, + resp_id: resp_id, chunk_src: s.nsdecode(idx).PublicKey, chunk_keys: deserialize(seq[string], s.nsdecode(idx)), ) of HasChunks: - var idx = 1 return RelayCommand( kind: HasChunks, + resp_id: resp_id, has_src: s.nsdecode(idx).PublicKey, has_keys: deserialize(seq[string], s.nsdecode(idx)), ) diff --git a/src/bucketsrelay/v2/proto2.nim b/src/bucketsrelay/v2/proto2.nim index 3042135..3be39f0 100644 --- a/src/bucketsrelay/v2/proto2.nim +++ b/src/bucketsrelay/v2/proto2.nim @@ -275,18 +275,20 @@ proc newRelay*[T](db: DbConn): Relay[T] = result.clients = newTable[PublicKey, RelayConnection[T]]() db.updateSchema() -template sendError*[T](conn: RelayConnection[T], msg: string, cmd: CommandKind, code: ErrorCode) = +template sendError*[T](conn: RelayConnection[T], cmd: RelayCommand, msg: string, code: ErrorCode) = conn.sendMessage(RelayMessage( kind: Error, + resp_id: cmd.resp_id, err_code: code, err_message: msg, - err_cmd: cmd, + err_cmd: cmd.kind, )) -template sendOkay*[T](conn: RelayConnection[T], cmd: CommandKind) = +template sendOkay*[T](conn: RelayConnection[T], cmd: RelayCommand) = conn.sendMessage(RelayMessage( kind: Okay, - ok_cmd: cmd, + resp_id: cmd.resp_id, + ok_cmd: cmd.kind, )) proc is_valid*(x: PublicKey): bool = @@ -308,6 +310,7 @@ proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = result.challenge = some(generateChallenge()) result.sendMessage(RelayMessage( kind: Who, + resp_id: 0, # Who messages are not triggered by a command who_challenge: result.challenge.get(), )) result.relay = relay @@ -480,6 +483,7 @@ proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = let row = orow.get() result = some(RelayMessage( kind: Data, + resp_id: 0, # Data messages are not triggered by recipient's command data_src: PublicKey.fromDB(row[0].b), data_val: row[1].b.string, )) @@ -502,23 +506,23 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay when LOG_COMMS: info "[" & conn.pubkey.abbr & "] DO " & $cmd if conn.pubkey.isNone and cmd.kind != Iam: - conn.sendError("Not allowed", cmd.kind, NotAllowed) + conn.sendError(cmd, "Not allowed", NotAllowed) return case cmd.kind of Iam: if conn.challenge.isNone: - conn.sendError("Already authenticated", cmd.kind, Generic) + conn.sendError(cmd, "Already authenticated", Generic) return let challenge = conn.challenge.get() conn.challenge = none[Challenge]() # disable future authentication attempts - + try: if not is_valid_answer(cmd.iam_pubkey, challenge, cmd.iam_answer): - conn.sendError("Invalid answer", cmd.kind, Generic) + conn.sendError(cmd, "Invalid answer", Generic) return except CatchableError: - conn.sendError("Invalid answer", cmd.kind, Generic) + conn.sendError(cmd, "Invalid answer", Generic) return # successful connection @@ -526,7 +530,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay conn.pubkey = some(pubkey) relay.clients[pubkey] = conn info &"[{conn.pubkey.abbr}] connected" - conn.sendOkay cmd.kind + conn.sendOkay(cmd) relay.db.record_event_stat( ip = conn.ip, pubkey = pubkey, @@ -550,13 +554,13 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay break of PublishNote: if cmd.pub_topic.len > RELAY_MAX_TOPIC_SIZE: - conn.sendError("Topic too long", cmd.kind, TooLarge) + conn.sendError(cmd, "Topic too long", TooLarge) elif cmd.pub_data.len > RELAY_MAX_NOTE_SIZE: - conn.sendError("Data too long", cmd.kind, TooLarge) + conn.sendError(cmd, "Data too long", TooLarge) else: let pubkey = conn.pubkey.get() if relay.noteCount(pubkey) >= RELAY_MAX_NOTES: - conn.sendError("Too many notes", cmd.kind, StorageLimitExceeded) + conn.sendError(cmd, "Too many notes", StorageLimitExceeded) else: relay.db.record_transfer_stat( ip = conn.ip, @@ -572,9 +576,10 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay if opubkey.isSome: # someone is waiting var other_conn = relay.clients[opubkey.get()] - conn.sendOkay cmd.kind + conn.sendOkay(cmd) other_conn.sendMessage(RelayMessage( kind: Note, + resp_id: 0, # Not triggered by other_conn's command note_data: cmd.pub_data, note_topic: cmd.pub_topic, )) @@ -592,12 +597,12 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay cmd.pub_data.DbBlob, pubkey, ) - conn.sendOkay cmd.kind + conn.sendOkay(cmd) except: - conn.sendError("Duplicate topic", cmd.kind, Generic) + conn.sendError(cmd, "Duplicate topic", Generic) of FetchNote: if cmd.fetch_topic.len > RELAY_MAX_TOPIC_SIZE: - conn.sendError("Topic too long", cmd.kind, TooLarge) + conn.sendError(cmd, "Topic too long", TooLarge) else: let odata = relay.popNote(cmd.fetch_topic) if odata.isSome(): @@ -605,6 +610,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay let data = odata.get() conn.sendMessage(RelayMessage( kind: Note, + resp_id: cmd.resp_id, # Response to FetchNote command note_data: data, note_topic: cmd.fetch_topic, )) @@ -618,13 +624,13 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay relay.addNoteSub(cmd.fetch_topic, conn.pubkey.get()) of SendData: if cmd.send_val.len > RELAY_MAX_MESSAGE_SIZE: - conn.sendError("Data too long", cmd.kind, TooLarge) + conn.sendError(cmd, "Data too long", TooLarge) elif not cmd.send_dst.is_valid(): - conn.sendError("Invalid pubkey", cmd.kind, InvalidParams) + conn.sendError(cmd, "Invalid pubkey", InvalidParams) else: let pubkey = conn.pubkey.get() if relay.max_transfer_rate != 0 and relay.db.current_data_in(pubkey) > relay.max_transfer_rate: - conn.sendError("Rate limit exceeded", cmd.kind, TransferLimitExceeeded) + conn.sendError(cmd, "Rate limit exceeded", TransferLimitExceeeded) else: relay.db.record_transfer_stat( ip = conn.ip, @@ -641,6 +647,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay var other_conn = relay.clients[cmd.send_dst] other_conn.sendMessage(RelayMessage( kind: Data, + resp_id: 0, # Not triggered by other_conn's command data_src: pubkey, data_val: cmd.send_val, )) @@ -655,17 +662,17 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay pubkey, cmd.send_dst, cmd.send_val.DbBlob) of StoreChunk: if cmd.chunk_key.len > RELAY_MAX_CHUNK_KEY_SIZE: - conn.sendError("Key too long", cmd.kind, TooLarge) + conn.sendError(cmd, "Key too long", TooLarge) elif cmd.chunk_val.len > RELAY_MAX_CHUNK_SIZE: - conn.sendError("Value too long", cmd.kind, TooLarge) + conn.sendError(cmd, "Value too long", TooLarge) elif cmd.chunk_dst.len > RELAY_MAX_CHUNK_DSTS: - conn.sendError("Too many recipients", cmd.kind, TooLarge) + conn.sendError(cmd, "Too many recipients", TooLarge) elif cmd.chunk_dst.any_invalid(): - conn.sendError("Invalid pubkey", cmd.kind, InvalidParams) + conn.sendError(cmd, "Invalid pubkey", InvalidParams) else: let pubkey = conn.pubkey.get() if relay.max_chunk_space > 0 and relay.db.chunk_space_used(pubkey) > relay.max_chunk_space: - conn.sendError("Too much chunk data", cmd.kind, StorageLimitExceeded) + conn.sendError(cmd, "Too much chunk data", StorageLimitExceeded) else: relay.db.record_event_stat( ip = conn.ip, @@ -696,7 +703,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay of GetChunks: for key in cmd.chunk_keys: if key.len > RELAY_MAX_CHUNK_KEY_SIZE: - conn.sendError("Key too long", cmd.kind, TooLarge) + conn.sendError(cmd, "Key too long", TooLarge) return relay.delExpiredChunks() let pubkey = conn.pubkey.get() @@ -718,6 +725,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay let row = orow.get() conn.sendMessage(RelayMessage( kind: Chunk, + resp_id: cmd.resp_id, chunk_src: cmd.chunk_src, chunk_key: key, chunk_val: some(row[0].b.string), @@ -725,6 +733,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay else: conn.sendMessage(RelayMessage( kind: Chunk, + resp_id: cmd.resp_id, chunk_src: cmd.chunk_src, chunk_key: key, chunk_val: none[string](), @@ -732,7 +741,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay of HasChunks: for key in cmd.has_keys: if key.len > RELAY_MAX_CHUNK_KEY_SIZE: - conn.sendError("Key too long", cmd.kind, TooLarge) + conn.sendError(cmd, "Key too long", TooLarge) return relay.delExpiredChunks() let pubkey = conn.pubkey.get() @@ -768,6 +777,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay absent.add(key) conn.sendMessage(RelayMessage( kind: ChunkStatus, + resp_id: cmd.resp_id, status_src: cmd.has_src, present: present, absent: absent, diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index 85f06f7..dba3d31 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -27,7 +27,7 @@ proc startServer(port: Port): Process = let compileProcess = startProcess( "nim", workingDir = currentSourcePath().parentDir().parentDir(), - args = ["c", "-d:testmode", "-o:" & bin, "src/bucketsrelay/server2.nim"], + args = ["c", "-d:testmode", "-o:" & bin, "src/bucketsrelay/v2/server2.nim"], options = {poStdErrToStdOut, poUsePath} ) let output = compileProcess.outputStream.readAll() diff --git a/tests/tproto2.nim b/tests/tproto2.nim index e33f63e..d5bcd37 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -80,12 +80,14 @@ proc authenticatedConn(relay: Relay, keys: KeyPair): RelayConnection[TestClient] let answer = who.who_challenge.answer(client.sk) relay.handleCommand(conn, RelayCommand( kind: Iam, + resp_id: 1, iam_answer: answer, iam_pubkey: client.pk )) let ok = conn.pop() doAssert ok.kind == Okay doAssert ok.ok_cmd == Iam + doAssert ok.resp_id == 1 return conn proc authenticatedConn(relay: Relay): RelayConnection[TestClient] = @@ -846,4 +848,173 @@ suite "stats": check db.stats_transfer_total(period="2010-01") == (1000+2000, 500+250, "", "".PublicKey, "2010-01") check db.stats_transfer_total(period="2010-02") == (3000+500, 100+100, "", "".PublicKey, "2010-02") - \ No newline at end of file +suite "resp_id": + + test "Who message has resp_id 0": + let relay = testRelay() + let client = newTestClient(genkeys()) + var conn = relay.initAuth(client) + let who = conn.pop(Who) + check who.resp_id == 0 + + test "Iam command response has matching resp_id": + let relay = testRelay() + let client = newTestClient(genkeys()) + var conn = relay.initAuth(client) + let who = conn.pop(Who) + let answer = who.who_challenge.answer(client.sk) + + relay.handleCommand(conn, RelayCommand( + kind: Iam, + resp_id: 42, + iam_answer: answer, + iam_pubkey: client.pk + )) + let ok = conn.pop(Okay) + check ok.resp_id == 42 + + test "PublishNote response has matching resp_id": + let relay = testRelay() + var alice = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + resp_id: 123, + pub_topic: "test", + pub_data: "data", + )) + let ok = alice.pop(Okay) + check ok.resp_id == 123 + + test "FetchNote response has matching resp_id": + let relay = testRelay() + var alice = relay.authenticatedConn() + + # Publish a note first + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "test", + pub_data: "data", + )) + discard alice.pop(Okay) + + # Fetch the note with resp_id + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + resp_id: 456, + fetch_topic: "test", + )) + let note = alice.pop(Note) + check note.resp_id == 456 + + test "Error response has matching resp_id": + let relay = testRelay() + var alice = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + resp_id: 789, + pub_topic: "a".repeat(RELAY_MAX_TOPIC_SIZE + 1), + pub_data: "data", + )) + let err = alice.pop(Error) + check err.resp_id == 789 + + test "Data message has resp_id 0 (no command trigger)": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + resp_id: 111, + send_dst: bob.pk, + send_val: "hello", + )) + + # Bob receives the Data message - it should have resp_id 0 + # because it wasn't triggered by Bob's command + let data = bob.pop(Data) + check data.resp_id == 0 + + test "StoreChunk command response has matching resp_id": + let relay = testRelay() + var alice = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + resp_id: 222, + chunk_dst: @[], + chunk_key: "key", + chunk_val: "val", + )) + # StoreChunk doesn't send a response by default, no message to check + check alice.msgCount == 0 + + test "GetChunks response has matching resp_id": + let relay = testRelay() + var alice = relay.authenticatedConn() + + # Store a chunk first + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: @[], + chunk_key: "key", + chunk_val: "val", + )) + + # Get the chunk with resp_id + relay.handleCommand(alice, RelayCommand( + kind: GetChunks, + resp_id: 333, + chunk_src: alice.pk, + chunk_keys: @["key"], + )) + let chunk = alice.pop(Chunk) + check chunk.resp_id == 333 + + test "HasChunks response has matching resp_id": + let relay = testRelay() + var alice = relay.authenticatedConn() + + # Store a chunk first + relay.handleCommand(alice, RelayCommand( + kind: StoreChunk, + chunk_dst: @[], + chunk_key: "key", + chunk_val: "val", + )) + + # Check if chunk exists with resp_id + relay.handleCommand(alice, RelayCommand( + kind: HasChunks, + resp_id: 444, + has_src: alice.pk, + has_keys: @["key"], + )) + let status = alice.pop(ChunkStatus) + check status.resp_id == 444 + + test "Multiple commands with different resp_ids": + let relay = testRelay() + var alice = relay.authenticatedConn() + + # Send multiple commands with different resp_ids + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + resp_id: 100, + pub_topic: "topic1", + pub_data: "data1", + )) + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + resp_id: 200, + pub_topic: "topic2", + pub_data: "data2", + )) + + let ok1 = alice.pop(Okay) + check ok1.resp_id == 100 + let ok2 = alice.pop(Okay) + check ok2.resp_id == 200 + diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 122362e..6d30580 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -17,13 +17,13 @@ test "CommandKind": test "RelayMessage": for kind in low(MessageKind)..high(MessageKind): let example = case kind - of Who: RelayMessage(kind: Who, who_challenge: generateChallenge()) - of Okay: RelayMessage(kind: Okay, ok_cmd: SendData) - of Error: RelayMessage(kind: Error, err_cmd: SendData, err_code: TooLarge, err_message: "foo") - of Note: RelayMessage(kind: Note, note_topic: "something", note_data: "data") - of Data: RelayMessage(kind: Data, data_src: "hey".PublicKey, data_val: "foo") - of Chunk: RelayMessage(kind: Chunk, chunk_src: "hey".PublicKey, chunk_key: "key", chunk_val: some("theval")) - of ChunkStatus: RelayMessage(kind: ChunkStatus, status_src: "a".PublicKey, present: @["foo"], absent: @["bar"]) + of Who: RelayMessage(kind: Who, resp_id: 0, who_challenge: generateChallenge()) + of Okay: RelayMessage(kind: Okay, resp_id: 42, ok_cmd: SendData) + of Error: RelayMessage(kind: Error, resp_id: 123, err_cmd: SendData, err_code: TooLarge, err_message: "foo") + of Note: RelayMessage(kind: Note, resp_id: 456, note_topic: "something", note_data: "data") + of Data: RelayMessage(kind: Data, resp_id: 0, data_src: "hey".PublicKey, data_val: "foo") + of Chunk: RelayMessage(kind: Chunk, resp_id: 789, chunk_src: "hey".PublicKey, chunk_key: "key", chunk_val: some("theval")) + of ChunkStatus: RelayMessage(kind: ChunkStatus, resp_id: 999, status_src: "a".PublicKey, present: @["foo"], absent: @["bar"]) let serialized = example.serialize() info $example info "serialized: " & serialized.nice @@ -34,6 +34,7 @@ test "RelayCommand": let example = case kind of Iam: RelayCommand( kind: Iam, + resp_id: 1, iam_pubkey: "hey".PublicKey, iam_answer: ( nonce: 1, @@ -41,18 +42,20 @@ test "RelayCommand": signature: "hey", ), ) - of PublishNote: RelayCommand(kind: PublishNote, pub_topic: "topic", pub_data: "data") - of FetchNote: RelayCommand(kind: FetchNote, fetch_topic: "topic") - of SendData: RelayCommand(kind: SendData, send_dst: "one".PublicKey, send_val: "data") + of PublishNote: RelayCommand(kind: PublishNote, resp_id: 100, pub_topic: "topic", pub_data: "data") + of FetchNote: RelayCommand(kind: FetchNote, resp_id: 200, fetch_topic: "topic") + of SendData: RelayCommand(kind: SendData, resp_id: 300, send_dst: "one".PublicKey, send_val: "data") of StoreChunk: RelayCommand( kind: StoreChunk, + resp_id: 400, chunk_dst: @["one".PublicKey], chunk_key: "theky", chunk_val: "someval" ) - of GetChunks: RelayCommand(kind: GetChunks, chunk_src: "hey".PublicKey, chunk_keys: @["foo", "bar"]) + of GetChunks: RelayCommand(kind: GetChunks, resp_id: 500, chunk_src: "hey".PublicKey, chunk_keys: @["foo", "bar"]) of HasChunks: RelayCommand( kind: HasChunks, + resp_id: 600, has_src: "hey".PublicKey, has_keys: @["foo", "Bar"], ) @@ -63,6 +66,7 @@ test "RelayCommand": test "Chunk with none": let chunk = RelayMessage(kind: Chunk, + resp_id: 888, chunk_src: "foo".PublicKey, chunk_key: "key", chunk_val: none[string](), @@ -77,3 +81,20 @@ test "ErrorCodes": let serialized = err.serialize() checkpoint "serialized.nice: " & nice($serialized) check ErrorCode.deserialize(serialized) == err + +test "resp_id serialization": + # Test that resp_id values are preserved during serialization + let msg1 = RelayMessage(kind: Okay, resp_id: 12345, ok_cmd: SendData) + check RelayMessage.deserialize(msg1.serialize()).resp_id == 12345 + + let msg2 = RelayMessage(kind: Error, resp_id: 99999, err_cmd: Iam, err_code: Generic, err_message: "test") + check RelayMessage.deserialize(msg2.serialize()).resp_id == 99999 + + let msg3 = RelayMessage(kind: Who, resp_id: 0, who_challenge: generateChallenge()) + check RelayMessage.deserialize(msg3.serialize()).resp_id == 0 + + let cmd1 = RelayCommand(kind: PublishNote, resp_id: 54321, pub_topic: "topic", pub_data: "data") + check RelayCommand.deserialize(cmd1.serialize()).resp_id == 54321 + + let cmd2 = RelayCommand(kind: SendData, resp_id: 0, send_dst: "dst".PublicKey, send_val: "val") + check RelayCommand.deserialize(cmd2.serialize()).resp_id == 0 From 8c6f7a3ca22ccc2892f09f2b78791f95a69be84b Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 3 Dec 2025 14:04:25 -0500 Subject: [PATCH 40/46] Notes must exist before you can fetch them, now. --- .../new-Notes-must-exist-20251203-140409.md | 1 + src/bucketsrelay/v2/objs.nim | 1 + src/bucketsrelay/v2/proto2.nim | 68 +++---------------- tests/tproto2.nim | 23 ++++--- 4 files changed, 25 insertions(+), 68 deletions(-) create mode 100644 changes/new-Notes-must-exist-20251203-140409.md diff --git a/changes/new-Notes-must-exist-20251203-140409.md b/changes/new-Notes-must-exist-20251203-140409.md new file mode 100644 index 0000000..f01702f --- /dev/null +++ b/changes/new-Notes-must-exist-20251203-140409.md @@ -0,0 +1 @@ +Notes must exist before you can fetch them, now. diff --git a/src/bucketsrelay/v2/objs.nim b/src/bucketsrelay/v2/objs.nim index f7714ec..272e3a8 100644 --- a/src/bucketsrelay/v2/objs.nim +++ b/src/bucketsrelay/v2/objs.nim @@ -46,6 +46,7 @@ type StorageLimitExceeded = 3 TransferLimitExceeeded = 4 InvalidParams = 5 + NotFound = 6 RelayMessage* = object resp_id*: int diff --git a/src/bucketsrelay/v2/proto2.nim b/src/bucketsrelay/v2/proto2.nim index 3be39f0..00bec58 100644 --- a/src/bucketsrelay/v2/proto2.nim +++ b/src/bucketsrelay/v2/proto2.nim @@ -240,14 +240,7 @@ proc updateSchema*(db: DbConn) = store INTEGER DEFAULT 0, PRIMARY KEY (period, ip, pubkey) )""") - - #----------- in-memory stuff - db.exec(sql"""CREATE TEMPORARY TABLE note_sub ( - topic TEXT PRIMARY KEY, - pubkey TEXT NOT NULL - )""") - db.exec(sql"CREATE INDEX note_sub_pubkey ON note_sub(pubkey)") - + #------------------------------------------------------------------- # Relay code @@ -318,7 +311,6 @@ proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = proc disconnect*[T](relay: Relay[T], conn: RelayConnection[T]) = if conn.pubkey.isSome: let pubkey = conn.pubkey.get() - relay.db.exec(sql"DELETE FROM note_sub WHERE pubkey=?", pubkey) relay.clients.del(pubkey) info &"[{conn.pubkey.abbr}] disconnected" @@ -416,21 +408,6 @@ proc delExpiredNotes(relay: Relay) = let offstring = &"{offset} seconds" relay.db.exec(sql"DELETE FROM note WHERE created <= datetime('now', ?)", offstring) -proc addNoteSub(relay: Relay, topic: string, pubkey: PublicKey) = - ## Record that a pubkey is subscribed to a topic - try: - relay.db.exec(sql"INSERT INTO note_sub (topic, pubkey) VALUES (?,?)", topic.DbBlob, pubkey) - info &"[{pubkey.abbr}] sub {topic}" - except CatchableError: - raise ValueError.newException("Topic already subscribed") - -proc getNoteSub(relay: Relay, topic: string): Option[PublicKey] = - ## Return a PublicKey who is listening for a note by topic. - relay.delExpiredNotes() - let orow = relay.db.getRow(sql"SELECT pubkey FROM note_sub WHERE topic = ?", topic.DbBlob) - if orow.isSome: - return some(PublicKey.fromDB(orow.get()[0].b)) - proc popNote(relay: Relay, topic: string): Option[string] = let db = relay.db relay.delExpiredNotes() @@ -449,10 +426,6 @@ proc popNote(relay: Relay, topic: string): Option[string] = warn &"[note] error " & getCurrentExceptionMsg() db.exec(sql"ROLLBACK") -proc delNoteSub(relay: Relay, topic: string) = - relay.db.exec(sql"DELETE FROM note_sub WHERE topic = ?", topic.DbBlob) - info &"[note] del {topic}" - proc noteCount(relay: Relay, pubkey: PublicKey): int = ## Return the number of notes currently published by this ip relay.db.getRow(sql"SELECT count(*) FROM note WHERE src = ?", pubkey).get()[0].i.int @@ -572,34 +545,15 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay pubkey = pubkey, publish = 1, ) - let opubkey = relay.getNoteSub(cmd.pub_topic) - if opubkey.isSome: - # someone is waiting - var other_conn = relay.clients[opubkey.get()] - conn.sendOkay(cmd) - other_conn.sendMessage(RelayMessage( - kind: Note, - resp_id: 0, # Not triggered by other_conn's command - note_data: cmd.pub_data, - note_topic: cmd.pub_topic, - )) - relay.delNoteSub(cmd.pub_topic) - relay.db.record_transfer_stat( - ip = other_conn.ip, - pubkey = other_conn.pubkey.get(), - data_out = cmd.pub_data.len, + try: + relay.db.exec(sql"INSERT INTO note (topic, data, src) VALUES (?, ?, ?)", + cmd.pub_topic.DbBlob, + cmd.pub_data.DbBlob, + pubkey, ) - else: - # no one is waiting - try: - relay.db.exec(sql"INSERT INTO note (topic, data, src) VALUES (?, ?, ?)", - cmd.pub_topic.DbBlob, - cmd.pub_data.DbBlob, - pubkey, - ) - conn.sendOkay(cmd) - except: - conn.sendError(cmd, "Duplicate topic", Generic) + conn.sendOkay(cmd) + except: + conn.sendError(cmd, "Duplicate topic", Generic) of FetchNote: if cmd.fetch_topic.len > RELAY_MAX_TOPIC_SIZE: conn.sendError(cmd, "Topic too long", TooLarge) @@ -620,8 +574,8 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay data_out = data.len, ) else: - # the note isn't here yet - relay.addNoteSub(cmd.fetch_topic, conn.pubkey.get()) + # the note doesn't exist + conn.sendError(cmd, "Topic not found", NotFound) of SendData: if cmd.send_val.len > RELAY_MAX_MESSAGE_SIZE: conn.sendError(cmd, "Data too long", TooLarge) diff --git a/tests/tproto2.nim b/tests/tproto2.nim index d5bcd37..801b909 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -247,15 +247,9 @@ suite "notes": kind: FetchNote, fetch_topic: "heyo", )) - relay.handleCommand(alice, RelayCommand( - kind: PublishNote, - pub_topic: "heyo", - pub_data: "foo", - )) - check alice.pop(Okay).ok_cmd == PublishNote - let note = alice.pop(Note) - check note.note_data == "foo" - check note.note_topic == "heyo" + let err = alice.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotFound test "publish max size topic": let relay = testRelay() @@ -308,7 +302,9 @@ suite "notes": kind: FetchNote, fetch_topic: "topic", )) - check alice.msgCount == 0 + let err = alice.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotFound test "fetch note again": let relay = testRelay() @@ -333,7 +329,9 @@ suite "notes": kind: FetchNote, fetch_topic: "sometopic", )) - check alice.msgCount == 0 + let err = alice.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotFound test "sub then disconnect, the pub": let relay = testRelay() @@ -344,6 +342,9 @@ suite "notes": kind: FetchNote, fetch_topic: "foo", )) + let err = bob.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotFound relay.disconnect(bob) relay.handleCommand(alice, RelayCommand( From 48c1eac32c06628165868674c10d02b59496037f Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 3 Dec 2025 14:07:08 -0500 Subject: [PATCH 41/46] Make sure fetch error has resp_id --- tests/tproto2.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 801b909..d08f1ab 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -245,11 +245,13 @@ suite "notes": var alice = relay.authenticatedConn() relay.handleCommand(alice, RelayCommand( kind: FetchNote, + resp_id: 34, fetch_topic: "heyo", )) let err = alice.pop(Error) check err.err_cmd == FetchNote check err.err_code == NotFound + check err.resp_id == 34 test "publish max size topic": let relay = testRelay() From 76fa4461c6a47edbfb0b50245e1d8f06308e6d27 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 3 Dec 2025 16:41:04 -0500 Subject: [PATCH 42/46] Rename public/secret --- ...ix-Rename-PublicKey-and-20251203-164052.md | 1 + src/bucketsrelay/v2/cli.nim | 28 +++++----- src/bucketsrelay/v2/objs.nim | 52 +++++++++--------- src/bucketsrelay/v2/proto2.nim | 54 +++++++++---------- src/bucketsrelay/v2/sampleclient.nim | 8 +-- src/bucketsrelay/v2/server2.nim | 20 +++---- tests/tproto2.nim | 42 +++++++-------- tests/tserde2.nim | 20 +++---- 8 files changed, 113 insertions(+), 112 deletions(-) create mode 100644 changes/fix-Rename-PublicKey-and-20251203-164052.md diff --git a/changes/fix-Rename-PublicKey-and-20251203-164052.md b/changes/fix-Rename-PublicKey-and-20251203-164052.md new file mode 100644 index 0000000..b208569 --- /dev/null +++ b/changes/fix-Rename-PublicKey-and-20251203-164052.md @@ -0,0 +1 @@ +Rename PublicKey and SecretKey to SignPublicKey and SignSecretKey diff --git a/src/bucketsrelay/v2/cli.nim b/src/bucketsrelay/v2/cli.nim index d8e6e8d..559a5cf 100644 --- a/src/bucketsrelay/v2/cli.nim +++ b/src/bucketsrelay/v2/cli.nim @@ -12,13 +12,13 @@ import ./proto2 type CmdContext = object - dst: PublicKey + dst: SignPublicKey -proc serialize(pubkey: PublicKey): string = +proc serialize(pubkey: SignPublicKey): string = base64.encode(pubkey.string) -proc deserialize(pubkey: typedesc[PublicKey], x: string): PublicKey = - base64.decode(x).PublicKey +proc deserialize(pubkey: typedesc[SignPublicKey], x: string): SignPublicKey = + base64.decode(x).SignPublicKey proc serialize(keys: KeyPair): string = base64.encode($(%* { @@ -28,7 +28,7 @@ proc serialize(keys: KeyPair): string = proc deserialize(pair: typedesc[KeyPair], x: string): KeyPair = let j = base64.decode(x).parseJson() - (j["pk"].getStr().PublicKey, j["sk"].getStr().SecretKey) + (j["pk"].getStr().SignPublicKey, j["sk"].getStr().SignSecretKey) proc loadKeys(src: string): KeyPair = let parts = src.split(":", 1) @@ -106,14 +106,14 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) echo data of "dst": if args.len == 0: - ctx.dst = "".PublicKey + ctx.dst = "".SignPublicKey else: - ctx.dst = PublicKey.deserialize(i.use(args)) + ctx.dst = SignPublicKey.deserialize(i.use(args)) echo "dst for future commands set to ", ctx.dst.serialize() of "send": var dst = ctx.dst if dst.string == "": - dst = PublicKey.deserialize(i.use(args)) + dst = SignPublicKey.deserialize(i.use(args)) let val = i.use(args) waitFor client.sendData(dst, val) of "recv": @@ -122,7 +122,7 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) of "store": var dst = ctx.dst if dst.string == "" or args.len >= 3: - dst = PublicKey.deserialize(i.use(args)) + dst = SignPublicKey.deserialize(i.use(args)) echo "Using key=" & dst.nice let key = i.use(args) let val = i.use(args) @@ -130,7 +130,7 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) of "get": var src = ctx.dst if src.string == "" or args.len >= 2: - src = PublicKey.deserialize(i.use(args)) + src = SignPublicKey.deserialize(i.use(args)) echo "Using key=" & src.serialize let key = i.use(args) let odata = waitFor client.getChunk(src, key) @@ -141,7 +141,7 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) of "has": var src = ctx.dst if src.string == "" or args.len >= 2: - src = PublicKey.deserialize(i.use(args)) + src = SignPublicKey.deserialize(i.use(args)) echo "Using key=" & src.serialize let key = i.use(args) let res = waitFor client.hasChunk(src, key) @@ -162,7 +162,7 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) else: echo "Unknown command ", cmd, " ", args -proc main(url: string, keys: KeyPair, dst = "".PublicKey) = +proc main(url: string, keys: KeyPair, dst = "".SignPublicKey) = # authenticate echo "...pubkey: ", keys.pk.serialize echo "...connecting..." @@ -202,9 +202,9 @@ when isMainModule: option("-d", "--dst", help = "Default destination") run: var dst = if opts.dst != "": - PublicKey.deserialize(opts.dst) + SignPublicKey.deserialize(opts.dst) else: - default(PublicKey) + default(SignPublicKey) main(opts.parentOpts.url, keys = loadKeys(opts.parentOpts.keys), dst = dst) try: diff --git a/src/bucketsrelay/v2/objs.nim b/src/bucketsrelay/v2/objs.nim index 272e3a8..e801027 100644 --- a/src/bucketsrelay/v2/objs.nim +++ b/src/bucketsrelay/v2/objs.nim @@ -16,8 +16,8 @@ import std/strformat import std/strutils type - PublicKey* = distinct string - SecretKey* = distinct string + SignPublicKey* = distinct string + SignSecretKey* = distinct string Challenge* = tuple bits: int @@ -63,14 +63,14 @@ type note_topic*: string note_data*: string of Data: - data_src*: PublicKey + data_src*: SignPublicKey data_val*: string of Chunk: - chunk_src*: PublicKey + chunk_src*: SignPublicKey chunk_key*: string chunk_val*: Option[string] of ChunkStatus: - status_src*: PublicKey + status_src*: SignPublicKey present*: seq[string] absent*: seq[string] @@ -87,7 +87,7 @@ type resp_id*: int case kind*: CommandKind of Iam: - iam_pubkey*: PublicKey + iam_pubkey*: SignPublicKey iam_answer*: ChallengeAnswer of PublishNote: pub_topic*: string @@ -95,17 +95,17 @@ type of FetchNote: fetch_topic*: string of SendData: - send_dst*: PublicKey + send_dst*: SignPublicKey send_val*: string of StoreChunk: - chunk_dst*: seq[PublicKey] + chunk_dst*: seq[SignPublicKey] chunk_key*: string chunk_val*: string of GetChunks: - chunk_src*: PublicKey + chunk_src*: SignPublicKey chunk_keys*: seq[string] of HasChunks: - has_src*: PublicKey + has_src*: SignPublicKey has_keys*: seq[string] const @@ -147,13 +147,13 @@ proc nicelong*(o: Option[string]): string = else: result = o.get().nicelong() -proc nice*(k: PublicKey): string = nice(k.string) -proc `$`*(k: PublicKey): string = k.nice() -proc hash*(p: PublicKey): Hash {.borrow.} -proc `==`*(a,b: PublicKey): bool {.borrow.} +proc nice*(k: SignPublicKey): string = nice(k.string) +proc `$`*(k: SignPublicKey): string = k.nice() +proc hash*(p: SignPublicKey): Hash {.borrow.} +proc `==`*(a,b: SignPublicKey): bool {.borrow.} -proc abbr*(a: PublicKey): string = abbr(a.nice) -proc abbr*(a: Option[PublicKey]): string = +proc abbr*(a: SignPublicKey): string = abbr(a.nice) +proc abbr*(a: Option[SignPublicKey]): string = if a.isSome: a.get.abbr else: @@ -402,14 +402,14 @@ proc deserialize*(typ: typedesc[ChallengeAnswer], val: string): ChallengeAnswer signature: val.nsdecode(idx), ) -proc serialize*(keys: seq[PublicKey]): string = +proc serialize*(keys: seq[SignPublicKey]): string = for key in keys: result &= nsencode(key.string) -proc deserializePubKeys*(val: string): seq[PublicKey] = +proc deserializePubKeys*(val: string): seq[SignPublicKey] = var val = val while val.len > 0: - result.add(val.nschop().PublicKey) + result.add(val.nschop().SignPublicKey) proc serialize*(s: seq[string]): string = for item in s: @@ -486,14 +486,14 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = return RelayMessage( kind: Data, resp_id: resp_id, - data_src: s.nsdecode(idx).PublicKey, + data_src: s.nsdecode(idx).SignPublicKey, data_val: s.nsdecode(idx), ) of Chunk: return RelayMessage( kind: Chunk, resp_id: resp_id, - chunk_src: s.nsdecode(idx).PublicKey, + chunk_src: s.nsdecode(idx).SignPublicKey, chunk_key: s.nsdecode(idx), chunk_val: if idx >= s.len: none[string]() @@ -504,7 +504,7 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = return RelayMessage( kind: ChunkStatus, resp_id: resp_id, - status_src: s.nsdecode(idx).PublicKey, + status_src: s.nsdecode(idx).SignPublicKey, present: deserialize(seq[string], s.nsdecode(idx)), absent: deserialize(seq[string], s.nsdecode(idx)), ) @@ -547,7 +547,7 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = return RelayCommand( kind: Iam, resp_id: resp_id, - iam_pubkey: s.nsdecode(idx).PublicKey, + iam_pubkey: s.nsdecode(idx).SignPublicKey, iam_answer: ChallengeAnswer.deserialize(s.nsdecode(idx)), ) of PublishNote: @@ -567,7 +567,7 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = return RelayCommand( kind: SendData, resp_id: resp_id, - send_dst: s.nsdecode(idx).PublicKey, + send_dst: s.nsdecode(idx).SignPublicKey, send_val: s.nsdecode(idx), ) of StoreChunk: @@ -582,13 +582,13 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = return RelayCommand( kind: GetChunks, resp_id: resp_id, - chunk_src: s.nsdecode(idx).PublicKey, + chunk_src: s.nsdecode(idx).SignPublicKey, chunk_keys: deserialize(seq[string], s.nsdecode(idx)), ) of HasChunks: return RelayCommand( kind: HasChunks, resp_id: resp_id, - has_src: s.nsdecode(idx).PublicKey, + has_src: s.nsdecode(idx).SignPublicKey, has_keys: deserialize(seq[string], s.nsdecode(idx)), ) diff --git a/src/bucketsrelay/v2/proto2.nim b/src/bucketsrelay/v2/proto2.nim index 00bec58..e839a9e 100644 --- a/src/bucketsrelay/v2/proto2.nim +++ b/src/bucketsrelay/v2/proto2.nim @@ -22,18 +22,18 @@ const TESTMODE = defined(testmode) and not defined(release) type KeyPair* = tuple - pk: PublicKey - sk: SecretKey + pk: SignPublicKey + sk: SignSecretKey Relay*[T] = object db*: DbConn - clients: TableRef[PublicKey, RelayConnection[T]] + clients: TableRef[SignPublicKey, RelayConnection[T]] max_chunk_space*: int max_transfer_rate*: int RelayConnection*[T] = ref object sender*: T - pubkey*: Option[PublicKey] ## The authenticated pubkey + pubkey*: Option[SignPublicKey] ## The authenticated pubkey challenge: Option[Challenge] relay*: Relay[T] ip*: string @@ -52,13 +52,13 @@ when TESTMODE: #------------------------------------------------------------------- proc genkeys*(): KeyPair = let (pk, sk) = crypto_sign_keypair() - result = (pk.PublicKey, sk.SecretKey) + result = (pk.SignPublicKey, sk.SignSecretKey) -proc sign*(key: SecretKey, message: string): string = +proc sign*(key: SignSecretKey, message: string): string = ## Sign a message with the given secret key result = crypto_sign_detached(key.string, message) -proc is_valid_signature*(key: PublicKey, plaintext: string, signature: string): bool = +proc is_valid_signature*(key: SignPublicKey, plaintext: string, signature: string): bool = try: crypto_sign_verify_detached(key.string, plaintext, signature) return true @@ -99,7 +99,7 @@ proc firstBits(s: string, n: int): string = raise ValueError.newException("String not long enough") return result -proc answer*(ch: Challenge, sk: SecretKey): ChallengeAnswer = +proc answer*(ch: Challenge, sk: SignSecretKey): ChallengeAnswer = ## Answer a hashcash challenge and sign the result var nonce = 0 let serialized = ch.serialize() @@ -121,7 +121,7 @@ proc answer*(ch: Challenge, sk: SecretKey): ChallengeAnswer = ) nonce.inc() -proc is_valid_answer*(pk: PublicKey, ch: Challenge, answer: ChallengeAnswer): bool = +proc is_valid_answer*(pk: SignPublicKey, ch: Challenge, answer: ChallengeAnswer): bool = ## Verify the signed challenge answer if not pk.is_valid_signature(sigContents(ch, answer.nonce, answer.output), answer.signature): return false @@ -143,11 +143,11 @@ func strval*(dbval: sqlite.DbValue): string = else: raise ValueError.newException("Can't get string from " & $dbval.kind) -proc dbValue*(p: PublicKey): DbValue = +proc dbValue*(p: SignPublicKey): DbValue = dbValue(p.string.DbBlob) -proc fromDB*(t: typedesc[PublicKey], v: DbBlob): PublicKey = - v.string.PublicKey +proc fromDB*(t: typedesc[SignPublicKey], v: DbBlob): SignPublicKey = + v.string.SignPublicKey template patch(db: untyped, applied: seq[string], name: string, body: untyped): untyped = block: @@ -254,7 +254,7 @@ proc `$`*[T](conn: RelayConnection[T]): string = result &= " cha=" & base64.encode(conn.challenge.get()) result &= ")" -proc `$`*[T](tab: TableRef[PublicKey, RelayConnection[T]]): string = +proc `$`*[T](tab: TableRef[SignPublicKey, RelayConnection[T]]): string = result = "TableRef(" for key in tab.keys(): let val = tab[key] @@ -265,7 +265,7 @@ proc newRelay*[T](db: DbConn): Relay[T] = when TESTMODE: resetSkew() result.db = db - result.clients = newTable[PublicKey, RelayConnection[T]]() + result.clients = newTable[SignPublicKey, RelayConnection[T]]() db.updateSchema() template sendError*[T](conn: RelayConnection[T], cmd: RelayCommand, msg: string, code: ErrorCode) = @@ -284,13 +284,13 @@ template sendOkay*[T](conn: RelayConnection[T], cmd: RelayCommand) = ok_cmd: cmd.kind, )) -proc is_valid*(x: PublicKey): bool = +proc is_valid*(x: SignPublicKey): bool = ## Return true if it looks like a valid public key if x.string.len == 32: return true return false -proc any_invalid(x: seq[PublicKey]): bool = +proc any_invalid(x: seq[SignPublicKey]): bool = ## Return true if any of the public keys are invalid for pk in x: if not pk.is_valid(): @@ -322,14 +322,14 @@ type data_in: int data_out: int ip: string - pubkey: PublicKey + pubkey: SignPublicKey period: string PeriodRange* = tuple a: string b: string -proc record_transfer_stat*(db: DbConn, ip: string, pubkey = "".PublicKey, data_in = 0, data_out = 0) = +proc record_transfer_stat*(db: DbConn, ip: string, pubkey = "".SignPublicKey, data_in = 0, data_out = 0) = db.exec(sql""" INSERT INTO stats_transfer (ip, pubkey, data_in, data_out) VALUES (?, ?, ?, ?) @@ -339,7 +339,7 @@ proc record_transfer_stat*(db: DbConn, ip: string, pubkey = "".PublicKey, data_i """, ip, pubkey, data_in, data_out) when TESTMODE: - proc record_transfer_stat_period*(db: DbConn, ip: string, pubkey = "".PublicKey, period = "", data_in = 0, data_out = 0) = + proc record_transfer_stat_period*(db: DbConn, ip: string, pubkey = "".SignPublicKey, period = "", data_in = 0, data_out = 0) = db.exec(sql""" INSERT INTO stats_transfer (ip, pubkey, period, data_in, data_out) VALUES (?, ?, ?, ?, ?) @@ -348,7 +348,7 @@ when TESTMODE: data_out = data_out + excluded.data_out; """, ip, pubkey, period, data_in, data_out) -proc record_event_stat*(db: DbConn, ip: string, pubkey: PublicKey, connect = 0, publish = 0, send = 0, store = 0) = +proc record_event_stat*(db: DbConn, ip: string, pubkey: SignPublicKey, connect = 0, publish = 0, send = 0, store = 0) = db.exec(sql""" INSERT INTO stats_event (ip, pubkey, connect, publish, send, store) VALUES (?, ?, ?, ?, ?, ?) @@ -359,13 +359,13 @@ proc record_event_stat*(db: DbConn, ip: string, pubkey: PublicKey, connect = 0, store = store + excluded.store """, ip, pubkey, connect, publish, send, store) -proc chunk_space_used*(db: DbConn, pubkey: PublicKey): int = +proc chunk_space_used*(db: DbConn, pubkey: SignPublicKey): int = ## Return the amount of space being used by the given public key db.getRow(sql""" SELECT coalesce(sum(length(val)), 0) FROM chunk WHERE src = ? """, pubkey).get()[0].i.int -proc current_data_in*(db: DbConn, pubkey: PublicKey): int = +proc current_data_in*(db: DbConn, pubkey: SignPublicKey): int = ## Return the amount of data that has been transferred in by the given ## public key for the current time period db.getRow(sql""" @@ -375,7 +375,7 @@ proc current_data_in*(db: DbConn, pubkey: PublicKey): int = AND period = strftime('%Y-%W') """, pubkey).get()[0].i.int -proc stats_transfer_total*(db: DbConn, ip = "", pubkey = "".PublicKey, period = ""): TransferTotal = +proc stats_transfer_total*(db: DbConn, ip = "", pubkey = "".SignPublicKey, period = ""): TransferTotal = var query = "SELECT sum(data_in), sum(data_out) FROM stats_transfer" var whereparts: seq[string] var params: seq[DbValue] @@ -426,7 +426,7 @@ proc popNote(relay: Relay, topic: string): Option[string] = warn &"[note] error " & getCurrentExceptionMsg() db.exec(sql"ROLLBACK") -proc noteCount(relay: Relay, pubkey: PublicKey): int = +proc noteCount(relay: Relay, pubkey: SignPublicKey): int = ## Return the number of notes currently published by this ip relay.db.getRow(sql"SELECT count(*) FROM note WHERE src = ?", pubkey).get()[0].i.int @@ -442,7 +442,7 @@ proc delExpiredMessages(relay: Relay) = let offstring = &"{offset} seconds" relay.db.exec(sql"DELETE FROM message WHERE created <= datetime('now', ?)", offstring) -proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = +proc nextMessage(relay: Relay, dst: SignPublicKey): Option[RelayMessage] = let orow = relay.db.getRow(sql""" SELECT src, data, id FROM message @@ -457,7 +457,7 @@ proc nextMessage(relay: Relay, dst: PublicKey): Option[RelayMessage] = result = some(RelayMessage( kind: Data, resp_id: 0, # Data messages are not triggered by recipient's command - data_src: PublicKey.fromDB(row[0].b), + data_src: SignPublicKey.fromDB(row[0].b), data_val: row[1].b.string, )) relay.db.exec(sql"DELETE FROM message WHERE id=?", row[2].i) @@ -644,7 +644,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay INSERT OR REPLACE INTO chunk (last_used, src, key, val) VALUES (datetime('now', ?), ?, ?, ?) """, offset, pubkey, cmd.chunk_key.DbBlob, cmd.chunk_val.DbBlob) - var dsts: seq[PublicKey] + var dsts: seq[SignPublicKey] dsts.add(cmd.chunk_dst) if pubkey notin dsts: dsts.add(pubkey) diff --git a/src/bucketsrelay/v2/sampleclient.nim b/src/bucketsrelay/v2/sampleclient.nim index cb3da9e..f3e3f3a 100644 --- a/src/bucketsrelay/v2/sampleclient.nim +++ b/src/bucketsrelay/v2/sampleclient.nim @@ -76,7 +76,7 @@ proc fetchNote*(ns: NetstringClient, topic: string): Future[string] {.async.} = else: raise ValueError.newException("No such note: " & topic) -proc sendData*(ns: NetstringClient, dst: PublicKey, val: string) {.async.} = +proc sendData*(ns: NetstringClient, dst: SignPublicKey, val: string) {.async.} = await ns.sendCommand(RelayCommand( kind: SendData, send_dst: dst, @@ -90,7 +90,7 @@ proc getData*(ns: NetstringClient): Future[string] {.async.} = else: raise ValueError.newException("Expecting Data but got: " & $res) -proc storeChunk*(ns: NetstringClient, dsts: seq[PublicKey], key: string, val: string) {.async.} = +proc storeChunk*(ns: NetstringClient, dsts: seq[SignPublicKey], key: string, val: string) {.async.} = await ns.sendCommand(RelayCommand( kind: StoreChunk, chunk_dst: dsts, @@ -98,7 +98,7 @@ proc storeChunk*(ns: NetstringClient, dsts: seq[PublicKey], key: string, val: st chunk_val: val, )) -proc getChunk*(ns: NetstringClient, src: PublicKey, key: string): Future[Option[string]] {.async.} = +proc getChunk*(ns: NetstringClient, src: SignPublicKey, key: string): Future[Option[string]] {.async.} = await ns.sendCommand(RelayCommand( kind: GetChunks, chunk_src: src, @@ -110,7 +110,7 @@ proc getChunk*(ns: NetstringClient, src: PublicKey, key: string): Future[Option[ else: raise ValueError.newException("Expecting Chunk but got: " & $res) -proc hasChunk*(ns: NetstringClient, src: PublicKey, key: string): Future[bool] {.async.} = +proc hasChunk*(ns: NetstringClient, src: SignPublicKey, key: string): Future[bool] {.async.} = await ns.sendCommand(RelayCommand( kind: HasChunks, has_src: src, diff --git a/src/bucketsrelay/v2/server2.nim b/src/bucketsrelay/v2/server2.nim index 20abb74..60f4a59 100644 --- a/src/bucketsrelay/v2/server2.nim +++ b/src/bucketsrelay/v2/server2.nim @@ -22,7 +22,7 @@ type buf: string socket: WebSocket ip: string - pubkey: Option[PublicKey] + pubkey: Option[SignPublicKey] QueuedMessage = tuple socket: NetstringSocket @@ -140,13 +140,13 @@ proc handleWebsocket(req: Request) {.async, gcsafe.} = type StorageStat = tuple - pubkey: PublicKey + pubkey: SignPublicKey message_size: int chunk_size: int total_size: int PubkeyEventStat = tuple - pubkey: PublicKey + pubkey: SignPublicKey count: int IPEventStat = tuple @@ -227,7 +227,7 @@ router myrouter: data_in: row[0].i.int, data_out: row[1].i.int, ip: row[3].s, - pubkey: default(PublicKey), + pubkey: default(SignPublicKey), period: "", )) @@ -252,7 +252,7 @@ router myrouter: data_in: row[0].i.int, data_out: row[1].i.int, ip: "", - pubkey: PublicKey.fromDB(row[3].b), + pubkey: SignPublicKey.fromDB(row[3].b), period: "", )) @@ -281,7 +281,7 @@ router myrouter: LIMIT 10; """): storage_by_pubkey.add(( - pubkey: PublicKey.fromDb(row[0].b), + pubkey: SignPublicKey.fromDb(row[0].b), message_size: row[1].i.int, chunk_size: row[2].i.int, total_size: row[3].i.int, @@ -303,7 +303,7 @@ router myrouter: LIMIT 10 """, datarange.a): connects_by_pubkey.add(( - pubkey: PublicKey.fromDb(row[0].b), + pubkey: SignPublicKey.fromDb(row[0].b), count: row[1].i.int, )) @@ -322,7 +322,7 @@ router myrouter: LIMIT 10 """, datarange.a): publish_by_pubkey.add(( - pubkey: PublicKey.fromDb(row[0].b), + pubkey: SignPublicKey.fromDb(row[0].b), count: row[1].i.int, )) @@ -341,7 +341,7 @@ router myrouter: LIMIT 10 """, datarange.a): send_by_pubkey.add(( - pubkey: PublicKey.fromDb(row[0].b), + pubkey: SignPublicKey.fromDb(row[0].b), count: row[1].i.int, )) @@ -360,7 +360,7 @@ router myrouter: LIMIT 10 """, datarange.a): store_by_pubkey.add(( - pubkey: PublicKey.fromDb(row[0].b), + pubkey: SignPublicKey.fromDb(row[0].b), count: row[1].i.int, )) diff --git a/tests/tproto2.nim b/tests/tproto2.nim index d08f1ab..68081c6 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -21,8 +21,8 @@ else: type TestClient* = ref object received: Deque[RelayMessage] - pk: PublicKey - sk: SecretKey + pk: SignPublicKey + sk: SignSecretKey proc `$`*(tc: TestClient): string = $tc[] @@ -63,8 +63,8 @@ proc pop(conn: var RelayConnection[TestClient], expected: MessageKind): RelayMes proc msgCount(conn: var RelayConnection[TestClient]): int = conn.sender.received.len -proc pk(conn: var RelayConnection[TestClient]): PublicKey = conn.sender.pk -proc sk(conn: var RelayConnection[TestClient]): SecretKey = conn.sender.sk +proc pk(conn: var RelayConnection[TestClient]): SignPublicKey = conn.sender.pk +proc sk(conn: var RelayConnection[TestClient]): SignSecretKey = conn.sender.sk proc keys(conn: var RelayConnection[TestClient]): KeyPair = (conn.sender.pk, conn.sender.sk) proc anonConn(relay: Relay): RelayConnection[TestClient] = @@ -494,7 +494,7 @@ suite "data": var alice = relay.authenticatedConn() relay.handleCommand(alice, RelayCommand( kind: SendData, - send_dst: "invalid".PublicKey, + send_dst: "invalid".SignPublicKey, send_val: "a", )) let err = alice.pop(Error) @@ -502,7 +502,7 @@ suite "data": check err.err_cmd == SendData -proc storeChunk(conn: var RelayConnection[TestClient], key: string, val: string, dst = newSeq[PublicKey]()) = +proc storeChunk(conn: var RelayConnection[TestClient], key: string, val: string, dst = newSeq[SignPublicKey]()) = conn.relay.handleCommand(conn, RelayCommand( kind: StoreChunk, chunk_dst: dst, @@ -707,7 +707,7 @@ suite "chunks": test "max dst.len": let relay = testRelay() var alice = relay.authenticatedConn() - var dsts: seq[PublicKey] + var dsts: seq[SignPublicKey] for i in 0..(RELAY_MAX_CHUNK_DSTS+1): dsts.add(genkeys().pk) relay.handleCommand(alice, RelayCommand( @@ -748,7 +748,7 @@ suite "chunks": var alice = relay.authenticatedConn() relay.handleCommand(alice, RelayCommand( kind: StoreChunk, - chunk_dst: @["fake".PublicKey], + chunk_dst: @["fake".SignPublicKey], chunk_key: "a", chunk_val: "b", )) @@ -831,25 +831,25 @@ suite "stats": test "transfer basics": let db = open(":memory:", "", "", "") db.updateSchema() - db.record_transfer_stat("ip1", "pubkey".PublicKey, data_in = 1000, data_out = 500) - db.record_transfer_stat("ip1", "pubkey".PublicKey, data_in = 2000, data_out = 250) - db.record_transfer_stat("ip2", "pubkey".PublicKey, data_in = 3000, data_out = 100) - db.record_transfer_stat("ip1", "pubkey2".PublicKey, data_in = 500, data_out = 100) + db.record_transfer_stat("ip1", "pubkey".SignPublicKey, data_in = 1000, data_out = 500) + db.record_transfer_stat("ip1", "pubkey".SignPublicKey, data_in = 2000, data_out = 250) + db.record_transfer_stat("ip2", "pubkey".SignPublicKey, data_in = 3000, data_out = 100) + db.record_transfer_stat("ip1", "pubkey2".SignPublicKey, data_in = 500, data_out = 100) - check db.stats_transfer_total(ip="ip1") == (1000+2000+500, 500+250+100, "ip1", "".PublicKey, "") - check db.stats_transfer_total(pubkey="pubkey".PublicKey) == (1000+2000+3000, 500+250+100, "", "pubkey".PublicKey, "") - check db.stats_transfer_total(pubkey="pubkey2".PublicKey) == (500, 100, "", "pubkey2".PublicKey, "") + check db.stats_transfer_total(ip="ip1") == (1000+2000+500, 500+250+100, "ip1", "".SignPublicKey, "") + check db.stats_transfer_total(pubkey="pubkey".SignPublicKey) == (1000+2000+3000, 500+250+100, "", "pubkey".SignPublicKey, "") + check db.stats_transfer_total(pubkey="pubkey2".SignPublicKey) == (500, 100, "", "pubkey2".SignPublicKey, "") test "transfer timeperiods": let db = open(":memory:", "", "", "") db.updateSchema() - db.record_transfer_stat_period("ip1", "pubkey".PublicKey, "2010-01", data_in = 1000, data_out = 500) - db.record_transfer_stat_period("ip1", "pubkey".PublicKey, "2010-01", data_in = 2000, data_out = 250) - db.record_transfer_stat_period("ip2", "pubkey".PublicKey, "2010-02", data_in = 3000, data_out = 100) - db.record_transfer_stat_period("ip1", "pubkey2".PublicKey, "2010-02", data_in = 500, data_out = 100) + db.record_transfer_stat_period("ip1", "pubkey".SignPublicKey, "2010-01", data_in = 1000, data_out = 500) + db.record_transfer_stat_period("ip1", "pubkey".SignPublicKey, "2010-01", data_in = 2000, data_out = 250) + db.record_transfer_stat_period("ip2", "pubkey".SignPublicKey, "2010-02", data_in = 3000, data_out = 100) + db.record_transfer_stat_period("ip1", "pubkey2".SignPublicKey, "2010-02", data_in = 500, data_out = 100) - check db.stats_transfer_total(period="2010-01") == (1000+2000, 500+250, "", "".PublicKey, "2010-01") - check db.stats_transfer_total(period="2010-02") == (3000+500, 100+100, "", "".PublicKey, "2010-02") + check db.stats_transfer_total(period="2010-01") == (1000+2000, 500+250, "", "".SignPublicKey, "2010-01") + check db.stats_transfer_total(period="2010-02") == (3000+500, 100+100, "", "".SignPublicKey, "2010-02") suite "resp_id": diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 6d30580..088fbc2 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -21,9 +21,9 @@ test "RelayMessage": of Okay: RelayMessage(kind: Okay, resp_id: 42, ok_cmd: SendData) of Error: RelayMessage(kind: Error, resp_id: 123, err_cmd: SendData, err_code: TooLarge, err_message: "foo") of Note: RelayMessage(kind: Note, resp_id: 456, note_topic: "something", note_data: "data") - of Data: RelayMessage(kind: Data, resp_id: 0, data_src: "hey".PublicKey, data_val: "foo") - of Chunk: RelayMessage(kind: Chunk, resp_id: 789, chunk_src: "hey".PublicKey, chunk_key: "key", chunk_val: some("theval")) - of ChunkStatus: RelayMessage(kind: ChunkStatus, resp_id: 999, status_src: "a".PublicKey, present: @["foo"], absent: @["bar"]) + of Data: RelayMessage(kind: Data, resp_id: 0, data_src: "hey".SignPublicKey, data_val: "foo") + of Chunk: RelayMessage(kind: Chunk, resp_id: 789, chunk_src: "hey".SignPublicKey, chunk_key: "key", chunk_val: some("theval")) + of ChunkStatus: RelayMessage(kind: ChunkStatus, resp_id: 999, status_src: "a".SignPublicKey, present: @["foo"], absent: @["bar"]) let serialized = example.serialize() info $example info "serialized: " & serialized.nice @@ -35,7 +35,7 @@ test "RelayCommand": of Iam: RelayCommand( kind: Iam, resp_id: 1, - iam_pubkey: "hey".PublicKey, + iam_pubkey: "hey".SignPublicKey, iam_answer: ( nonce: 1, output: "foo", @@ -44,19 +44,19 @@ test "RelayCommand": ) of PublishNote: RelayCommand(kind: PublishNote, resp_id: 100, pub_topic: "topic", pub_data: "data") of FetchNote: RelayCommand(kind: FetchNote, resp_id: 200, fetch_topic: "topic") - of SendData: RelayCommand(kind: SendData, resp_id: 300, send_dst: "one".PublicKey, send_val: "data") + of SendData: RelayCommand(kind: SendData, resp_id: 300, send_dst: "one".SignPublicKey, send_val: "data") of StoreChunk: RelayCommand( kind: StoreChunk, resp_id: 400, - chunk_dst: @["one".PublicKey], + chunk_dst: @["one".SignPublicKey], chunk_key: "theky", chunk_val: "someval" ) - of GetChunks: RelayCommand(kind: GetChunks, resp_id: 500, chunk_src: "hey".PublicKey, chunk_keys: @["foo", "bar"]) + of GetChunks: RelayCommand(kind: GetChunks, resp_id: 500, chunk_src: "hey".SignPublicKey, chunk_keys: @["foo", "bar"]) of HasChunks: RelayCommand( kind: HasChunks, resp_id: 600, - has_src: "hey".PublicKey, + has_src: "hey".SignPublicKey, has_keys: @["foo", "Bar"], ) let serialized = example.serialize() @@ -67,7 +67,7 @@ test "RelayCommand": test "Chunk with none": let chunk = RelayMessage(kind: Chunk, resp_id: 888, - chunk_src: "foo".PublicKey, + chunk_src: "foo".SignPublicKey, chunk_key: "key", chunk_val: none[string](), ) @@ -96,5 +96,5 @@ test "resp_id serialization": let cmd1 = RelayCommand(kind: PublishNote, resp_id: 54321, pub_topic: "topic", pub_data: "data") check RelayCommand.deserialize(cmd1.serialize()).resp_id == 54321 - let cmd2 = RelayCommand(kind: SendData, resp_id: 0, send_dst: "dst".PublicKey, send_val: "val") + let cmd2 = RelayCommand(kind: SendData, resp_id: 0, send_dst: "dst".SignPublicKey, send_val: "val") check RelayCommand.deserialize(cmd2.serialize()).resp_id == 0 From b560ba20deefc81c11c2e27d1f761640eb430207 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Wed, 10 Dec 2025 10:22:34 -0500 Subject: [PATCH 43/46] Chunks are gone in favor of overwriteable Data messages --- README.md | 40 +- .../new-Chunks-are-gone-20251210-102219.md | 1 + src/bucketsrelay/v2/cli.nim | 40 +- src/bucketsrelay/v2/objs.nim | 158 +----- src/bucketsrelay/v2/proto2.nim | 223 ++------ src/bucketsrelay/v2/sampleclient.nim | 39 +- src/bucketsrelay/v2/server2.nim | 64 +-- src/bucketsrelay/v2/templates/index.nimja | 4 +- src/bucketsrelay/v2/templates/stats.nimja | 24 +- tests/tfunctional.nim | 47 +- tests/tproto2.nim | 497 +++++++----------- tests/tserde2.nim | 34 +- 12 files changed, 317 insertions(+), 854 deletions(-) create mode 100644 changes/new-Chunks-are-gone-20251210-102219.md diff --git a/README.md b/README.md index 383ac09..0447336 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,6 @@ Clients send the following commands: | `PublishNote` | Send a few bytes to another client addressed by topic (good for key exchange) | | `FetchNote` | Request a note addressed by topic | | `SendData` | Store/forward bytes to other clients, addressed by relay-authenticated public keys | -| `StoreChunk` | Store bytes for other clients to fetch addressed by key and public key. | -| `GetChunk` | Request stored chunk | -| `ChunksPresent` | Ask which chunks exist | - ### Server Events @@ -75,8 +71,6 @@ The relay server sends the following events: | `Who` | Challenge for authenticating a client's public/private keys and spam mitigation | | `Note` | Data payload of a note requested by `FetchNote` | | `Data` | Data payload from another client, addressed by relay-authenticated public key | -| `Chunk` | Data payload response to `GetChunk` request | -| `ChunkStatus` | Response to `ChunksPresent` indicating which chunks exist/don't | ### Authentication @@ -102,13 +96,12 @@ Client Relay ### Data -There are 3 ways clients can exchange data: +There are 2 ways clients can exchange data: 1. Notes - public notes that are accessed by knowing the note *topic*. Notes are a good way to do key exchange. Notes expire after a short time. -2. Messages - ordered, stored-and-forwarded messages sent from one client to another client. These are automatically sent to a client upon connection, and deleted when sent. Messages expire after a while. -3. Chunks - clients store chunks with a string *key* and choose which clients (by their public key) are allowed to fetch uploaded chunks. Chunks may be overwritten. Chunks expire a while after their last update. +2. Messages - ordered, stored-and-forwarded messages sent from one client to another client. These are automatically sent to a client upon connection, and deleted when sent. Messages expire after a while. Messages with a non-blank key will overwrite undelivered messages with the same key. -All forms of exchanging data are unreliable. Build with that in mind. +Build relay clients with the understanding that all forms of exchanging data through this relay are unreliable. #### Notes @@ -146,6 +139,33 @@ Alice Relay Bob │ │ │ ``` +#### Messages with keys + +1. Alice sends `SendData(dst=BOBPK, key=apple, data=core)` +2. Alice sends `SendData(dst=BOBPK, key=banana, data=boat)` +3. Alice sends `SendData(dst=BOBPK, key=apple, data=pie)` (replacing prior `key=apple` message) +4. Bob connects +5. Server sends to Bob `Data(src=ALICEPK, key=banana, data=boat)` +6. Server sends to Bob `Data(src=ALICEPK, key=apple, data=pie)` + +``` +Alice Relay Bob + │ │ │ + ├───Authenticated─┤ │ + │ │ │ + │ SendData(Bob,1) │ │ + ├────────────────►│ │ + │ SendData(Bob,2) │ │ + ├────────────────►│ │ + │ SendData(Bob,1) │ │ + ├────────────────►│ │ + │ │ Data(Alice, 2) │ + │ ├────────────────►│ + │ │ Data(Alice, 1) │ + │ ├────────────────►│ + │ │ │ +``` + #### Chunks 1. Alice sends `StoreChunk(dst=[BOBPK], key=apple, val=seed)` diff --git a/changes/new-Chunks-are-gone-20251210-102219.md b/changes/new-Chunks-are-gone-20251210-102219.md new file mode 100644 index 0000000..f53efcb --- /dev/null +++ b/changes/new-Chunks-are-gone-20251210-102219.md @@ -0,0 +1 @@ +Chunks are gone in favor of overwriteable Data messages diff --git a/src/bucketsrelay/v2/cli.nim b/src/bucketsrelay/v2/cli.nim index 559a5cf..057e01e 100644 --- a/src/bucketsrelay/v2/cli.nim +++ b/src/bucketsrelay/v2/cli.nim @@ -115,48 +115,22 @@ proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) if dst.string == "": dst = SignPublicKey.deserialize(i.use(args)) let val = i.use(args) - waitFor client.sendData(dst, val) + let key = if i < args.len: + i.use(args) + else: + "" + waitFor client.sendData(@[dst], val, key) of "recv": let data = waitFor client.getData() echo data - of "store": - var dst = ctx.dst - if dst.string == "" or args.len >= 3: - dst = SignPublicKey.deserialize(i.use(args)) - echo "Using key=" & dst.nice - let key = i.use(args) - let val = i.use(args) - waitFor client.storeChunk(@[dst], key, val) - of "get": - var src = ctx.dst - if src.string == "" or args.len >= 2: - src = SignPublicKey.deserialize(i.use(args)) - echo "Using key=" & src.serialize - let key = i.use(args) - let odata = waitFor client.getChunk(src, key) - if odata.isSome: - echo odata.get() - else: - echo "(none)" - of "has": - var src = ctx.dst - if src.string == "" or args.len >= 2: - src = SignPublicKey.deserialize(i.use(args)) - echo "Using key=" & src.serialize - let key = i.use(args) - let res = waitFor client.hasChunk(src, key) - echo $res of "help": echo """ post TOPIC DATA fetch TOPIC dst PUBKEY - Set the destination PUBKEY for future commands - send [PUBKEY] DATA + Set the destination PUBKEY for future sends + send [PUBKEY] DATA [KEY] recv - store [PUBKEY] KEY VAL - get [PUBKEY] KEY - has [PUBKEY] KEY help """ else: diff --git a/src/bucketsrelay/v2/objs.nim b/src/bucketsrelay/v2/objs.nim index e801027..70e6bea 100644 --- a/src/bucketsrelay/v2/objs.nim +++ b/src/bucketsrelay/v2/objs.nim @@ -7,7 +7,7 @@ ## This file should be kept free of dependencies other than the stdlib ## and should not include async stuff ## as it's meant to be referenced by outside libraries that may -## want to do things there own way. +## want to do things their own way. import std/hashes import std/options @@ -36,8 +36,6 @@ type Error Note Data - Chunk - ChunkStatus ErrorCode* = enum Generic = 0 @@ -63,25 +61,15 @@ type note_topic*: string note_data*: string of Data: + data_key*: string data_src*: SignPublicKey data_val*: string - of Chunk: - chunk_src*: SignPublicKey - chunk_key*: string - chunk_val*: Option[string] - of ChunkStatus: - status_src*: SignPublicKey - present*: seq[string] - absent*: seq[string] CommandKind* = enum Iam PublishNote FetchNote SendData - StoreChunk - GetChunks - HasChunks RelayCommand* = object resp_id*: int @@ -95,30 +83,18 @@ type of FetchNote: fetch_topic*: string of SendData: - send_dst*: SignPublicKey + send_key*: string + send_dst*: seq[SignPublicKey] send_val*: string - of StoreChunk: - chunk_dst*: seq[SignPublicKey] - chunk_key*: string - chunk_val*: string - of GetChunks: - chunk_src*: SignPublicKey - chunk_keys*: seq[string] - of HasChunks: - has_src*: SignPublicKey - has_keys*: seq[string] const RELAY_MAX_TOPIC_SIZE* = 512 - RELAY_MAX_NOTE_SIZE* = 4096 + RELAY_MAX_NOTE_SIZE* = 4096 * 2 RELAY_MAX_NOTES* = 1000 RELAY_NOTE_DURATION* = 5 * 24 * 60 * 60 - RELAY_MAX_MESSAGE_SIZE* = 4096 - RELAY_MAX_CHUNK_KEY_SIZE* = 4096 - RELAY_MAX_CHUNK_SIZE* = 65536 - RELAY_MAX_CHUNK_DSTS* = 32 + RELAY_MAX_MESSAGE_SIZE* = 65536 * 2 + RELAY_MAX_KEY_SIZE* = 4096 RELAY_MESSAGE_DURATION* = 30 * 24 * 60 * 60 - RELAY_PUBKEY_MEMORY_SECONDS* = 60 * 24 * 60 * 60 const nicestart = 'a' # '!' @@ -175,17 +151,9 @@ proc `$`*(msg: RelayMessage): string = of Error: result.add &"cmd={msg.err_cmd} code={msg.err_code} msg={msg.err_message.nice}" of Note: - result.add &"'{msg.note_topic.nice}' val={msg.note_data.nicelong}" + result.add &"'{msg.note_topic.nice.abbr}' val={msg.note_data.nicelong}" of Data: - result.add &"{msg.data_src.nice.abbr} val={msg.data_val.nicelong}" - of Chunk: - result.add &"{msg.chunk_src.nice.abbr} {msg.chunk_key.nice.abbr}={msg.chunk_val.nicelong}" - of ChunkStatus: - result.add &"{msg.status_src.nice.abbr} present=[" - result.add msg.present.mapIt(it.nice.abbr).join(", ") - result.add "] absent=[" - result.add msg.absent.mapIt(it.nice.abbr).join(", ") - result.add "]" + result.add &"'{msg.data_key.nice.abbr}' src={msg.data_src.nice.abbr} val={msg.data_val.nicelong}" result.add ")" proc `==`*(a, b: RelayMessage): bool = @@ -202,11 +170,7 @@ proc `==`*(a, b: RelayMessage): bool = of Note: return a.note_data == b.note_data and a.note_topic == b.note_topic of Data: - return a.data_src == b.data_src and a.data_val == b.data_val - of Chunk: - return a.chunk_src == b.chunk_src and a.chunk_key == b.chunk_key and a.chunk_val == b.chunk_val - of ChunkStatus: - return a.status_src == b.status_src and a.present == b.present and a.absent == b.absent + return a.data_src == b.data_src and a.data_val == b.data_val and a.data_key == b.data_key proc `$`*(cmd: RelayCommand): string = result.add $cmd.kind & "(" @@ -218,19 +182,8 @@ proc `$`*(cmd: RelayCommand): string = of FetchNote: result.add &"'{cmd.fetch_topic.nice.abbr}'" of SendData: - result.add &"{cmd.send_dst.nice.abbr} val={cmd.send_val.nicelong}" - of StoreChunk: - result.add &"{cmd.chunk_key.nice.abbr}={cmd.chunk_val.nicelong} dst=[" - result.add cmd.chunk_dst.mapIt(it.nice.abbr).join(", ") - result.add "]" - of GetChunks: - result.add &"{cmd.chunk_src.nice.abbr} keys=[" - result.add cmd.chunk_keys.mapIt(it.nice.abbr).join(", ") - result.add "]" - of HasChunks: - result.add &"{cmd.has_src.nice.abbr} keys=[" - result.add cmd.has_keys.mapIt(it.nice.abbr).join(", ") - result.add "]" + result.add &"key={cmd.send_key.nice.abbr} val={cmd.send_val.nicelong} " + result.add cmd.send_dst.mapIt(it.nice.abbr).join(", ") result.add ")" proc `==`*(a, b: RelayCommand): bool = @@ -245,13 +198,7 @@ proc `==`*(a, b: RelayCommand): bool = of FetchNote: return a.fetch_topic == b.fetch_topic of SendData: - return a.send_dst == b.send_dst and a.send_val == b.send_val - of StoreChunk: - return a.chunk_dst == b.chunk_dst and a.chunk_key == b.chunk_key and a.chunk_val == b.chunk_val - of GetChunks: - return a.chunk_src == b.chunk_src and a.chunk_keys == b.chunk_keys - of HasChunks: - return a.has_src == b.has_src and a.has_keys == b.has_keys + return a.send_dst == b.send_dst and a.send_val == b.send_val and a.send_key == b.send_key #-------------------------------------------------------------- # serialization @@ -331,8 +278,6 @@ proc serialize*(kind: MessageKind): char = of Error: '-' of Note: 'n' of Data: 'd' - of Chunk: 'k' - of ChunkStatus: 's' proc deserialize*(kind: typedesc[MessageKind], val: char): MessageKind = case val @@ -341,8 +286,6 @@ proc deserialize*(kind: typedesc[MessageKind], val: char): MessageKind = of '-': Error of 'n': Note of 'd': Data - of 'k': Chunk - of 's': ChunkStatus else: raise ValueError.newException("Unknown MessageKind: " & val) proc serialize*(kind: CommandKind): char = @@ -351,9 +294,6 @@ proc serialize*(kind: CommandKind): char = of PublishNote: 'p' of FetchNote: 'f' of SendData: 's' - of StoreChunk: 'c' - of GetChunks: 'g' - of HasChunks: 't' proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = case val: @@ -361,9 +301,6 @@ proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = of 'p': PublishNote of 'f': FetchNote of 's': SendData - of 'c': StoreChunk - of 'g': GetChunks - of 't': HasChunks else: raise ValueError.newException("Unknown CommandKind: " & val) proc serialize*(err: ErrorCode): char = @@ -438,18 +375,9 @@ proc serialize*(msg: RelayMessage): string = result &= msg.note_topic.nsencode result &= msg.note_data.nsencode of Data: + result &= msg.data_key.nsencode result &= msg.data_src.string.nsencode result &= msg.data_val.nsencode - of Chunk: - result &= msg.chunk_src.string.nsencode - result &= msg.chunk_key.nsencode - if msg.chunk_val.isSome: - result &= msg.chunk_val.get().nsencode - of ChunkStatus: - result &= msg.status_src.string.nsencode - result &= nsencode(msg.present.serialize()) - result &= nsencode(msg.absent.serialize()) - proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = if s.len == 0: @@ -486,28 +414,10 @@ proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = return RelayMessage( kind: Data, resp_id: resp_id, + data_key: s.nsdecode(idx), data_src: s.nsdecode(idx).SignPublicKey, data_val: s.nsdecode(idx), ) - of Chunk: - return RelayMessage( - kind: Chunk, - resp_id: resp_id, - chunk_src: s.nsdecode(idx).SignPublicKey, - chunk_key: s.nsdecode(idx), - chunk_val: if idx >= s.len: - none[string]() - else: - some(s.nsdecode(idx)), - ) - of ChunkStatus: - return RelayMessage( - kind: ChunkStatus, - resp_id: resp_id, - status_src: s.nsdecode(idx).SignPublicKey, - present: deserialize(seq[string], s.nsdecode(idx)), - absent: deserialize(seq[string], s.nsdecode(idx)), - ) proc serialize*(cmd: RelayCommand): string = result &= cmd.kind.serialize @@ -522,18 +432,9 @@ proc serialize*(cmd: RelayCommand): string = of FetchNote: result &= cmd.fetch_topic.nsencode of SendData: - result &= cmd.send_dst.string.nsencode + result &= cmd.send_key.nsencode + result &= nsencode(cmd.send_dst.serialize()) result &= cmd.send_val.nsencode - of StoreChunk: - result &= nsencode(cmd.chunk_dst.serialize()) - result &= cmd.chunk_key.nsencode - result &= cmd.chunk_val.nsencode - of GetChunks: - result &= cmd.chunk_src.string.nsencode - result &= nsencode(cmd.chunk_keys.serialize()) - of HasChunks: - result &= cmd.has_src.string.nsencode - result &= nsencode(cmd.has_keys.serialize()) proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = if s.len == 0: @@ -567,28 +468,7 @@ proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = return RelayCommand( kind: SendData, resp_id: resp_id, - send_dst: s.nsdecode(idx).SignPublicKey, + send_key: s.nsdecode(idx), + send_dst: deserializePubKeys(s.nsdecode(idx)), send_val: s.nsdecode(idx), ) - of StoreChunk: - return RelayCommand( - kind: StoreChunk, - resp_id: resp_id, - chunk_dst: deserializePubKeys(s.nsdecode(idx)), - chunk_key: s.nsdecode(idx), - chunk_val: s.nsdecode(idx), - ) - of GetChunks: - return RelayCommand( - kind: GetChunks, - resp_id: resp_id, - chunk_src: s.nsdecode(idx).SignPublicKey, - chunk_keys: deserialize(seq[string], s.nsdecode(idx)), - ) - of HasChunks: - return RelayCommand( - kind: HasChunks, - resp_id: resp_id, - has_src: s.nsdecode(idx).SignPublicKey, - has_keys: deserialize(seq[string], s.nsdecode(idx)), - ) diff --git a/src/bucketsrelay/v2/proto2.nim b/src/bucketsrelay/v2/proto2.nim index e839a9e..fe97346 100644 --- a/src/bucketsrelay/v2/proto2.nim +++ b/src/bucketsrelay/v2/proto2.nim @@ -17,7 +17,7 @@ import libsodium/sodium_sizes import ./objs; export objs -const LOG_COMMS* = not defined(release) +const LOG_COMMS* = not defined(release) or defined(relaynologcomms) const TESTMODE = defined(testmode) and not defined(release) type @@ -196,29 +196,16 @@ proc updateSchema*(db: DbConn) = db.exec(sql"""CREATE TABLE message ( id INTEGER PRIMARY KEY AUTOINCREMENT, created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + key BLOB NOT NULL, src TEXT NOT NULL, dst TEXT NOT NULL, data BLOB NOT NULL )""") db.exec(sql"CREATE INDEX message_created ON message(created)") - db.exec(sql"CREATE INDEX message_dst ON message(dst)") - - # chunks - db.exec(sql"""CREATE TABLE chunk ( - src TEXT NOT NULL, - key TEXT NOT NULL, - last_used TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - val BLOB NOT NULL, - PRIMARY KEY (src, key) - )""") - db.exec(sql"CREATE INDEX chunk_last_used ON chunk(last_used)") - db.exec(sql"""CREATE TABLE chunk_dst ( - src TEXT NOT NULL, - key TEXT NOT NULL, - dst TEXT NOT NULL, - PRIMARY KEY (src, key, dst), - FOREIGN KEY (src, key) REFERENCES chunk(src, key) ON DELETE CASCADE - )""") + db.exec(sql"""CREATE UNIQUE INDEX message_dst_key + ON message(dst, key) + WHERE key IS NOT x'' + """) # stats db.exec(sql"""CREATE TABLE stats_transfer ( @@ -237,7 +224,6 @@ proc updateSchema*(db: DbConn) = connect INTEGER DEFAULT 0, publish INTEGER DEFAULT 0, send INTEGER DEFAULT 0, - store INTEGER DEFAULT 0, PRIMARY KEY (period, ip, pubkey) )""") @@ -348,22 +334,15 @@ when TESTMODE: data_out = data_out + excluded.data_out; """, ip, pubkey, period, data_in, data_out) -proc record_event_stat*(db: DbConn, ip: string, pubkey: SignPublicKey, connect = 0, publish = 0, send = 0, store = 0) = +proc record_event_stat*(db: DbConn, ip: string, pubkey: SignPublicKey, connect = 0, publish = 0, send = 0) = db.exec(sql""" - INSERT INTO stats_event (ip, pubkey, connect, publish, send, store) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO stats_event (ip, pubkey, connect, publish, send) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(period, ip, pubkey) DO UPDATE SET connect = connect + excluded.connect, publish = publish + excluded.publish, - send = send + excluded.send, - store = store + excluded.store - """, ip, pubkey, connect, publish, send, store) - -proc chunk_space_used*(db: DbConn, pubkey: SignPublicKey): int = - ## Return the amount of space being used by the given public key - db.getRow(sql""" - SELECT coalesce(sum(length(val)), 0) FROM chunk WHERE src = ? - """, pubkey).get()[0].i.int + send = send + excluded.send + """, ip, pubkey, connect, publish, send) proc current_data_in*(db: DbConn, pubkey: SignPublicKey): int = ## Return the amount of data that has been transferred in by the given @@ -444,7 +423,7 @@ proc delExpiredMessages(relay: Relay) = proc nextMessage(relay: Relay, dst: SignPublicKey): Option[RelayMessage] = let orow = relay.db.getRow(sql""" - SELECT src, data, id + SELECT key, src, data, id FROM message WHERE dst = ? @@ -457,10 +436,11 @@ proc nextMessage(relay: Relay, dst: SignPublicKey): Option[RelayMessage] = result = some(RelayMessage( kind: Data, resp_id: 0, # Data messages are not triggered by recipient's command - data_src: SignPublicKey.fromDB(row[0].b), - data_val: row[1].b.string, + data_key: row[0].b.string, + data_src: SignPublicKey.fromDB(row[1].b), + data_val: row[2].b.string, )) - relay.db.exec(sql"DELETE FROM message WHERE id=?", row[2].i) + relay.db.exec(sql"DELETE FROM message WHERE id=?", row[3].i) proc delExpiredChunks(relay: Relay) = let offset = when TESTMODE: @@ -521,7 +501,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay relay.db.record_transfer_stat( ip = conn.ip, pubkey = pubkey, - data_out = msg.data_val.len, + data_out = msg.data_val.len + msg.data_key.len, ) else: break @@ -579,7 +559,9 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay of SendData: if cmd.send_val.len > RELAY_MAX_MESSAGE_SIZE: conn.sendError(cmd, "Data too long", TooLarge) - elif not cmd.send_dst.is_valid(): + elif cmd.send_key.len > RELAY_MAX_KEY_SIZE: + conn.sendError(cmd, "Key too long", TooLarge) + elif cmd.send_dst.any_invalid(): conn.sendError(cmd, "Invalid pubkey", InvalidParams) else: let pubkey = conn.pubkey.get() @@ -589,150 +571,33 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay relay.db.record_transfer_stat( ip = conn.ip, pubkey = pubkey, - data_in = cmd.send_val.len, + data_in = cmd.send_val.len + cmd.send_key.len, ) relay.db.record_event_stat( ip = conn.ip, pubkey = pubkey, send = 1, ) - if relay.clients.hasKey(cmd.send_dst): - # dst is online - var other_conn = relay.clients[cmd.send_dst] - other_conn.sendMessage(RelayMessage( - kind: Data, - resp_id: 0, # Not triggered by other_conn's command - data_src: pubkey, - data_val: cmd.send_val, - )) - relay.db.record_transfer_stat( - ip = other_conn.ip, - pubkey = other_conn.pubkey.get(), - data_in = cmd.send_val.len, - ) - else: - # dst is offline - relay.db.exec(sql"INSERT INTO message (src, dst, data) VALUES (?, ?, ?)", - pubkey, cmd.send_dst, cmd.send_val.DbBlob) - of StoreChunk: - if cmd.chunk_key.len > RELAY_MAX_CHUNK_KEY_SIZE: - conn.sendError(cmd, "Key too long", TooLarge) - elif cmd.chunk_val.len > RELAY_MAX_CHUNK_SIZE: - conn.sendError(cmd, "Value too long", TooLarge) - elif cmd.chunk_dst.len > RELAY_MAX_CHUNK_DSTS: - conn.sendError(cmd, "Too many recipients", TooLarge) - elif cmd.chunk_dst.any_invalid(): - conn.sendError(cmd, "Invalid pubkey", InvalidParams) - else: - let pubkey = conn.pubkey.get() - if relay.max_chunk_space > 0 and relay.db.chunk_space_used(pubkey) > relay.max_chunk_space: - conn.sendError(cmd, "Too much chunk data", StorageLimitExceeded) - else: - relay.db.record_event_stat( - ip = conn.ip, - pubkey = pubkey, - store = 1, - ) - relay.db.exec(sql"BEGIN") - try: - relay.db.exec(sql"DELETE FROM chunk_dst WHERE src=? AND key=?", pubkey, cmd.chunk_key.DbBlob) - let offset = when TESTMODE: - $TIME_SKEW & " seconds" - else: - "0 seconds" - relay.db.exec(sql""" - INSERT OR REPLACE INTO chunk (last_used, src, key, val) - VALUES (datetime('now', ?), ?, ?, ?) - """, offset, pubkey, cmd.chunk_key.DbBlob, cmd.chunk_val.DbBlob) - var dsts: seq[SignPublicKey] - dsts.add(cmd.chunk_dst) - if pubkey notin dsts: - dsts.add(pubkey) - for dst in dsts: - relay.db.exec(sql"INSERT INTO chunk_dst (src, key, dst) VALUES (?, ?, ?)", - pubkey, cmd.chunk_key.DbBlob, dst) - relay.db.exec(sql"COMMIT") - except CatchableError: - relay.db.exec(sql"ROLLBACK") - of GetChunks: - for key in cmd.chunk_keys: - if key.len > RELAY_MAX_CHUNK_KEY_SIZE: - conn.sendError(cmd, "Key too long", TooLarge) - return - relay.delExpiredChunks() - let pubkey = conn.pubkey.get() - for key in cmd.chunk_keys: - let orow = relay.db.getRow(sql""" - SELECT - c.val - FROM - chunk_dst AS d - JOIN chunk AS c - ON d.src = c.src - AND d.key = c.key - WHERE - d.src = ? - AND d.key = ? - AND d.dst = ? - """, cmd.chunk_src, key.DbBlob, pubkey) - if orow.isSome: - let row = orow.get() - conn.sendMessage(RelayMessage( - kind: Chunk, - resp_id: cmd.resp_id, - chunk_src: cmd.chunk_src, - chunk_key: key, - chunk_val: some(row[0].b.string), - )) - else: - conn.sendMessage(RelayMessage( - kind: Chunk, - resp_id: cmd.resp_id, - chunk_src: cmd.chunk_src, - chunk_key: key, - chunk_val: none[string](), - )) - of HasChunks: - for key in cmd.has_keys: - if key.len > RELAY_MAX_CHUNK_KEY_SIZE: - conn.sendError(cmd, "Key too long", TooLarge) - return - relay.delExpiredChunks() - let pubkey = conn.pubkey.get() - var present: seq[string] - var absent: seq[string] - for key in cmd.has_keys: - let orow = relay.db.getRow(sql""" - SELECT - 1 - FROM - chunk_dst AS d - JOIN chunk AS c - ON d.src = c.src - AND d.key = c.key - WHERE - d.src = ? - AND d.key = ? - AND d.dst = ? - """, cmd.has_src, key.DbBlob, pubkey) - if orow.isSome: - present.add(key) - if cmd.has_src == pubkey: - # reset the expiration of the chunk, since the owner - # is touching it - let offset = when TESTMODE: - $TIME_SKEW & " seconds" - else: - "0 seconds" - relay.db.exec(sql""" - UPDATE chunk SET last_used = datetime('now', ?) WHERE src = ? AND key = ? - """, offset, cmd.has_src, key.DbBlob) - else: - absent.add(key) - conn.sendMessage(RelayMessage( - kind: ChunkStatus, - resp_id: cmd.resp_id, - status_src: cmd.has_src, - present: present, - absent: absent, - )) + for dst_pubkey in cmd.send_dst: + if relay.clients.hasKey(dst_pubkey): + # dst is online + var other_conn = relay.clients[dst_pubkey] + other_conn.sendMessage(RelayMessage( + kind: Data, + resp_id: 0, # Not triggered by other_conn's command + data_key: cmd.send_key, + data_src: pubkey, + data_val: cmd.send_val, + )) + relay.db.record_transfer_stat( + ip = other_conn.ip, + pubkey = other_conn.pubkey.get(), + data_out = cmd.send_val.len + cmd.send_key.len, + ) + else: + # dst is offline + relay.db.exec(sql""" + INSERT OR REPLACE INTO message + (key, src, dst, data) + VALUES (?, ?, ?, ?)""", + cmd.send_key.DbBlob, pubkey, dst_pubkey, cmd.send_val.DbBlob) diff --git a/src/bucketsrelay/v2/sampleclient.nim b/src/bucketsrelay/v2/sampleclient.nim index f3e3f3a..4589901 100644 --- a/src/bucketsrelay/v2/sampleclient.nim +++ b/src/bucketsrelay/v2/sampleclient.nim @@ -76,48 +76,17 @@ proc fetchNote*(ns: NetstringClient, topic: string): Future[string] {.async.} = else: raise ValueError.newException("No such note: " & topic) -proc sendData*(ns: NetstringClient, dst: SignPublicKey, val: string) {.async.} = +proc sendData*(ns: NetstringClient, dst: seq[SignPublicKey], val: string, key: string) {.async.} = await ns.sendCommand(RelayCommand( kind: SendData, + send_key: key, send_dst: dst, send_val: val, )) -proc getData*(ns: NetstringClient): Future[string] {.async.} = +proc getData*(ns: NetstringClient): Future[tuple[key: string, val: string]] {.async.} = let res = await ns.receiveMessage() if res.kind == Data: - return res.data_val + return (res.data_key, res.data_val) else: raise ValueError.newException("Expecting Data but got: " & $res) - -proc storeChunk*(ns: NetstringClient, dsts: seq[SignPublicKey], key: string, val: string) {.async.} = - await ns.sendCommand(RelayCommand( - kind: StoreChunk, - chunk_dst: dsts, - chunk_key: key, - chunk_val: val, - )) - -proc getChunk*(ns: NetstringClient, src: SignPublicKey, key: string): Future[Option[string]] {.async.} = - await ns.sendCommand(RelayCommand( - kind: GetChunks, - chunk_src: src, - chunk_keys: @[key], - )) - let res = await ns.receiveMessage() - if res.kind == Chunk: - return res.chunk_val - else: - raise ValueError.newException("Expecting Chunk but got: " & $res) - -proc hasChunk*(ns: NetstringClient, src: SignPublicKey, key: string): Future[bool] {.async.} = - await ns.sendCommand(RelayCommand( - kind: HasChunks, - has_src: src, - has_keys: @[key], - )) - let res = await ns.receiveMessage() - if res.kind == ChunkStatus: - return key in res.present - else: - raise ValueError.newException("Expecting ChunkStatus but got: " & $res) diff --git a/src/bucketsrelay/v2/server2.nim b/src/bucketsrelay/v2/server2.nim index 60f4a59..f5b922e 100644 --- a/src/bucketsrelay/v2/server2.nim +++ b/src/bucketsrelay/v2/server2.nim @@ -29,10 +29,12 @@ type msg: RelayMessage const - VERSION = slurp"../CHANGELOG.md".split(" ")[1] + RELAY_VERSION = slurp"../../../CHANGELOG.md".split(" ")[1] logo_png = slurp"./static/logo.png" favicon_png = slurp"./static/favicon.png" +echo "RELAY_VERSION: ", RELAY_VERSION + let ADMIN_USERNAME = getEnv("ADMIN_USERNAME", "admin") let ADMIN_PWHASH = when defined(release): getEnv("ADMIN_PWHASH", "") @@ -141,9 +143,7 @@ proc handleWebsocket(req: Request) {.async, gcsafe.} = type StorageStat = tuple pubkey: SignPublicKey - message_size: int - chunk_size: int - total_size: int + size: int PubkeyEventStat = tuple pubkey: SignPublicKey @@ -200,12 +200,10 @@ router myrouter: # total stored let total_stored_note = relay.db.getRow(sql"SELECT coalesce(sum(length(data)), 0) FROM note").get()[0].i - let total_stored_message = relay.db.getRow(sql"SELECT coalesce(sum(length(data)), 0) FROM message").get()[0].i - let total_stored_chunk = relay.db.getRow(sql"SELECT coalesce(sum(length(val)), 0) FROM chunk").get()[0].i - let total_stored = total_stored_note + total_stored_message + total_stored_chunk + let total_stored_message = relay.db.getRow(sql"SELECT sum(coalesce(length(data), 0) + coalesce(length(key), 0)) FROM message").get()[0].i + let total_stored = total_stored_note + total_stored_message let num_note = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM note").get()[0].i let num_message = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM message").get()[0].i - let num_chunk = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM chunk").get()[0].i # top traffic by ip var traffic_by_ip: seq[TransferTotal] @@ -259,32 +257,19 @@ router myrouter: # top storage by pubkey var storage_by_pubkey: seq[StorageStat] for row in relay.db.getAllRows(sql""" - WITH msg AS ( - SELECT src, SUM(coalesce(LENGTH(data), 0)) AS msg_bytes - FROM message - GROUP BY src - ), - chunksize AS ( - SELECT src, SUM(coalesce(LENGTH(val), 0)) AS chunk_bytes - FROM chunk - GROUP BY src - ) SELECT - COALESCE(m.src, c.src) AS src, - COALESCE(m.msg_bytes, 0) AS msg_bytes, - COALESCE(c.chunk_bytes, 0) AS chunk_bytes, - COALESCE(m.msg_bytes, 0) + COALESCE(c.chunk_bytes, 0) AS total_bytes - FROM msg AS m - FULL OUTER JOIN chunksize AS c - ON m.src = c.src - ORDER BY total_bytes DESC - LIMIT 10; + src, + SUM(COALESCE(LENGTH(data), 0) + COALESCE(LENGTH(key), 0)) AS msg_bytes + FROM + message + GROUP BY + src + ORDER BY 2 DESC + LIMIT 10 """): storage_by_pubkey.add(( pubkey: SignPublicKey.fromDb(row[0].b), - message_size: row[1].i.int, - chunk_size: row[2].i.int, - total_size: row[3].i.int, + size: row[1].i.int, )) # top events by pubkey @@ -344,25 +329,6 @@ router myrouter: pubkey: SignPublicKey.fromDb(row[0].b), count: row[1].i.int, )) - - var store_by_pubkey: seq[PubkeyEventStat] - for row in relay.db.getAllRows(sql""" - SELECT - pubkey, - COALESCE(SUM(store), 0) - FROM - stats_event - WHERE - period >= ? - AND pubkey <> '' - GROUP BY 1 - ORDER BY 2 DESC - LIMIT 10 - """, datarange.a): - store_by_pubkey.add(( - pubkey: SignPublicKey.fromDb(row[0].b), - count: row[1].i.int, - )) var html = "" compileTemplateFile("stats.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") diff --git a/src/bucketsrelay/v2/templates/index.nimja b/src/bucketsrelay/v2/templates/index.nimja index 56c949c..3f9ff1f 100644 --- a/src/bucketsrelay/v2/templates/index.nimja +++ b/src/bucketsrelay/v2/templates/index.nimja @@ -41,11 +41,11 @@ Buckets Relay
- {{ VERSION }} + {{ RELAY_VERSION }}

- If you use Buckets, this relay lets you securely share your budget among your devices. Data is stored on this relay for a time, but is removed after periods of inactivity. All data passing through this relay is encrypted end-to-end. + If you use Buckets, this relay lets you securely share your budget among your devices. Data is stored until recipients receive it or it expires. All data passing through this relay is encrypted end-to-end.

diff --git a/src/bucketsrelay/v2/templates/stats.nimja b/src/bucketsrelay/v2/templates/stats.nimja index ddde498..518efe1 100644 --- a/src/bucketsrelay/v2/templates/stats.nimja +++ b/src/bucketsrelay/v2/templates/stats.nimja @@ -60,21 +60,18 @@ Notes Messages - Chunks Total Bytes {{ total_stored_note.wcommas }} {{ total_stored_message.wcommas }} - {{ total_stored_chunk.wcommas }} {{ total_stored.wcommas }} Count {{ num_note.wcommas }} {{ num_message.wcommas }} - {{ num_chunk.wcommas }} @@ -119,14 +116,12 @@ - - + {% for tot in storage_by_pubkey %} - - + {% endfor %}
PubkeyMessage bytesChunk bytesBytes
{{ tot.pubkey.abbr }}{{ tot.message_size.wcommas }}{{ tot.chunk_size.wcommas }}{{ tot.size.wcommas }}
@@ -177,21 +172,6 @@ {% endfor %} - -

- - - - - - {% for st in store_by_pubkey %} - - - - - {% endfor %} -
PubkeyStore
{{ st.pubkey.abbr }}{{ st.count.wcommas }}
-
\ No newline at end of file diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim index dba3d31..d97297a 100644 --- a/tests/tfunctional.nim +++ b/tests/tfunctional.nim @@ -84,8 +84,8 @@ suite "auth": var alice = testClient(keys) var alice2 = testClient(keys) var bob = testClient() - waitFor bob.sendData(keys.pk, "this is bob") - check (waitFor alice2.getData()) == "this is bob" + waitFor bob.sendData(@[keys.pk], "this is bob", "") + check (waitFor alice2.getData()) == ("", "this is bob") suite "publishnote": @@ -112,47 +112,24 @@ suite "data": var bkeys = genkeys() var alice = testClient(akeys) var bob = testClient(bkeys) - waitFor alice.sendData(bkeys.pk, "hey, bob?") - check (waitFor bob.getData()) == "hey, bob?" - waitFor bob.sendData(akeys.pk, "hi, alice!") - check (waitFor alice.getData()) == "hi, alice!" + waitFor alice.sendData(@[bkeys.pk], "hey, bob?", "") + check (waitFor bob.getData()) == ("", "hey, bob?") + waitFor bob.sendData(@[akeys.pk], "hi, alice!", "hey") + check (waitFor alice.getData()) == ("hey", "hi, alice!") test "offline": var akeys = genkeys() var bkeys = genkeys() var alice = testClient(akeys) - waitFor alice.sendData(bkeys.pk, "message \x01") - waitFor alice.sendData(bkeys.pk, "message \x02") - waitFor alice.sendData(bkeys.pk, "message \x00null") + waitFor alice.sendData(@[bkeys.pk], "message \x01", "") + waitFor alice.sendData(@[bkeys.pk], "message \x02", "") + waitFor alice.sendData(@[bkeys.pk], "message \x00null", "") var bob = testClient(bkeys) - check (waitFor bob.getData()) == "message \x01" - check (waitFor bob.getData()) == "message \x02" - check (waitFor bob.getData()) == "message \x00null" + check (waitFor bob.getData()) == ("", "message \x01") + check (waitFor bob.getData()) == ("", "message \x02") + check (waitFor bob.getData()) == ("", "message \x00null") -suite "chunks": - - test "basic": - var akeys = genkeys() - var bkeys = genkeys() - var ckeys = genkeys() - var alice = testClient(akeys) - var bob = testClient(bkeys) - var carl = testClient(ckeys) - waitFor alice.storeChunk(@[bkeys.pk], "chunk1", "data1") - waitFor alice.storeChunk(@[bkeys.pk, ckeys.pk], "chunk2", "data2") - waitFor alice.storeChunk(@[bkeys.pk], "chunk3", "data3") - waitFor alice.storeChunk(@[bkeys.pk], "chunk3", "data3updated") - - check (waitFor bob.getChunk(akeys.pk, "chunk1")) == some("data1") - check (waitFor bob.getChunk(akeys.pk, "chunk2")) == some("data2") - check (waitFor bob.getChunk(akeys.pk, "chunk3")) == some("data3updated") - check (waitFor bob.getChunk(akeys.pk, "chunk4")).isNone() - - check (waitFor carl.getChunk(akeys.pk, "chunk1")).isNone() - check (waitFor carl.getChunk(akeys.pk, "chunk2")) == some("data2") - check (waitFor carl.getChunk(akeys.pk, "chunk3")).isNone() - check (waitFor carl.getChunk(akeys.pk, "chunk4")).isNone() suite "invalid": diff --git a/tests/tproto2.nim b/tests/tproto2.nim index 68081c6..bfdf2b7 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -415,11 +415,29 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - send_dst: bob.pk, + send_dst: @[bob.pk], send_val: "hel\x00lo", )) let data = bob.pop(Data) + check data.data_key == "" + check data.data_src == alice.pk + check data.data_val == "hel\x00lo" + + test "basic w/ key": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_key: "foom", + send_dst: @[bob.pk], + send_val: "hel\x00lo", + )) + + let data = bob.pop(Data) + check data.data_key == "foom" check data.data_src == alice.pk check data.data_val == "hel\x00lo" @@ -431,14 +449,27 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - send_dst: bob1.pk, + send_dst: @[bob1.pk], send_val: "hel\x00lo", )) + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_dst: @[bob1.pk], + send_val: "foo", + send_key: "bar", + )) var bob2 = relay.authenticatedConn(bob1.keys) - let data = bob2.pop(Data) - check data.data_src == alice.pk - check data.data_val == "hel\x00lo" + block: + let data = bob2.pop(Data) + check data.data_key == "" + check data.data_src == alice.pk + check data.data_val == "hel\x00lo" + block: + let data = bob2.pop(Data) + check data.data_key == "bar" + check data.data_src == alice.pk + check data.data_val == "foo" test "max data size": let relay = testRelay() @@ -446,12 +477,26 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - send_dst: alice.pk, + send_dst: @[alice.pk], send_val: "a".repeat(RELAY_MAX_MESSAGE_SIZE + 1), )) let err = alice.pop(Error) check err.err_code == TooLarge check err.err_cmd == SendData + + test "max key size": + let relay = testRelay() + var alice = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_key: "a".repeat(RELAY_MAX_KEY_SIZE + 1), + send_dst: @[alice.pk], + send_val: "a", + )) + let err = alice.pop(Error) + check err.err_code == TooLarge + check err.err_cmd == SendData when not defined(release): test "expiration": @@ -462,7 +507,7 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, - send_dst: bob.pk, + send_dst: @[bob.pk], send_val: "hello", )) @@ -472,16 +517,16 @@ suite "data": test "max transfer": var relay = testRelay() - let chunksize = RELAY_MAX_MESSAGE_SIZE div 2 - relay.max_transfer_rate = chunksize * 10 + let msgsize = RELAY_MAX_MESSAGE_SIZE div 2 + relay.max_transfer_rate = msgsize * 10 var alice = relay.authenticatedConn() var bob = relay.authenticatedConn() - let count = relay.max_transfer_rate div chunksize + 2 + let count = relay.max_transfer_rate div msgsize + 2 for i in 0..count: relay.handleCommand(alice, RelayCommand( kind: SendData, - send_dst: bob.pk, - send_val: "a".repeat(chunksize), + send_dst: @[bob.pk], + send_val: "a".repeat(msgsize), )) let err = alice.pop(Error) check err.err_code == TransferLimitExceeeded @@ -489,272 +534,173 @@ suite "data": test "invalid pubkey": var relay = testRelay() - let chunksize = RELAY_MAX_MESSAGE_SIZE div 2 - relay.max_transfer_rate = chunksize * 10 + let msgsize = RELAY_MAX_MESSAGE_SIZE div 2 + relay.max_transfer_rate = msgsize * 10 var alice = relay.authenticatedConn() relay.handleCommand(alice, RelayCommand( kind: SendData, - send_dst: "invalid".SignPublicKey, + send_dst: @["invalid".SignPublicKey], send_val: "a", )) let err = alice.pop(Error) check err.err_code == InvalidParams check err.err_cmd == SendData - -proc storeChunk(conn: var RelayConnection[TestClient], key: string, val: string, dst = newSeq[SignPublicKey]()) = - conn.relay.handleCommand(conn, RelayCommand( - kind: StoreChunk, - chunk_dst: dst, - chunk_key: key, - chunk_val: val, - )) - -proc getChunk(conn: var RelayConnection[TestClient], src: var RelayConnection[TestClient], key: string): Option[string] = - conn.relay.handleCommand(conn, RelayCommand( - kind: GetChunks, - chunk_src: src.pk, - chunk_keys: @[key], - )) - let chunk = conn.pop(Chunk) - return chunk.chunk_val - -proc chunkExists(conn: var RelayConnection[TestClient], src: var RelayConnection[TestClient], key: string): bool = - conn.relay.handleCommand(conn, RelayCommand( - kind: HasChunks, - has_src: src.pk, - has_keys: @[key], - )) - let resp = conn.pop(ChunkStatus) - return key in resp.present - -suite "chunks": - - test "basic": + test "overwrite key": let relay = testRelay() var alice = relay.authenticatedConn() - var bob = relay.authenticatedConn() + var bob1 = relay.authenticatedConn() + relay.disconnect(bob1) relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[bob.pk], - chunk_key: "key1", - chunk_val: "\x00data1", + kind: SendData, + send_key: "apple", + send_dst: @[bob1.pk], + send_val: "core", )) relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[bob.pk], - chunk_key: "key2", - chunk_val: "data2", - )) - check bob.msgCount == 0 - - relay.handleCommand(bob, RelayCommand( - kind: GetChunks, - chunk_src: alice.pk, - chunk_keys: @["key1", "key2"], + kind: SendData, + send_key: "banana", + send_dst: @[bob1.pk], + send_val: "boat", )) - block: - let chunk = bob.pop(Chunk) - check chunk.chunk_src == alice.pk - check chunk.chunk_key == "key1" - check chunk.chunk_val.get() == "\x00data1" - block: - let chunk = bob.pop(Chunk) - check chunk.chunk_src == alice.pk - check chunk.chunk_key == "key2" - check chunk.chunk_val.get() == "data2" - relay.handleCommand(alice, RelayCommand( - kind: GetChunks, - chunk_src: alice.pk, - chunk_keys: @["key2"], + kind: SendData, + send_key: "apple", + send_dst: @[bob1.pk], + send_val: "pie", )) + + var bob2 = relay.authenticatedConn(bob1.keys) block: - let chunk = alice.pop(Chunk) - check chunk.chunk_src == alice.pk - check chunk.chunk_key == "key2" - check chunk.chunk_val.get() == "data2" - - test "overwrite": - let relay = testRelay() - var alice = relay.authenticatedConn() - alice.storeChunk("key", "first") - alice.storeChunk("key", "second") - check alice.getChunk(alice, "key").get() == "second" - check alice.chunkExists(alice, "key") + let data = bob2.pop(Data) + check data.data_key == "banana" + check data.data_src == alice.pk + check data.data_val == "boat" + block: + let data = bob2.pop(Data) + check data.data_key == "apple" + check data.data_src == alice.pk + check data.data_val == "pie" - test "multiple dst": - let relay = testRelay() - var alice = relay.authenticatedConn() - var bob = relay.authenticatedConn() - var carl = relay.authenticatedConn() - alice.storeChunk("key", "val", @[bob.pk, carl.pk]) - check bob.getChunk(alice, "key").get() == "val" - check carl.getChunk(alice, "key").get() == "val" - - test "dne": + test "no overwrite empty key": let relay = testRelay() var alice = relay.authenticatedConn() + var bob1 = relay.authenticatedConn() + relay.disconnect(bob1) + relay.handleCommand(alice, RelayCommand( - kind: GetChunks, - chunk_src: alice.pk, - chunk_keys: @["dne"], + kind: SendData, + send_key: "", + send_dst: @[bob1.pk], + send_val: "first", + )) + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_key: "", + send_dst: @[bob1.pk], + send_val: "second", )) + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_key: "", + send_dst: @[bob1.pk], + send_val: "third", + )) + + var bob2 = relay.authenticatedConn(bob1.keys) + block: + let data = bob2.pop(Data) + check data.data_key == "" + check data.data_src == alice.pk + check data.data_val == "first" block: - let chunk = alice.pop(Chunk) - check chunk.chunk_src == alice.pk - check chunk.chunk_key == "dne" - check chunk.chunk_val.isNone() - check alice.chunkExists(alice, "dne") == false + let data = bob2.pop(Data) + check data.data_key == "" + check data.data_src == alice.pk + check data.data_val == "second" + block: + let data = bob2.pop(Data) + check data.data_key == "" + check data.data_src == alice.pk + check data.data_val == "third" - test "only dst allowed": + test "deliver once per pubkey": let relay = testRelay() var alice = relay.authenticatedConn() var bob = relay.authenticatedConn() - alice.storeChunk("key", "first") - check bob.getChunk(alice, "key").isNone() - check alice.chunkExists(alice, "key") - check bob.chunkExists(alice, "key") == false - - when not defined(release): - test "expiration": - let relay = testRelay() - var alice = relay.authenticatedConn() - alice.storeChunk("key", "foo") - check alice.chunkExists(alice, "key") - skewTime(RELAY_MESSAGE_DURATION + 1) - check alice.getChunk(alice, "key").isNone() - check alice.chunkExists(alice, "key") == false - - when not defined(release): - test "expiration update": - let relay = testRelay() - var alice = relay.authenticatedConn() - alice.storeChunk("key", "foo") - skewTime(RELAY_MESSAGE_DURATION - 1) - alice.storeChunk("key", "foo") - skewTime(3) - check alice.getChunk(alice, "key").get() == "foo" - - when not defined(release): - test "expiration update status": - let relay = testRelay() - var alice = relay.authenticatedConn() - alice.storeChunk("key", "foo") - checkpoint $relay.db.getAllRows(sql"SELECT src, key, last_used FROM chunk") - skewTime(RELAY_MESSAGE_DURATION - 1) - check alice.chunkExists(alice, "key") - checkpoint $relay.db.getAllRows(sql"SELECT src, key, last_used FROM chunk") - skewTime(3) - checkpoint $relay.db.getAllRows(sql"SELECT src, key, last_used FROM chunk") - check alice.chunkExists(alice, "key") + var carl = relay.authenticatedConn() - test "remove dst": - let relay = testRelay() - var alice = relay.authenticatedConn() - var bob = relay.authenticatedConn() - var sam = relay.authenticatedConn() - alice.storeChunk("key", "first", @[bob.pk, sam.pk]) - check bob.getChunk(alice, "key").get() == "first" - check bob.chunkExists(alice, "key") - check sam.getChunk(alice, "key").get() == "first" - check sam.chunkExists(alice, "key") - alice.storeChunk("key", "first", @[bob.pk]) - check bob.getChunk(alice, "key").get() == "first" - check bob.chunkExists(alice, "key") - check sam.getChunk(alice, "key").isNone() - check sam.chunkExists(alice, "key") == false - - test "max key len": - let relay = testRelay() - var alice = relay.authenticatedConn() relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[], - chunk_key: "a".repeat(RELAY_MAX_CHUNK_KEY_SIZE + 1), - chunk_val: "data1", + kind: SendData, + send_key: "", + send_dst: @[bob.pk, carl.pk], + send_val: "hi", )) - let err = alice.pop(Error) - check err.err_cmd == StoreChunk - check err.err_code == TooLarge + + block: + let data = bob.pop(Data) + check data.data_key == "" + check data.data_src == alice.pk + check data.data_val == "hi" + block: + let data = carl.pop(Data) + check data.data_key == "" + check data.data_src == alice.pk + check data.data_val == "hi" - test "max val len": + test "deliver to a, update val, deliver to a, b": let relay = testRelay() var alice = relay.authenticatedConn() - relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[], - chunk_key: "a", - chunk_val: "a".repeat(RELAY_MAX_CHUNK_SIZE + 1), - )) - let err = alice.pop(Error) - check err.err_cmd == StoreChunk - check err.err_code == TooLarge + var bob = relay.authenticatedConn() + var carl = relay.authenticatedConn() + relay.disconnect(carl) - test "max key len get": - let relay = testRelay() - var alice = relay.authenticatedConn() relay.handleCommand(alice, RelayCommand( - kind: GetChunks, - chunk_src: alice.pk, - chunk_keys: @["a".repeat(RELAY_MAX_CHUNK_KEY_SIZE + 1)], + kind: SendData, + send_key: "apple", + send_dst: @[bob.pk, carl.pk], + send_val: "core", )) - let err = alice.pop(Error) - check err.err_cmd == GetChunks - check err.err_code == TooLarge - - test "max dst.len": - let relay = testRelay() - var alice = relay.authenticatedConn() - var dsts: seq[SignPublicKey] - for i in 0..(RELAY_MAX_CHUNK_DSTS+1): - dsts.add(genkeys().pk) relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: dsts, - chunk_key: "a", - chunk_val: "b", + kind: SendData, + send_key: "apple", + send_dst: @[bob.pk, carl.pk], + send_val: "cider", )) - let err = alice.pop(Error) - check err.err_cmd == StoreChunk - check err.err_code == TooLarge + var carl2 = relay.authenticatedConn(carl.keys) + block: + check bob.pop(Data).data_val == "core" + check bob.pop(Data).data_val == "cider" + block: + check carl2.pop(Data).data_val == "cider" - test "max storage": - var relay = testRelay() - relay.max_chunk_space = RELAY_MAX_CHUNK_SIZE * 3 - 1 + test "drop recipient": + let relay = testRelay() var alice = relay.authenticatedConn() var bob = relay.authenticatedConn() + relay.disconnect(bob) + var carl = relay.authenticatedConn() + relay.disconnect(carl) - for i in 0..<3: - relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[bob.pk], - chunk_key: "key1" & $i, - chunk_val: "a".repeat(RELAY_MAX_CHUNK_SIZE), - )) relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[bob.pk], - chunk_key: "lastkey", - chunk_val: "a".repeat(RELAY_MAX_CHUNK_SIZE), + kind: SendData, + send_key: "apple", + send_dst: @[bob.pk, carl.pk], + send_val: "core", )) - let err = alice.pop(Error) - check err.err_cmd == StoreChunk - check err.err_code == StorageLimitExceeded - - test "invalid pubkey": - let relay = testRelay() - var alice = relay.authenticatedConn() relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @["fake".SignPublicKey], - chunk_key: "a", - chunk_val: "b", + kind: SendData, + send_key: "apple", + send_dst: @[bob.pk], + send_val: "cider", )) - let err = alice.pop(Error) - check err.err_code == InvalidParams - check err.err_cmd == StoreChunk + block: + var bob2 = relay.authenticatedConn(bob.keys) + check bob2.pop(Data).data_val == "cider" + block: + var carl2 = relay.authenticatedConn(carl.keys) + check carl2.pop(Data).data_val == "core" suite "anon": @@ -790,41 +736,12 @@ suite "anon": discard alice.pop(Who) relay.handleCommand(alice, RelayCommand( kind: SendData, - send_dst: keys.pk, + send_dst: @[keys.pk], send_val: "bar", )) let err = alice.pop(Error) check err.err_cmd == SendData check err.err_code == NotAllowed - - test "StoreChunk": - let relay = testRelay() - var keys = genkeys() - var alice = relay.anonConn() - discard alice.pop(Who) - relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[keys.pk], - chunk_key: "foo", - chunk_val: "bar", - )) - let err = alice.pop(Error) - check err.err_cmd == StoreChunk - check err.err_code == NotAllowed - - test "GetChunks": - let relay = testRelay() - var keys = genkeys() - var alice = relay.anonConn() - discard alice.pop(Who) - relay.handleCommand(alice, RelayCommand( - kind: GetChunks, - chunk_src: keys.pk, - chunk_keys: @["foo"], - )) - let err = alice.pop(Error) - check err.err_cmd == GetChunks - check err.err_code == NotAllowed suite "stats": @@ -931,7 +848,7 @@ suite "resp_id": relay.handleCommand(alice, RelayCommand( kind: SendData, resp_id: 111, - send_dst: bob.pk, + send_dst: @[bob.pk], send_val: "hello", )) @@ -940,64 +857,6 @@ suite "resp_id": let data = bob.pop(Data) check data.resp_id == 0 - test "StoreChunk command response has matching resp_id": - let relay = testRelay() - var alice = relay.authenticatedConn() - - relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - resp_id: 222, - chunk_dst: @[], - chunk_key: "key", - chunk_val: "val", - )) - # StoreChunk doesn't send a response by default, no message to check - check alice.msgCount == 0 - - test "GetChunks response has matching resp_id": - let relay = testRelay() - var alice = relay.authenticatedConn() - - # Store a chunk first - relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[], - chunk_key: "key", - chunk_val: "val", - )) - - # Get the chunk with resp_id - relay.handleCommand(alice, RelayCommand( - kind: GetChunks, - resp_id: 333, - chunk_src: alice.pk, - chunk_keys: @["key"], - )) - let chunk = alice.pop(Chunk) - check chunk.resp_id == 333 - - test "HasChunks response has matching resp_id": - let relay = testRelay() - var alice = relay.authenticatedConn() - - # Store a chunk first - relay.handleCommand(alice, RelayCommand( - kind: StoreChunk, - chunk_dst: @[], - chunk_key: "key", - chunk_val: "val", - )) - - # Check if chunk exists with resp_id - relay.handleCommand(alice, RelayCommand( - kind: HasChunks, - resp_id: 444, - has_src: alice.pk, - has_keys: @["key"], - )) - let status = alice.pop(ChunkStatus) - check status.resp_id == 444 - test "Multiple commands with different resp_ids": let relay = testRelay() var alice = relay.authenticatedConn() diff --git a/tests/tserde2.nim b/tests/tserde2.nim index 088fbc2..0077938 100644 --- a/tests/tserde2.nim +++ b/tests/tserde2.nim @@ -21,9 +21,7 @@ test "RelayMessage": of Okay: RelayMessage(kind: Okay, resp_id: 42, ok_cmd: SendData) of Error: RelayMessage(kind: Error, resp_id: 123, err_cmd: SendData, err_code: TooLarge, err_message: "foo") of Note: RelayMessage(kind: Note, resp_id: 456, note_topic: "something", note_data: "data") - of Data: RelayMessage(kind: Data, resp_id: 0, data_src: "hey".SignPublicKey, data_val: "foo") - of Chunk: RelayMessage(kind: Chunk, resp_id: 789, chunk_src: "hey".SignPublicKey, chunk_key: "key", chunk_val: some("theval")) - of ChunkStatus: RelayMessage(kind: ChunkStatus, resp_id: 999, status_src: "a".SignPublicKey, present: @["foo"], absent: @["bar"]) + of Data: RelayMessage(kind: Data, resp_id: 0, data_src: "hey".SignPublicKey, data_val: "foo", data_key: "hey") let serialized = example.serialize() info $example info "serialized: " & serialized.nice @@ -44,38 +42,12 @@ test "RelayCommand": ) of PublishNote: RelayCommand(kind: PublishNote, resp_id: 100, pub_topic: "topic", pub_data: "data") of FetchNote: RelayCommand(kind: FetchNote, resp_id: 200, fetch_topic: "topic") - of SendData: RelayCommand(kind: SendData, resp_id: 300, send_dst: "one".SignPublicKey, send_val: "data") - of StoreChunk: RelayCommand( - kind: StoreChunk, - resp_id: 400, - chunk_dst: @["one".SignPublicKey], - chunk_key: "theky", - chunk_val: "someval" - ) - of GetChunks: RelayCommand(kind: GetChunks, resp_id: 500, chunk_src: "hey".SignPublicKey, chunk_keys: @["foo", "bar"]) - of HasChunks: RelayCommand( - kind: HasChunks, - resp_id: 600, - has_src: "hey".SignPublicKey, - has_keys: @["foo", "Bar"], - ) + of SendData: RelayCommand(kind: SendData, resp_id: 300, send_dst: @["one".SignPublicKey, "two".SignPublicKey], send_val: "data", send_key: "key") let serialized = example.serialize() info $example info "serialized: " & serialized check RelayCommand.deserialize(serialized) == example -test "Chunk with none": - let chunk = RelayMessage(kind: Chunk, - resp_id: 888, - chunk_src: "foo".SignPublicKey, - chunk_key: "key", - chunk_val: none[string](), - ) - info $chunk - let serialized = chunk.serialize() - info "serialized: " & serialized - check RelayMessage.deserialize(serialized) == chunk - test "ErrorCodes": for err in low(ErrorCode)..high(ErrorCode): let serialized = err.serialize() @@ -96,5 +68,5 @@ test "resp_id serialization": let cmd1 = RelayCommand(kind: PublishNote, resp_id: 54321, pub_topic: "topic", pub_data: "data") check RelayCommand.deserialize(cmd1.serialize()).resp_id == 54321 - let cmd2 = RelayCommand(kind: SendData, resp_id: 0, send_dst: "dst".SignPublicKey, send_val: "val") + let cmd2 = RelayCommand(kind: SendData, resp_id: 0, send_dst: @["dst".SignPublicKey], send_val: "val", send_key: "foo") check RelayCommand.deserialize(cmd2.serialize()).resp_id == 0 From 165294483c6201c2d8988920c0f45d1094f781cf Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 12 Dec 2025 12:18:52 -0500 Subject: [PATCH 44/46] Improved data storage efficiency of multi-recipient messages --- src/bucketsrelay/v2/objs.nim | 2 +- src/bucketsrelay/v2/proto2.nim | 103 +++++++++++++++++++++++---------- tests/tproto2.nim | 41 ++++++++++++- 3 files changed, 112 insertions(+), 34 deletions(-) diff --git a/src/bucketsrelay/v2/objs.nim b/src/bucketsrelay/v2/objs.nim index 70e6bea..ecc306b 100644 --- a/src/bucketsrelay/v2/objs.nim +++ b/src/bucketsrelay/v2/objs.nim @@ -93,7 +93,7 @@ const RELAY_MAX_NOTES* = 1000 RELAY_NOTE_DURATION* = 5 * 24 * 60 * 60 RELAY_MAX_MESSAGE_SIZE* = 65536 * 2 - RELAY_MAX_KEY_SIZE* = 4096 + RELAY_MAX_KEY_SIZE* = 512 RELAY_MESSAGE_DURATION* = 30 * 24 * 60 * 60 const diff --git a/src/bucketsrelay/v2/proto2.nim b/src/bucketsrelay/v2/proto2.nim index fe97346..a5320a1 100644 --- a/src/bucketsrelay/v2/proto2.nim +++ b/src/bucketsrelay/v2/proto2.nim @@ -192,19 +192,42 @@ proc updateSchema*(db: DbConn) = db.exec(sql"CREATE INDEX note_created ON note(created)") db.exec(sql"CREATE INDEX note_src ON note(src)") - # message - db.exec(sql"""CREATE TABLE message ( + # message - stores message data once + db.exec(sql"""CREATE TABLE message_data ( id INTEGER PRIMARY KEY AUTOINCREMENT, created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, key BLOB NOT NULL, - src TEXT NOT NULL, - dst TEXT NOT NULL, + src BLOB NOT NULL, data BLOB NOT NULL )""") - db.exec(sql"CREATE INDEX message_created ON message(created)") - db.exec(sql"""CREATE UNIQUE INDEX message_dst_key - ON message(dst, key) - WHERE key IS NOT x'' + db.exec(sql"CREATE INDEX message_data_created ON message_data(created)") + + # message recipients - tracks who needs to receive each message + db.exec(sql"""CREATE TABLE message_recipient ( + message_id INTEGER NOT NULL, + dst BLOB NOT NULL, + src BLOB NOT NULL, + key BLOB NOT NULL, + PRIMARY KEY (message_id, dst), + FOREIGN KEY (message_id) REFERENCES message_data(id) ON DELETE CASCADE + )""") + db.exec(sql"CREATE INDEX message_recipient_dst ON message_recipient(dst)") + # Ensure only one message per (dst, src, key) when key is not empty + # Different senders can send messages with the same key to the same recipient + db.exec(sql"""CREATE UNIQUE INDEX message_recipient_dst_src_key + ON message_recipient(dst, src, key) + WHERE key != x'' + """) + + # Automatically delete message_data when last recipient is removed + db.exec(sql"""CREATE TRIGGER cleanup_message_data + AFTER DELETE ON message_recipient + WHEN NOT EXISTS ( + SELECT 1 FROM message_recipient WHERE message_id = OLD.message_id + ) + BEGIN + DELETE FROM message_data WHERE id = OLD.message_id; + END """) # stats @@ -419,20 +442,23 @@ proc delExpiredMessages(relay: Relay) = else: -RELAY_MESSAGE_DURATION let offstring = &"{offset} seconds" - relay.db.exec(sql"DELETE FROM message WHERE created <= datetime('now', ?)", offstring) + # Delete expired message data; CASCADE will automatically delete associated recipients + relay.db.exec(sql"DELETE FROM message_data WHERE created <= datetime('now', ?)", offstring) proc nextMessage(relay: Relay, dst: SignPublicKey): Option[RelayMessage] = let orow = relay.db.getRow(sql""" - SELECT key, src, data, id - FROM message + SELECT md.key, md.src, md.data, md.id + FROM message_recipient mr + JOIN message_data md ON mr.message_id = md.id WHERE - dst = ? + mr.dst = ? ORDER BY - created ASC, - id ASC + md.created ASC, + md.id ASC LIMIT 1""", dst) if orow.isSome: let row = orow.get() + let message_id = row[3].i result = some(RelayMessage( kind: Data, resp_id: 0, # Data messages are not triggered by recipient's command @@ -440,16 +466,9 @@ proc nextMessage(relay: Relay, dst: SignPublicKey): Option[RelayMessage] = data_src: SignPublicKey.fromDB(row[1].b), data_val: row[2].b.string, )) - relay.db.exec(sql"DELETE FROM message WHERE id=?", row[3].i) - -proc delExpiredChunks(relay: Relay) = - let offset = when TESTMODE: - -RELAY_MESSAGE_DURATION + TIME_SKEW - else: - -RELAY_MESSAGE_DURATION - let offstring = &"{offset} seconds" - relay.db.exec(sql"DELETE FROM chunk WHERE last_used <= datetime('now', ?)", offstring) + # Delete this recipient entry (trigger will auto-cleanup message_data if this was the last recipient) + relay.db.exec(sql"DELETE FROM message_recipient WHERE message_id = ? AND dst = ?", message_id, dst) #------------------------------------------------------------------- # relay command handling @@ -532,7 +551,7 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay pubkey, ) conn.sendOkay(cmd) - except: + except CatchableError: conn.sendError(cmd, "Duplicate topic", Generic) of FetchNote: if cmd.fetch_topic.len > RELAY_MAX_TOPIC_SIZE: @@ -578,9 +597,12 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay pubkey = pubkey, send = 1, ) + + # Collect offline recipients + var offline_dsts: seq[SignPublicKey] for dst_pubkey in cmd.send_dst: if relay.clients.hasKey(dst_pubkey): - # dst is online + # dst is online - send directly var other_conn = relay.clients[dst_pubkey] other_conn.sendMessage(RelayMessage( kind: Data, @@ -595,9 +617,28 @@ proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: Relay data_out = cmd.send_val.len + cmd.send_key.len, ) else: - # dst is offline - relay.db.exec(sql""" - INSERT OR REPLACE INTO message - (key, src, dst, data) - VALUES (?, ?, ?, ?)""", - cmd.send_key.DbBlob, pubkey, dst_pubkey, cmd.send_val.DbBlob) + # dst is offline - collect for batch storage + offline_dsts.add(dst_pubkey) + + # Store message data once for all offline recipients + if offline_dsts.len > 0: + relay.db.exec(sql"BEGIN") + try: + # Insert message data once and get the ID + let message_id = relay.db.insertID(sql""" + INSERT INTO message_data (key, src, data) + VALUES (?, ?, ?)""", + cmd.send_key.DbBlob, pubkey, cmd.send_val.DbBlob) + + # Add recipient entries for each offline destination + # Use INSERT OR REPLACE to handle duplicate (dst, src, key) - database-enforced + for dst_pubkey in offline_dsts: + relay.db.exec(sql""" + INSERT OR REPLACE INTO message_recipient (message_id, dst, src, key) + VALUES (?, ?, ?, ?)""", + message_id, dst_pubkey, pubkey, cmd.send_key.DbBlob) + + relay.db.exec(sql"COMMIT") + except CatchableError: + relay.db.exec(sql"ROLLBACK") + raise diff --git a/tests/tproto2.nim b/tests/tproto2.nim index bfdf2b7..c8d7c81 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -676,6 +676,8 @@ suite "data": check carl2.pop(Data).data_val == "cider" test "drop recipient": + # When updating a keyed message to a subset of recipients, + # recipients not in the new send still get their old pending message let relay = testRelay() var alice = relay.authenticatedConn() var bob = relay.authenticatedConn() @@ -692,7 +694,7 @@ suite "data": relay.handleCommand(alice, RelayCommand( kind: SendData, send_key: "apple", - send_dst: @[bob.pk], + send_dst: @[bob.pk], # Carl not included in update send_val: "cider", )) block: @@ -700,7 +702,42 @@ suite "data": check bob2.pop(Data).data_val == "cider" block: var carl2 = relay.authenticatedConn(carl.keys) - check carl2.pop(Data).data_val == "core" + # Carl still has his original pending message + check carl2.pop(Data).data_val == "core" + + test "multiple senders same key": + # Different senders can send messages with the same key to the same recipient + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + var carl = relay.authenticatedConn() + relay.disconnect(carl) + + relay.handleCommand(alice, RelayCommand( + kind: SendData, + send_key: "status", + send_dst: @[carl.pk], + send_val: "alice_v1", + )) + relay.handleCommand(bob, RelayCommand( + kind: SendData, + send_key: "status", + send_dst: @[carl.pk], + send_val: "bob_v1", + )) + + var carl2 = relay.authenticatedConn(carl.keys) + # Carl should receive both messages, one from alice and one from bob + block: + let msg1 = carl2.pop(Data) + let msg2 = carl2.pop(Data) + # Order might vary, so check both possibilities + check ( + (msg1.data_src == alice.pk and msg1.data_val == "alice_v1" and + msg2.data_src == bob.pk and msg2.data_val == "bob_v1") or + (msg1.data_src == bob.pk and msg1.data_val == "bob_v1" and + msg2.data_src == alice.pk and msg2.data_val == "alice_v1") + ) suite "anon": From 56b91ede275c860e693318a8a64f18df6ab52643 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 12 Dec 2025 13:01:13 -0500 Subject: [PATCH 45/46] Fix stats --- src/bucketsrelay/v2/server2.nim | 32 ++++++++++++++++++----- src/bucketsrelay/v2/templates/stats.nimja | 4 +-- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/bucketsrelay/v2/server2.nim b/src/bucketsrelay/v2/server2.nim index f5b922e..a8cd936 100644 --- a/src/bucketsrelay/v2/server2.nim +++ b/src/bucketsrelay/v2/server2.nim @@ -200,10 +200,14 @@ router myrouter: # total stored let total_stored_note = relay.db.getRow(sql"SELECT coalesce(sum(length(data)), 0) FROM note").get()[0].i - let total_stored_message = relay.db.getRow(sql"SELECT sum(coalesce(length(data), 0) + coalesce(length(key), 0)) FROM message").get()[0].i + let total_stored_message = relay.db.getRow(sql""" + SELECT + COALESCE(SUM(LENGTH(md.data) + LENGTH(md.key)), 0) + FROM message_data md + """).get()[0].i let total_stored = total_stored_note + total_stored_message let num_note = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM note").get()[0].i - let num_message = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM message").get()[0].i + let num_message = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM message_recipient").get()[0].i # top traffic by ip var traffic_by_ip: seq[TransferTotal] @@ -259,11 +263,25 @@ router myrouter: for row in relay.db.getAllRows(sql""" SELECT src, - SUM(COALESCE(LENGTH(data), 0) + COALESCE(LENGTH(key), 0)) AS msg_bytes - FROM - message - GROUP BY - src + SUM(total_bytes) AS total_bytes + FROM ( + -- Note storage + SELECT + src, + SUM(LENGTH(data)) AS total_bytes + FROM note + GROUP BY src + + UNION ALL + + -- Message data storage + SELECT + md.src, + SUM(LENGTH(md.data) + LENGTH(md.key)) AS total_bytes + FROM message_data md + GROUP BY md.src + ) + GROUP BY src ORDER BY 2 DESC LIMIT 10 """): diff --git a/src/bucketsrelay/v2/templates/stats.nimja b/src/bucketsrelay/v2/templates/stats.nimja index 518efe1..825def6 100644 --- a/src/bucketsrelay/v2/templates/stats.nimja +++ b/src/bucketsrelay/v2/templates/stats.nimja @@ -112,7 +112,7 @@ {% endfor %} -

Top storage by Pubkey

+

Top Storage by Pubkey

@@ -126,7 +126,7 @@ {% endfor %}
Pubkey
-

Top events by Pubkey

+

Top Events by Pubkey

From 69d868f1284c3ba3aecf50b2aca95800d934a7bb Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 12 Dec 2025 13:14:18 -0500 Subject: [PATCH 46/46] Fix test assertions --- tests/tproto2.nim | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/tproto2.nim b/tests/tproto2.nim index c8d7c81..4fca179 100644 --- a/tests/tproto2.nim +++ b/tests/tproto2.nim @@ -727,17 +727,16 @@ suite "data": )) var carl2 = relay.authenticatedConn(carl.keys) - # Carl should receive both messages, one from alice and one from bob block: - let msg1 = carl2.pop(Data) - let msg2 = carl2.pop(Data) - # Order might vary, so check both possibilities - check ( - (msg1.data_src == alice.pk and msg1.data_val == "alice_v1" and - msg2.data_src == bob.pk and msg2.data_val == "bob_v1") or - (msg1.data_src == bob.pk and msg1.data_val == "bob_v1" and - msg2.data_src == alice.pk and msg2.data_val == "alice_v1") - ) + let msg = carl2.pop(Data) + check msg.data_key == "status" + check msg.data_src == alice.pk + check msg.data_val == "alice_v1" + block: + let msg = carl2.pop(Data) + check msg.data_key == "status" + check msg.data_src == bob.pk + check msg.data_val == "bob_v1" suite "anon":