diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e3c1e83..4c238da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,10 +11,9 @@ jobs: strategy: matrix: version: - - binary:1.6.18 + - binary:2.2.4 os: - ubuntu-latest - # - macOS-latest steps: - uses: actions/checkout@v1 - uses: iffy/install-nim@v5 @@ -22,37 +21,22 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: version: ${{ matrix.version }} - - name: Update nimble - run: nimble install -y nimble - #---------------------------------------- - # multi user mode (default) - #---------------------------------------- - - name: Install - run: nimble install -y - - name: Build bins - run: nimble multiuserbins - - name: Run tests - run: | - export PATH="${PATH}:$(pwd)/bin" - nimble test - - name: Command-line tests + - name: Install libsodium run: | - export PATH="${PATH}:$(pwd)/bin" - tests/func1.sh - #---------------------------------------- - # single user mode - #---------------------------------------- - - name: Install (single user mode) - run: nimble singleuserbins - - name: Run tests (single user mode) + sudo apt update -q + sudo apt install -y libsodium-dev + - name: Install pkger + run: nimble install -y https://github.com/iffy/pkger/ + - name: Install deps + run: pkger fetch + - name: Run tests run: | - export PATH="${PATH}:$(pwd)/bin" - nimble -d:relaysingleusermode test + export SHOW_LOGS=1 + tests/all.sh docker: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - run: docker build --file docker/singleuser.Dockerfile . - - run: docker build --file docker/multiuser.Dockerfile . + - run: docker build --file docker/Dockerfile . diff --git a/.gitignore b/.gitignore index 06f0478..2bbbdf9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,9 @@ bucketsrelay.sqlite _tests !TODO fly.toml +relay.sqlite +libs +*.keys +*.sqlite +tests/bin +*.sqlite-journal diff --git a/README.md b/README.md index 8b54453..0447336 100644 --- a/README.md +++ b/README.md @@ -6,38 +6,32 @@ This repository contains the open source code for the [Buckets](https://www.budgetwithbuckets.com) relay server, which allows users to share budget data between their devices in an end-to-end encrypted way. -## Quickstart - single user mode +You can use the publicly available relay at -If you want to run the relay on your own computer with only one user account, do the following: +## Quickstart w/ Docker/Podman -1. Install [Nim](https://nim-lang.org/) -2. Build the relay: +If you want to run the relay on with docker: + +1. Get the code: ``` git clone https://github.com/buckets/relay.git buckets-relay.git cd buckets-relay.git -nimble singleuserbins ``` -3. Run the server: +2. Build the image -```sh -RELAY_USERNAME=someusername -RELAY_PASSWORD=somepassword -bin/brelay server +``` +docker build -f docker/Dockerfile -t buckets/relay . ``` -This will launch the relay on the default port. Run `brelay --help` for more options. - -## Multi-user mode - -If instead of `nimble singleuserbins` you run `nimble multiuserbins` the server will be built in multi-user mode. - -Register users via `brelay adduser ...` or through the web interface. +3. Run it: -Registration-related emails are sent through [Postmark](https://postmarkapp.com/). Set `POSTMARK_API_KEY` to your Postmark key to use it. Otherwise, disable emails with `-d:nopostmark`. +``` +docker run -it --rm -p 8080:8080 buckets/relay +``` -Users can authenticate with their Buckets license if you set an environment variable `AUTH_LICENSE_PUBKEY=` +Read `docker/Dockerfile` to get an idea of how to build it yourself if you'd like. ## Security @@ -45,93 +39,46 @@ Users can authenticate with their Buckets license if you set an environment vari - This relay server can see all traffic, so clients should encrypt data intended for other clients. - Clients should also authenticate each other through the relay and not trust the authentication done by this server. -## Development - -To run the server locally: - -```sh -nimble run brelay server -``` - -## Deployment to fly.io - -If you'd like to run a relay server on [fly.io](https://fly.io/), sign up for the service then do one of the following. If you'd like to host somewhere else, you could use the Dockerfiles in [docker/](./docker/) as a starting point. - -### Single-user mode - -```sh -fly launch --dockerfile docker/singleuser.Dockerfile -fly secrets set RELAY_USERNAME='someusername' RELAY_PASSWORD='somepassword' -``` - -| Variable | Description | -|---|---| -| `RELAY_USERNAME` | Username or email you'll use to authenticate to the relay. | -| `RELAY_PASSWORD` | Password you'll use to authenticate to the relay. | - -### Multi-user mode - -```sh -fly launch --dockerfile docker/multiuser.Dockerfile -fly secrets set POSTMARK_API_KEY='your key' AUTH_LICENSE_PUBKEY='the key' LICENSE_HASH_SALT='choose something here' -``` - -| Variable | Description | -|---|---| -| `POSTMARK_API_KEY` | API key from [Postmark](https://postmarkapp.com/) | -| `AUTH_LICENSE_PUBKEY` | RSA public key of Buckets licenses. If empty, license authentication is disabled. | -| `LICENSE_HASH_SALT` | A hashing salt for the case when a license needs to be disabled. Any random, but consistent value is fine. | - ## Protocol -Relay clients communicate with the relay server using the following protocol. See [./src/bucketsrelay/proto.nim](./src/bucketsrelay/proto.nim) for more information, and [./src/bucketsrelay/stringproto.nim](./src/bucketsrelay/stringproto.nim) for encoding details. +Relay clients communicate with the relay server using the following protocol. See [./src/proto2.nim](./src/proto2.nim) for more information. -In summary, devices connect with websockets and exchange messages. Messages sent from client to server are called commands. Messages sent from server to client are called events. +In summary, devices connect with websockets and exchange messages. Messages sent from client to server are called *commands*. Messages sent from server to client are called *events*. ### Authentication -Clients authenticate with the server in two ways: - -1. With a relay account via HTTP Basic authentication. This is used to group together a user's various clients and prevent abuse. -2. With a public/private key. This is used to identify and connect individual clients. - -A single relay account can have multiple public/private keys; typically one for each device. +Clients authenticate with the server using a public/private key. A single person may have multiple public/private keys; typically one for each device. ### Client Commands Clients send the following commands: -| Command | Description | -|--------------|-------------| -| `Iam` | In response to a `Who` event, proves that this client has the private key for their public key. | -| `Connect` | Asks the server for a connection to another client identified by the client's public key. | -| `Disconnect` | Asks the server to disconnect a connection to another client. | -| `SendData` | Sends bytes to another client. | +| Command | Description | +|----------------|-------------| +| `Iam` | In response to a `Who` event, proves that this client has the private key and does some spam mitigation | +| `PublishNote` | Send a few bytes to another client addressed by topic (good for key exchange) | +| `FetchNote` | Request a note addressed by topic | +| `SendData` | Store/forward bytes to other clients, addressed by relay-authenticated public keys | ### Server Events The relay server sends the following events: -| Event | Description | -|-----------------|-------------| -| `Who` | Challenge for authenticating a client's public/private keys | -| `Authenticated` | Sent when a client successfully completes authentication | -| `Connected` | Sent when a client has connected to another client | -| `Disconnected` | Sent when a client has been disconnected from another client | -| `Data` | Data payload from another, connected client | -| `Entered` | Sent when a client within the same user account has authenticated to the relay | -| `Exited` | Sent when a client within the same user account has disconnected from the relay | -| `ErrorEvent` | Sent when errors happen with authentication, connection or message sending | +| Event | Description | +|-----------------|------------------------------------| +| `Okay` | Sent when certain commands succeed | +| `Error` | Sent when commands fail | +| `Who` | Challenge for authenticating a client's public/private keys and spam mitigation | +| `Note` | Data payload of a note requested by `FetchNote` | +| `Data` | Data payload from another client, addressed by relay-authenticated public key | -### Sequences and Usage - -#### Authentication +### Authentication Authentication happens like this: 1. On connection, server sends `Who(challenge=ABCD...)` -2. Client responds with `Iam(pubkey=MYPK..., signature=SIGN...)` -3. If the signature is correct, server sends `Authenticated` +2. Client responds with a signed PoW hash `Iam(pubkey=MYPK..., signature=SIGN...)` +3. If the signature is correct, server sends `Okay(cmd=Iam)` ``` Client Relay @@ -142,49 +89,99 @@ Client Relay │ Iam │ ├────────────────►│ │ │ - │ Authenticated │ + │ Okay │ │◄────────────────┤ │ │ ``` -#### Client-to-client connection +### Data + +There are 2 ways clients can exchange data: + +1. Notes - public notes that are accessed by knowing the note *topic*. Notes are a good way to do key exchange. Notes expire after a short time. +2. Messages - ordered, stored-and-forwarded messages sent from one client to another client. These are automatically sent to a client upon connection, and deleted when sent. Messages expire after a while. Messages with a non-blank key will overwrite undelivered messages with the same key. + +Build relay clients with the understanding that all forms of exchanging data through this relay are unreliable. + +#### Notes -After authenticating, clients connect to each other and send data like this: +1. Alice sends `PublishNote(topic=apple, data=something)` +2. Bob sends `FetchNote(topic=apple)` +3. Server sends to Bob `Note(topic=apple, data=something)` -1. Alice sends `Connect(pubkey=BOBPK)` -2. Bob sends `Connect(pubkey=ALICEPK)` -3. Server sends Alice `Connected(pubkey=BOBPK)` -4. Server sends Bob `Connected(pubkey=ALICEPK)` -5. Alice sends data with `SendData(data=hello, pubkey=BOBPK)` -6. Server sends Bob data with `Data(data=hello, sender=ALICEPK)` +``` +Alice Relay Bob + │ │ │ + ├───────Authenticated─|─Authenticated──────┤ + │ │ │ + │ PublishNote(apple) | │ + ├────────────────────►│ FetchNote(apple) | + │ │◄───────────────────┤ + │ │ │ + │ │ Note(apple) │ + │ │───────────────────►│ + │ │ │ +``` + +#### Messages + +1. Alice sends `SendData(dst=BOBPK, data=hello)` +2. Server sends to Bob `Data(src=ALICEPK, data=hello)` ``` Alice Relay Bob │ │ │ ├───Authenticated─┼─Authenticated───┤ │ │ │ - │Connect(Bob) │ │ - ├────────────────►│ Connect(Alice) │ - │ │◄────────────────┤ - │ │ │ - │ Connected(Bob) │ Connected(Alice)│ - │◄────────────────┼────────────────►│ - │ │ │ - │SendData(Bob) │ │ + │ SendData(Bob) │ │ ├────────────────►│ Data(Alice) │ │ ├────────────────►│ │ │ │ ``` -#### Same-user presence notifications +#### Messages with keys -The relay server will announce client presence to all clients that use the same HTTP Auth credentials. For example, if both Alice and Bob signed in as `alicenbob@example.com` the following would happen: +1. Alice sends `SendData(dst=BOBPK, key=apple, data=core)` +2. Alice sends `SendData(dst=BOBPK, key=banana, data=boat)` +3. Alice sends `SendData(dst=BOBPK, key=apple, data=pie)` (replacing prior `key=apple` message) +4. Bob connects +5. Server sends to Bob `Data(src=ALICEPK, key=banana, data=boat)` +6. Server sends to Bob `Data(src=ALICEPK, key=apple, data=pie)` -1. Alice finishes authenticating -2. Bob finishes authenticating -3. Server sends Alice `Entered(pubkey=BOBPK)` -4. Server sends Bob `Entered(pubkey=ALICEPK)` -5. Alice disconnects -6. Server sends Bob `Exited(pubkey=ALICEPK)` +``` +Alice Relay Bob + │ │ │ + ├───Authenticated─┤ │ + │ │ │ + │ SendData(Bob,1) │ │ + ├────────────────►│ │ + │ SendData(Bob,2) │ │ + ├────────────────►│ │ + │ SendData(Bob,1) │ │ + ├────────────────►│ │ + │ │ Data(Alice, 2) │ + │ ├────────────────►│ + │ │ Data(Alice, 1) │ + │ ├────────────────►│ + │ │ │ +``` + +#### Chunks +1. Alice sends `StoreChunk(dst=[BOBPK], key=apple, val=seed)` +2. Bob sends `GetChunks(src=ALICEPK, keys=[apple])` +3. Server sends `Chunk(src=ALICEPK, key=apple, val=seed)` +``` +Alice Relay Bob + │ │ │ + ├───────Authenticated─|─Authenticated──────┤ + │ │ │ + │ StoreChunk(apple) | │ + ├────────────────────►│ GetChunks([apple]) | + │ │◄───────────────────┤ + │ │ │ + │ │ Chunk(apple) │ + │ │───────────────────►│ + │ │ │ +``` diff --git a/bucketsrelay.nimble b/bucketsrelay.nimble index 8d22aa5..688d6cf 100644 --- a/bucketsrelay.nimble +++ b/bucketsrelay.nimble @@ -1,39 +1,7 @@ -# Package - version = "0.3.1" author = "Matt Haggard" description = "The relay service for the Buckets budgeting app" license = "MIT" srcDir = "src" -installExt = @["nim", "mustache", "png"] - - -# Dependencies -requires "nim >= 1.6.0" -requires "argparse == 4.0.1" -requires "libsodium == 0.6.0" -requires "mustache == 0.4.3" -requires "ndb == 0.19.9" -requires "https://github.com/status-im/nim-stew.git#d085e48e89062de307aab0d0629fba2f867cb49a" -requires "https://github.com/status-im/nim-serialization.git#9f56a0738c616061382928b9f60e1c5721622f51" -requires "https://github.com/status-im/nim-json-serialization.git#b068e1440d4cb2cf3ede6b3567eaaeecd6c8c96a" -requires "https://github.com/status-im/nim-chronos.git#ba143e029f35fd9b4cd3d89d007cc834d0d5ba3c" -requires "https://github.com/cheatfate/nimcrypto.git#a065c1741836462762d18d2fced1fedd46095b02" -requires "https://github.com/status-im/nim-websock.git#fea05cde8b123b38d1a0a8524b77efbc84daa848" -requires "https://github.com/yglukhov/bearssl_pkey_decoder.git#546f8d9bb887ae1d8a23f62155c583acb0358046" - - -# dependency locks -requires "https://github.com/status-im/nim-zlib.git#826e2fc013f55b4478802d4f2e39f187c50d520a" - -import std/os - -task singleuserbins, "Build single user brelay and bclient bins": - exec("mkdir -p bin") - exec("nimble c -d:relaysingleusermode -o:bin/brelay src/brelay.nim") - exec("nimble c -d:relaysingleusermode -o:bin/bclient src/bclient.nim") -task multiuserbins, "Build multi user brelay and bclient bins": - exec("mkdir -p bin") - exec("nimble c -o:bin/brelay src/brelay.nim") - exec("nimble c -o:bin/bclient src/bclient.nim") +requires "nim >= 2.2.4" diff --git a/changes/fix-Rename-PublicKey-and-20251203-164052.md b/changes/fix-Rename-PublicKey-and-20251203-164052.md new file mode 100644 index 0000000..b208569 --- /dev/null +++ b/changes/fix-Rename-PublicKey-and-20251203-164052.md @@ -0,0 +1 @@ +Rename PublicKey and SecretKey to SignPublicKey and SignSecretKey diff --git a/changes/new-Chunks-are-gone-20251210-102219.md b/changes/new-Chunks-are-gone-20251210-102219.md new file mode 100644 index 0000000..f53efcb --- /dev/null +++ b/changes/new-Chunks-are-gone-20251210-102219.md @@ -0,0 +1 @@ +Chunks are gone in favor of overwriteable Data messages diff --git a/changes/new-Move-to-store-20251028-153340.md b/changes/new-Move-to-store-20251028-153340.md new file mode 100644 index 0000000..accc1d0 --- /dev/null +++ b/changes/new-Move-to-store-20251028-153340.md @@ -0,0 +1 @@ +Change the relay from being a pure relay to having store-and-forward capabilities. diff --git a/changes/new-Notes-must-exist-20251203-140409.md b/changes/new-Notes-must-exist-20251203-140409.md new file mode 100644 index 0000000..f01702f --- /dev/null +++ b/changes/new-Notes-must-exist-20251203-140409.md @@ -0,0 +1 @@ +Notes must exist before you can fetch them, now. diff --git a/changes/new-RelayMessages-have-a-20251203-103527.md b/changes/new-RelayMessages-have-a-20251203-103527.md new file mode 100644 index 0000000..eb3335c --- /dev/null +++ b/changes/new-RelayMessages-have-a-20251203-103527.md @@ -0,0 +1 @@ +RelayMessages have a resp_id to associate them with RelayCommand that caused the message diff --git a/config.nims b/config.nims index 1a49a70..56f95fa 100644 --- a/config.nims +++ b/config.nims @@ -1,3 +1,25 @@ # See LICENSE.md for licensing switch("gc", "orc") -switch("threads", "on") +switch("threads", "off") +switch("d", "useStdLib") # rather than httpBeast + +import os +const ROOT = currentSourcePath.parentDir() +switch("dynlibOverride", "libsodium") + +const archsegment = block: + when hostCPU == "i386": + "x32" + elif hostCPU == "arm64": + "arm64" + else: + "x64" + +when defined(macosx): + switch("passL", ROOT/"libs"/"libsodium"/"macos"/archsegment/"libsodium.a") +elif defined(linux): + switch("cincludes", ROOT/"libs"/"libsodium"/"linux"/archsegment/"include") + switch("clibdir", ROOT/"libs"/"libsodium"/"linux"/archsegment/"lib") + switch("passL", "-lsodium") +elif defined(windows): + switch("passL", ROOT/"libs"/"libsodium"/"win"/archsegment/"libsodium.a") \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..6135e7c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,23 @@ +# -- Stage 1 -- # +FROM nimlang/nim:2.2.6-alpine-regular as builder +WORKDIR /app +RUN apk update && apk add libsodium-static libsodium musl-dev +RUN nimble refresh +RUN nimble install -y https://github.com/iffy/pkger/ +COPY pkger.json . +COPY ./pkger ./pkger +RUN pkger fetch +COPY ./ ./ +COPY docker/config.nims . +RUN find . +RUN nim c -d:release -o:brelay src/bucketsrelay/v2/server2.nim +RUN strip brelay + +# -- Stage 2 -- # +FROM alpine:3.22.2 +WORKDIR /root/ +RUN apk update && apk add libressl4.1-libcrypto sqlite-dev +COPY --from=builder /app/brelay /usr/local/bin/ +RUN mkdir -p /data +EXPOSE 8080 +CMD ["/usr/local/bin/brelay", "--database", "/data/bucketsrelay.sqlite", "server", "--address", "0.0.0.0", "--port", "8080"] diff --git a/docker/config.nims b/docker/config.nims index dd44dfc..11792ee 100644 --- a/docker/config.nims +++ b/docker/config.nims @@ -1,8 +1,6 @@ -switch("gc", "orc") - -when defined(linux): - import os - switch("dynlibOverride", "libsodium") - switch("cincludes", "/usr/include") - switch("clibdir", "/usr/lib") - switch("passL", "-lsodium") +switch("threads", "off") +switch("dynlibOverride", "libsodium") +switch("cincludes", "/usr/include") +switch("clibdir", "/usr/lib") +switch("passL", "-lsodium") +switch("d", "useStdLib") # rather than httpBeast diff --git a/docker/multiuser.Dockerfile b/docker/multiuser.Dockerfile deleted file mode 100644 index de64ce0..0000000 --- a/docker/multiuser.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# -- Stage 1 -- # -FROM nimlang/nim:1.6.18-alpine@sha256:e54f241d4cc4c7e677641a535df6f5cae2e6fa527cb36f53a4c7bd77214b1b80 as builder -WORKDIR /app -RUN apk update && apk add libsodium-static libsodium musl-dev -RUN nimble refresh -RUN nimble install -y nimble -COPY bucketsrelay.nimble . -RUN nimble install -y --depsOnly --verbose -COPY . . -COPY docker/config.nims . -RUN nimble c -o:brelay -d:release src/brelay - -# -- Stage 2 -- # -FROM alpine:3.13.12@sha256:16fd981ddc557fd3b38209d15e7ee8e3e6d9d4d579655e8e47243e2c8525b503 -WORKDIR /root/ -RUN apk update && apk add libressl3.1-libcrypto sqlite-dev -COPY --from=builder /app/brelay /usr/local/bin/ -RUN mkdir -p /data -CMD ["/usr/local/bin/brelay", "--database", "/data/bucketsrelay.sqlite", "server", "--address", "0.0.0.0", "--port", "8080"] diff --git a/docker/singleuser.Dockerfile b/docker/singleuser.Dockerfile deleted file mode 100644 index 1e1aae0..0000000 --- a/docker/singleuser.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -- Stage 1 -- # -FROM nimlang/nim:1.6.18-alpine@sha256:e54f241d4cc4c7e677641a535df6f5cae2e6fa527cb36f53a4c7bd77214b1b80 as builder -WORKDIR /app -RUN apk update && apk add libsodium-static libsodium musl-dev -RUN nimble refresh -RUN nimble install -y nimble -COPY bucketsrelay.nimble . -RUN nimble install -y --depsOnly --verbose -COPY . . -COPY docker/config.nims . -RUN nimble c -o:brelay -d:relaysingleusermode -d:release src/brelay - -# -- Stage 2 -- # -FROM alpine:3.13.12@sha256:16fd981ddc557fd3b38209d15e7ee8e3e6d9d4d579655e8e47243e2c8525b503 -WORKDIR /root/ -RUN apk update && apk add libressl3.1-libcrypto sqlite-dev -COPY --from=builder /app/brelay /usr/local/bin/ -CMD ["/usr/local/bin/brelay", "server", "--address", "0.0.0.0", "--port", "8080"] diff --git a/nim.cfg b/nim.cfg new file mode 100644 index 0000000..804914f --- /dev/null +++ b/nim.cfg @@ -0,0 +1,10 @@ + +### PKGER START - DO NOT EDIT BELOW ######### +--path:"pkger/lazy/lowdb/src" +--path:"pkger/lazy/ws/src" +--path:"pkger/lazy/nimja/src" +--path:"pkger/lazy/db_connector/src" +--path:"pkger/lazy/argparse/src" +--path:"pkger/lazy/libsodium" +--path:"pkger/lazy/jester" +### PKGER END - DO NOT EDIT ABOVE ########### diff --git a/pkger.json b/pkger.json new file mode 100644 index 0000000..15f1ae6 --- /dev/null +++ b/pkger.json @@ -0,0 +1,3 @@ +{ + "dir": "pkger" +} diff --git a/pkger/.gitignore b/pkger/.gitignore new file mode 100644 index 0000000..36f068a --- /dev/null +++ b/pkger/.gitignore @@ -0,0 +1 @@ +lazy diff --git a/pkger/deps.json b/pkger/deps.json new file mode 100644 index 0000000..5dd831f --- /dev/null +++ b/pkger/deps.json @@ -0,0 +1,67 @@ +{ + "pinned": { + "lowdb": { + "pkgname": "lowdb", + "parent": "", + "src": { + "url": "https://github.com/PhilippMDoerner/lowdb", + "kind": "git" + }, + "sha": "c8bd2c631ed923b25c0bab4cd1ccbe84bf56400b" + }, + "ws": { + "pkgname": "ws", + "parent": "", + "src": { + "url": "https://github.com/treeform/ws", + "kind": "git" + }, + "sha": "cbb8f763b436669392d10baec2a45778395395cc" + }, + "nimja": { + "pkgname": "nimja", + "parent": "", + "src": { + "url": "https://github.com/iffy/nimja", + "kind": "git" + }, + "sha": "3ce4c093e3699317b4cdf69111bca38e8e0f2b69" + }, + "db_connector": { + "pkgname": "db_connector", + "parent": "", + "src": { + "url": "https://github.com/nim-lang/db_connector", + "kind": "git" + }, + "sha": "74aef399e5c232f95c9fc5c987cebac846f09d62" + }, + "argparse": { + "pkgname": "argparse", + "parent": "", + "src": { + "url": "https://github.com/iffy/nim-argparse", + "kind": "git" + }, + "sha": "98c7c99bfbcaae750ac515a6fd603f85ed68668f" + }, + "libsodium": { + "pkgname": "libsodium", + "parent": "", + "src": { + "url": "https://github.com/iffy/nim-libsodium", + "kind": "git" + }, + "sha": "d3a116e1047b6b1363dfee08876a8f1c1bb1a128" + }, + "jester": { + "pkgname": "jester", + "parent": "", + "src": { + "url": "https://github.com/dom96/jester/", + "kind": "git" + }, + "sha": "ac9b8541dce64feff9b53b700cab8496c1816651" + } + } +} \ No newline at end of file diff --git a/src/bucketsrelay/asyncstdin.nim b/src/bucketsrelay/v1/asyncstdin.nim similarity index 100% rename from src/bucketsrelay/asyncstdin.nim rename to src/bucketsrelay/v1/asyncstdin.nim diff --git a/src/bclient.nim b/src/bucketsrelay/v1/bclient.nim similarity index 98% rename from src/bclient.nim rename to src/bucketsrelay/v1/bclient.nim index db70a92..05af65b 100644 --- a/src/bclient.nim +++ b/src/bucketsrelay/v1/bclient.nim @@ -11,9 +11,9 @@ import std/os import chronos except debug, info, warn, error -import bucketsrelay/client -import bucketsrelay/proto -import bucketsrelay/asyncstdin +import ./client +import ./proto +import ./asyncstdin type SendHandler = ref object diff --git a/src/brelay.nim b/src/bucketsrelay/v1/brelay.nim similarity index 99% rename from src/brelay.nim rename to src/bucketsrelay/v1/brelay.nim index 192e70d..dfd95ec 100644 --- a/src/brelay.nim +++ b/src/bucketsrelay/v1/brelay.nim @@ -9,8 +9,8 @@ import std/json import chronos -import bucketsrelay/common -import bucketsrelay/server +import ./common +import ./server proc monitorMemory() {.async.} = var diff --git a/src/bucketsrelay/client.nim b/src/bucketsrelay/v1/client.nim similarity index 100% rename from src/bucketsrelay/client.nim rename to src/bucketsrelay/v1/client.nim diff --git a/src/bucketsrelay/common.nim b/src/bucketsrelay/v1/common.nim similarity index 100% rename from src/bucketsrelay/common.nim rename to src/bucketsrelay/v1/common.nim diff --git a/src/bucketsrelay/dbschema.nim b/src/bucketsrelay/v1/dbschema.nim similarity index 100% rename from src/bucketsrelay/dbschema.nim rename to src/bucketsrelay/v1/dbschema.nim diff --git a/src/bucketsrelay/httpreq.nim b/src/bucketsrelay/v1/httpreq.nim similarity index 100% rename from src/bucketsrelay/httpreq.nim rename to src/bucketsrelay/v1/httpreq.nim diff --git a/src/bucketsrelay/jwtrsaonly.nim b/src/bucketsrelay/v1/jwtrsaonly.nim similarity index 100% rename from src/bucketsrelay/jwtrsaonly.nim rename to src/bucketsrelay/v1/jwtrsaonly.nim diff --git a/src/bucketsrelay/licenses.nim b/src/bucketsrelay/v1/licenses.nim similarity index 100% rename from src/bucketsrelay/licenses.nim rename to src/bucketsrelay/v1/licenses.nim diff --git a/src/bucketsrelay/mailer.nim b/src/bucketsrelay/v1/mailer.nim similarity index 100% rename from src/bucketsrelay/mailer.nim rename to src/bucketsrelay/v1/mailer.nim diff --git a/src/bucketsrelay/netstring.nim b/src/bucketsrelay/v1/netstring.nim similarity index 100% rename from src/bucketsrelay/netstring.nim rename to src/bucketsrelay/v1/netstring.nim diff --git a/src/bucketsrelay/proto.nim b/src/bucketsrelay/v1/proto.nim similarity index 100% rename from src/bucketsrelay/proto.nim rename to src/bucketsrelay/v1/proto.nim diff --git a/src/bucketsrelay/server.nim b/src/bucketsrelay/v1/server.nim similarity index 100% rename from src/bucketsrelay/server.nim rename to src/bucketsrelay/v1/server.nim diff --git a/src/bucketsrelay/stringproto.nim b/src/bucketsrelay/v1/stringproto.nim similarity index 100% rename from src/bucketsrelay/stringproto.nim rename to src/bucketsrelay/v1/stringproto.nim diff --git a/src/bucketsrelay/v2/cli.nim b/src/bucketsrelay/v2/cli.nim new file mode 100644 index 0000000..057e01e --- /dev/null +++ b/src/bucketsrelay/v2/cli.nim @@ -0,0 +1,188 @@ +import std/asyncdispatch +import std/base64 +import std/json +import std/options +import std/os +import std/rdstdin +import std/strformat +import std/strutils + +import ./sampleclient +import ./proto2 + +type + CmdContext = object + dst: SignPublicKey + +proc serialize(pubkey: SignPublicKey): string = + base64.encode(pubkey.string) + +proc deserialize(pubkey: typedesc[SignPublicKey], x: string): SignPublicKey = + base64.decode(x).SignPublicKey + +proc serialize(keys: KeyPair): string = + base64.encode($(%* { + "pk": keys.pk.string, + "sk": keys.sk.string, + })) + +proc deserialize(pair: typedesc[KeyPair], x: string): KeyPair = + let j = base64.decode(x).parseJson() + (j["pk"].getStr().SignPublicKey, j["sk"].getStr().SignSecretKey) + +proc loadKeys(src: string): KeyPair = + let parts = src.split(":", 1) + case parts[0] + of "keys": + KeyPair.deserialize(parts[1]) + of "file": + KeyPair.deserialize(readFile(parts[1])) + else: + raise ValueError.newException("Error loading keys") + +proc saveKeys(keys: KeyPair, filename: string) = + writeFile(filename, keys.serialize()) + +proc parseLine(line: string): seq[string] = + result = @[] + var i = 0 + var current = "" + var inQuote = false + var quoteChar = '\0' + + while i < line.len: + let c = line[i] + + if inQuote: + if c == quoteChar: + inQuote = false + result.add(current) + current = "" + else: + current.add(c) + else: + if c in {'\'', '"'}: + inQuote = true + quoteChar = c + if current.len > 0: + result.add(current) + current = "" + elif c == ' ': + if current.len > 0: + result.add(current) + current = "" + elif c == '\\': + inc(i) + if i < line.len: + current.add(line[i]) + else: + current.add(c) + inc(i) + + if current.len > 0: + result.add(current) + + # Handle unclosed quote as literal + if inQuote and current.len > 0: + result.add(current) + +proc use(i: var int, s: seq[string]): string = + result = s[i] + i.inc() + +proc doCommand(client: NetstringClient, full: seq[string], ctx: var CmdContext) = + let cmd = full[0] + let args = full[1..^1] + var i = 0 + case cmd + of "post": + let topic = i.use(args) + let data = i.use(args) + waitFor client.publishNote(topic, data) + echo "posted ", topic + of "fetch": + let topic = i.use(args) + let data = waitFor client.fetchNote(topic) + echo data + of "dst": + if args.len == 0: + ctx.dst = "".SignPublicKey + else: + ctx.dst = SignPublicKey.deserialize(i.use(args)) + echo "dst for future commands set to ", ctx.dst.serialize() + of "send": + var dst = ctx.dst + if dst.string == "": + dst = SignPublicKey.deserialize(i.use(args)) + let val = i.use(args) + let key = if i < args.len: + i.use(args) + else: + "" + waitFor client.sendData(@[dst], val, key) + of "recv": + let data = waitFor client.getData() + echo data + of "help": + echo """ + post TOPIC DATA + fetch TOPIC + dst PUBKEY + Set the destination PUBKEY for future sends + send [PUBKEY] DATA [KEY] + recv + help + """ + else: + echo "Unknown command ", cmd, " ", args + +proc main(url: string, keys: KeyPair, dst = "".SignPublicKey) = + # authenticate + echo "...pubkey: ", keys.pk.serialize + echo "...connecting..." + var client = newRelayClient(url, keys) + var ctx: CmdContext + ctx.dst = dst + echo "...connected" + while true: + let line = readLineFromStdin(&"{ctx.dst.serialize}> ") + if line == nil or line.strip() in ["exit", "quit"]: + echo "...goodbye" + break + + if line.strip() == "": + continue + + let args = parseLine(line) + try: + client.doCommand(args, ctx) + except IndexDefect: + echo "ERROR: " & getCurrentExceptionMsg() + except CatchableError: + echo "ERROR: " & getCurrentExceptionMsg() + +when isMainModule: + import argparse + + var p = newParser: + option("-u", "--url", default = some("ws://127.0.0.1:9000/ws")) + option("-k", "--keys", default = some("file:client1.keys")) + command("genkeys"): + arg("filename") + run: + let keys = genkeys() + keys.saveKeys(opts.filename) + command("repl"): + option("-d", "--dst", help = "Default destination") + run: + var dst = if opts.dst != "": + SignPublicKey.deserialize(opts.dst) + else: + default(SignPublicKey) + main(opts.parentOpts.url, keys = loadKeys(opts.parentOpts.keys), dst = dst) + + try: + p.run(commandLineParams()) + except UsageError as e: + stderr.writeLine getCurrentExceptionMsg() + quit(1) \ No newline at end of file diff --git a/src/bucketsrelay/v2/objs.nim b/src/bucketsrelay/v2/objs.nim new file mode 100644 index 0000000..ecc306b --- /dev/null +++ b/src/bucketsrelay/v2/objs.nim @@ -0,0 +1,474 @@ +# Copyright (c) One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +## These are the objects used for the protocol. +## This file should be kept free of dependencies other than the stdlib +## and should not include async stuff +## as it's meant to be referenced by outside libraries that may +## want to do things their own way. + +import std/hashes +import std/options +import std/sequtils +import std/strformat +import std/strutils + +type + SignPublicKey* = distinct string + SignSecretKey* = distinct string + + Challenge* = tuple + bits: int + rand: string + opslimit: int + memlimit: int + + ChallengeAnswer* = tuple + nonce: int + output: string + signature: string + + MessageKind* = enum + Who + Okay + Error + Note + Data + + ErrorCode* = enum + Generic = 0 + NotAllowed = 1 + TooLarge = 2 + StorageLimitExceeded = 3 + TransferLimitExceeeded = 4 + InvalidParams = 5 + NotFound = 6 + + RelayMessage* = object + resp_id*: int + case kind*: MessageKind + of Who: + who_challenge*: Challenge + of Okay: + ok_cmd*: CommandKind + of Error: + err_code*: ErrorCode + err_message*: string + err_cmd*: CommandKind + of Note: + note_topic*: string + note_data*: string + of Data: + data_key*: string + data_src*: SignPublicKey + data_val*: string + + CommandKind* = enum + Iam + PublishNote + FetchNote + SendData + + RelayCommand* = object + resp_id*: int + case kind*: CommandKind + of Iam: + iam_pubkey*: SignPublicKey + iam_answer*: ChallengeAnswer + of PublishNote: + pub_topic*: string + pub_data*: string + of FetchNote: + fetch_topic*: string + of SendData: + send_key*: string + send_dst*: seq[SignPublicKey] + send_val*: string + +const + RELAY_MAX_TOPIC_SIZE* = 512 + RELAY_MAX_NOTE_SIZE* = 4096 * 2 + RELAY_MAX_NOTES* = 1000 + RELAY_NOTE_DURATION* = 5 * 24 * 60 * 60 + RELAY_MAX_MESSAGE_SIZE* = 65536 * 2 + RELAY_MAX_KEY_SIZE* = 512 + RELAY_MESSAGE_DURATION* = 30 * 24 * 60 * 60 + +const + nicestart = 'a' # '!' + niceend = 'z' # '~' + nicesize = ord(niceend) - ord(nicestart) + +proc nice*(s: string): string = + for c in s: + case c + of {'0'..'9', 'a'..'z', 'A'..'Z', ' '}: result.add c + else: + result.add chr(ord(c) mod nicesize + ord(nicestart)) + +proc abbr*(s: string, size = 6): string = + if s.len > size: + result.add s.substr(0, size) & "..." + else: + result.add(s) + +proc nicelong*(s: string): string = + result = $s.len & ":" & s.nice.abbr & "," + +proc nicelong*(o: Option[string]): string = + if o.isNone: + result = "none" + else: + result = o.get().nicelong() + +proc nice*(k: SignPublicKey): string = nice(k.string) +proc `$`*(k: SignPublicKey): string = k.nice() +proc hash*(p: SignPublicKey): Hash {.borrow.} +proc `==`*(a,b: SignPublicKey): bool {.borrow.} + +proc abbr*(a: SignPublicKey): string = abbr(a.nice) +proc abbr*(a: Option[SignPublicKey]): string = + if a.isSome: + a.get.abbr + else: + "none" + +proc `$`*(ch: Challenge): string = + result = &"({ch.bits} {ch.opslimit} {ch.memlimit} rand={ch.rand.nice})" + +proc `$`*(ans: ChallengeAnswer): string = + result = &"({ans.nonce} {ans.output} {ans.signature.nicelong})" + +proc `$`*(msg: RelayMessage): string = + result.add $msg.kind & "(" + case msg.kind + of Who: + result.add "challenge=" & $msg.who_challenge + of Okay: + result.add &"cmd={msg.ok_cmd}" + of Error: + result.add &"cmd={msg.err_cmd} code={msg.err_code} msg={msg.err_message.nice}" + of Note: + result.add &"'{msg.note_topic.nice.abbr}' val={msg.note_data.nicelong}" + of Data: + result.add &"'{msg.data_key.nice.abbr}' src={msg.data_src.nice.abbr} val={msg.data_val.nicelong}" + result.add ")" + +proc `==`*(a, b: RelayMessage): bool = + if a.kind != b.kind or a.resp_id != b.resp_id: + return false + else: + case a.kind + of Who: + return a.who_challenge == b.who_challenge + of Okay: + return a.ok_cmd == b.ok_cmd + of Error: + return a.err_cmd == b.err_cmd and a.err_code == b.err_code and a.err_message == b.err_message + of Note: + return a.note_data == b.note_data and a.note_topic == b.note_topic + of Data: + return a.data_src == b.data_src and a.data_val == b.data_val and a.data_key == b.data_key + +proc `$`*(cmd: RelayCommand): string = + result.add $cmd.kind & "(" + case cmd.kind + of Iam: + result.add &"{cmd.iam_pubkey.nice.abbr} {cmd.iam_answer}" + of PublishNote: + result.add &"'{cmd.pub_topic.nice.abbr}' val={cmd.pub_data.nicelong}" + of FetchNote: + result.add &"'{cmd.fetch_topic.nice.abbr}'" + of SendData: + result.add &"key={cmd.send_key.nice.abbr} val={cmd.send_val.nicelong} " + result.add cmd.send_dst.mapIt(it.nice.abbr).join(", ") + result.add ")" + +proc `==`*(a, b: RelayCommand): bool = + if a.kind != b.kind or a.resp_id != b.resp_id: + return false + else: + case a.kind + of Iam: + return a.iam_pubkey == b.iam_pubkey and a.iam_answer == b.iam_answer + of PublishNote: + return a.pub_topic == b.pub_topic and a.pub_data == b.pub_data + of FetchNote: + return a.fetch_topic == b.fetch_topic + of SendData: + return a.send_dst == b.send_dst and a.send_val == b.send_val and a.send_key == b.send_key + +#-------------------------------------------------------------- +# serialization +# +# TODO: consider if this should belong in a different file +#-------------------------------------------------------------- + +const MAX_NETSTRING = 65536 + +type + NetstringError* = object of CatchableError + IncompleteNetstring* = object of NetstringError + +proc nsencode*(x: string, terminal = ','): string = + ## Encode a string as a netstring + $len(x) & ":" & x & terminal + +proc nsdecode*(x: string, start: var int, maxlen = MAX_NETSTRING): string = + ## Read the netstring from x starting at index `start` + ## start will be moved to the next netstring location + if x.len == 0: + raise IncompleteNetstring.newException("Empty string is invalid netstring") + var cursor = start + # 1. get length prefix + var expectedLength = 0 + block: + var buf = "" + while true: + let ch = try: + x[cursor] + except IndexDefect: + raise IncompleteNetstring.newException("Not complete") + cursor.inc() + case ch + of '0'..'9': + buf.add(ch) + if buf.parseInt > maxlen: + raise NetstringError.newException("Exceeds max length") + of ':': + if buf.len == 0: + raise NetstringError.newException("Missing starting length") + if buf.len >= 2 and buf[0] == '0': + raise NetstringError.newException("Invalid starting length") + expectedLength = buf.parseInt() + break + else: + raise NetstringError.newException("Invalid length character: " & ch & " at position " & $cursor) + + # 2. check for terminal and length + let terminalIdx = cursor + expectedLength + if terminalIdx >= x.len: + raise IncompleteNetstring.newException("Netstring incomplete") + + let terminalCh = x[terminalIdx] + if terminalCh notin {',','\n'}: + raise NetstringError.newException("Invalid terminal character: " & terminalCh) + + # 3. get string + result = x[cursor..(cursor + expectedLength - 1)] + start = terminalIdx + 1 + +proc nsdecode*(x: string, maxlen = MAX_NETSTRING): string = + var idx = 0 + return nsdecode(x, idx, maxlen = maxlen) + +proc nschop*(x: var string, maxlen = MAX_NETSTRING): string = + ## Get the first netstring from a string and return it. + ## Also remove the first netstring from the passed-in string + var idx = 0 + result = nsdecode(x, idx, maxlen = maxlen) + x.delete(0..(idx-1)) + +proc serialize*(kind: MessageKind): char = + case kind + of Who: '?' + of Okay: '+' + of Error: '-' + of Note: 'n' + of Data: 'd' + +proc deserialize*(kind: typedesc[MessageKind], val: char): MessageKind = + case val + of '?': Who + of '+': Okay + of '-': Error + of 'n': Note + of 'd': Data + else: raise ValueError.newException("Unknown MessageKind: " & val) + +proc serialize*(kind: CommandKind): char = + case kind: + of Iam: 'i' + of PublishNote: 'p' + of FetchNote: 'f' + of SendData: 's' + +proc deserialize*(kind: typedesc[CommandKind], val: char): CommandKind = + case val: + of 'i': Iam + of 'p': PublishNote + of 'f': FetchNote + of 's': SendData + else: raise ValueError.newException("Unknown CommandKind: " & val) + +proc serialize*(err: ErrorCode): char = + chr(err.ord) + +proc deserialize*(typ: typedesc[ErrorCode], ch: char): ErrorCode = + try: + ErrorCode(ord(ch)) + except: + raise ValueError.newException("Unknown ErrorCode: " & ch) + +proc serialize*(chal: Challenge): string = + result.add nsencode($chal.bits) + result.add nsencode(chal.rand) + result.add nsencode($chal.opslimit) + result.add nsencode($chal.memlimit) + +proc deserialize*(typ: typedesc[Challenge], val: string): Challenge = + var idx = 0 + let bits = val.nsdecode(idx).parseInt() + let rand = val.nsdecode(idx) + let opslimit = val.nsdecode(idx).parseInt() + let memlimit = val.nsdecode(idx).parseInt() + return (bits, rand, opslimit, memlimit) + +proc serialize*(ans: ChallengeAnswer): string = + result &= nsencode($ans.nonce) + result &= nsencode(ans.output) + result &= nsencode(ans.signature) + +proc deserialize*(typ: typedesc[ChallengeAnswer], val: string): ChallengeAnswer = + var idx = 0 + return ( + nonce: val.nsdecode(idx).parseInt(), + output: val.nsdecode(idx), + signature: val.nsdecode(idx), + ) + +proc serialize*(keys: seq[SignPublicKey]): string = + for key in keys: + result &= nsencode(key.string) + +proc deserializePubKeys*(val: string): seq[SignPublicKey] = + var val = val + while val.len > 0: + result.add(val.nschop().SignPublicKey) + +proc serialize*(s: seq[string]): string = + for item in s: + result &= nsencode(item) + +proc deserialize*(typ: typedesc[seq[string]], val: string): seq[string] = + var idx = 0 + while idx < val.len: + result.add(val.nsdecode(idx)) + +proc serialize*(msg: RelayMessage): string = + result &= msg.kind.serialize() + # For Who and Data messages, resp_id is always omitted (always 0) + if msg.kind notin {Who, Data}: + result &= nsencode($msg.resp_id) + case msg.kind + of Who: + result &= msg.who_challenge.serialize() + of Okay: + result &= msg.ok_cmd.serialize() + of Error: + result &= msg.err_cmd.serialize() + result &= msg.err_code.serialize() + result &= msg.err_message + of Note: + result &= msg.note_topic.nsencode + result &= msg.note_data.nsencode + of Data: + result &= msg.data_key.nsencode + result &= msg.data_src.string.nsencode + result &= msg.data_val.nsencode + +proc deserialize*(typ: typedesc[RelayMessage], s: string): RelayMessage = + if s.len == 0: + raise ValueError.newException("Empty RelayMessage") + var idx = 0 + let kind = MessageKind.deserialize(s[idx]) + idx.inc() + # For Who and Data messages, resp_id is always 0 and not serialized + let resp_id = if kind in {Who, Data}: + 0 + else: + s.nsdecode(idx).parseInt() + case kind + of Who: + return RelayMessage(kind: Who, resp_id: resp_id, who_challenge: Challenge.deserialize(s[idx..^1])) + of Okay: + return RelayMessage(kind: Okay, resp_id: resp_id, ok_cmd: CommandKind.deserialize(s[idx])) + of Error: + return RelayMessage( + kind: Error, + resp_id: resp_id, + err_cmd: CommandKind.deserialize(s[idx]), + err_code: ErrorCode.deserialize(s[idx+1]), + err_message: s[(idx+2)..^1] + ) + of Note: + return RelayMessage( + kind: Note, + resp_id: resp_id, + note_topic: s.nsdecode(idx), + note_data: s.nsdecode(idx), + ) + of Data: + return RelayMessage( + kind: Data, + resp_id: resp_id, + data_key: s.nsdecode(idx), + data_src: s.nsdecode(idx).SignPublicKey, + data_val: s.nsdecode(idx), + ) + +proc serialize*(cmd: RelayCommand): string = + result &= cmd.kind.serialize + result &= nsencode($cmd.resp_id) + case cmd.kind + of Iam: + result &= cmd.iam_pubkey.string.nsencode + result &= cmd.iam_answer.serialize().nsencode + of PublishNote: + result &= cmd.pub_topic.nsencode + result &= cmd.pub_data.nsencode + of FetchNote: + result &= cmd.fetch_topic.nsencode + of SendData: + result &= cmd.send_key.nsencode + result &= nsencode(cmd.send_dst.serialize()) + result &= cmd.send_val.nsencode + +proc deserialize*(typ: typedesc[RelayCommand], s: string): RelayCommand = + if s.len == 0: + raise ValueError.newException("Empty RelayCommand") + var idx = 0 + let kind = CommandKind.deserialize(s[idx]) + idx.inc() + let resp_id = s.nsdecode(idx).parseInt() + case kind + of Iam: + return RelayCommand( + kind: Iam, + resp_id: resp_id, + iam_pubkey: s.nsdecode(idx).SignPublicKey, + iam_answer: ChallengeAnswer.deserialize(s.nsdecode(idx)), + ) + of PublishNote: + return RelayCommand( + kind: PublishNote, + resp_id: resp_id, + pub_topic: s.nsdecode(idx), + pub_data: s.nsdecode(idx), + ) + of FetchNote: + return RelayCommand( + kind: FetchNote, + resp_id: resp_id, + fetch_topic: s.nsdecode(idx), + ) + of SendData: + return RelayCommand( + kind: SendData, + resp_id: resp_id, + send_key: s.nsdecode(idx), + send_dst: deserializePubKeys(s.nsdecode(idx)), + send_val: s.nsdecode(idx), + ) diff --git a/src/bucketsrelay/v2/proto2.nim b/src/bucketsrelay/v2/proto2.nim new file mode 100644 index 0000000..a5320a1 --- /dev/null +++ b/src/bucketsrelay/v2/proto2.nim @@ -0,0 +1,644 @@ +# Copyright (c) One Part Rain, LLC. All rights reserved. +# +# This work is licensed under the terms of the MIT license. +# For a copy, see LICENSE.md in this repository. + +import std/base64 +import std/logging +import std/options +import std/strutils +import std/strformat +import std/tables +import std/times + +import lowdb/sqlite +import libsodium/sodium +import libsodium/sodium_sizes + +import ./objs; export objs + +const LOG_COMMS* = not defined(release) or defined(relaynologcomms) +const TESTMODE = defined(testmode) and not defined(release) + +type + KeyPair* = tuple + pk: SignPublicKey + sk: SignSecretKey + + Relay*[T] = object + db*: DbConn + clients: TableRef[SignPublicKey, RelayConnection[T]] + max_chunk_space*: int + max_transfer_rate*: int + + RelayConnection*[T] = ref object + sender*: T + pubkey*: Option[SignPublicKey] ## The authenticated pubkey + challenge: Option[Challenge] + relay*: Relay[T] + ip*: string + +when TESTMODE: + var TIME_SKEW = 0 + proc skewTime*(seconds: int) = + TIME_SKEW += seconds + proc skewTime*(dur: Duration) = + TIME_SKEW += dur.inSeconds() + proc resetSkew*() = + TIME_SKEW = 0 + +#------------------------------------------------------------------- +# Utilities +#------------------------------------------------------------------- +proc genkeys*(): KeyPair = + let (pk, sk) = crypto_sign_keypair() + result = (pk.SignPublicKey, sk.SignSecretKey) + +proc sign*(key: SignSecretKey, message: string): string = + ## Sign a message with the given secret key + result = crypto_sign_detached(key.string, message) + +proc is_valid_signature*(key: SignPublicKey, plaintext: string, signature: string): bool = + try: + crypto_sign_verify_detached(key.string, plaintext, signature) + return true + except SodiumError: + return false + except CatchableError: + return false + +const + CHALLENGE_BITS = when TESTMODE: 1 else: 5 + +proc generateChallenge*(bits = CHALLENGE_BITS, opslimit = crypto_pwhash_opslimit_interactive().int, memlimit = crypto_pwhash_memlimit_interactive().int): Challenge = + return ( + bits: bits, + rand: randombytes(32) & $epochTime(), + opslimit: opslimit, + memlimit: memlimit, + ) + +proc sigContents*(ch: Challenge, nonce: int, output: string): string = + ch.serialize & nsencode($nonce) & output + +proc firstBits(s: string, n: int): string = + ## Returns the first `n` bits of the string `s` as a binary string. + if s.len * 8 < n: + raise ValueError.newException("String not long enough") + var bitsLeft = n + for i in 0.. 0: + result.add(if (byte and (1'u8 shl bit)) != 0: '1' else: '0') + dec bitsLeft + else: + return result + if bitsLeft > 0: + # this should never happen, but just in case + raise ValueError.newException("String not long enough") + return result + +proc answer*(ch: Challenge, sk: SignSecretKey): ChallengeAnswer = + ## Answer a hashcash challenge and sign the result + var nonce = 0 + let serialized = ch.serialize() + var start = getTime() + var expected_prefix = '0'.repeat(ch.bits) + while true: + let inp = serialized & ":" & $nonce + let output = crypto_pwhash_str(inp, + opslimit = ch.opslimit.csize_t, + memlimit = ch.memlimit.csize_t) + let hashpart = base64.decode(output.split('$')[^1]) + let bits = hashpart.firstBits(ch.bits) + if bits == expected_prefix: + var diff = getTime() - start + return ( + nonce: nonce, + output: output, + signature: sk.sign(sigContents(ch, nonce, output)), + ) + nonce.inc() + +proc is_valid_answer*(pk: SignPublicKey, ch: Challenge, answer: ChallengeAnswer): bool = + ## Verify the signed challenge answer + if not pk.is_valid_signature(sigContents(ch, answer.nonce, answer.output), answer.signature): + return false + let inp = ch.serialize() & ":" & $answer.nonce + if crypto_pwhash_str_verify(answer.output, inp) == false: + return false + return true + + +#------------------------------------------------------------------- +# Database +#------------------------------------------------------------------- +func strval*(dbval: sqlite.DbValue): string = + case dbval.kind + of dvkString: + dbval.s + of dvkNull: + "" + else: + raise ValueError.newException("Can't get string from " & $dbval.kind) + +proc dbValue*(p: SignPublicKey): DbValue = + dbValue(p.string.DbBlob) + +proc fromDB*(t: typedesc[SignPublicKey], v: DbBlob): SignPublicKey = + v.string.SignPublicKey + +template patch(db: untyped, applied: seq[string], name: string, body: untyped): untyped = + block: + if name notin applied: + info name, " - applying..." + db.exec(sql"BEGIN") + try: + body + db.exec(sql"INSERT INTO _schema_patches (name) VALUES (?)", name) + db.exec(sql"COMMIT") + except CatchableError: + error name, " - error applying patch: " & getCurrentExceptionMsg() + db.exec(sql"ROLLBACK") + raise + else: + debug name, " - applied" + +proc updateSchema*(db: DbConn) = + db.exec(sql"PRAGMA foreign_keys = ON") + + ## Upgrade the schema + db.exec(sql"""CREATE TABLE IF NOT EXISTS _schema_patches ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE, + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP + )""") + + var applied: seq[string] + for row in db.getAllRows(sql"SELECT name FROM _schema_patches"): + applied.add(row[0].strval) + + info "Already applied patches: ", applied.join(",") + + db.patch(applied, "initial"): + # note + db.exec(sql"""CREATE TABLE note ( + topic TEXT PRIMARY KEY, + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + src TEXT NOT NULL, + data BLOB DEFAULT '' + )""") + db.exec(sql"CREATE INDEX note_created ON note(created)") + db.exec(sql"CREATE INDEX note_src ON note(src)") + + # message - stores message data once + db.exec(sql"""CREATE TABLE message_data ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + key BLOB NOT NULL, + src BLOB NOT NULL, + data BLOB NOT NULL + )""") + db.exec(sql"CREATE INDEX message_data_created ON message_data(created)") + + # message recipients - tracks who needs to receive each message + db.exec(sql"""CREATE TABLE message_recipient ( + message_id INTEGER NOT NULL, + dst BLOB NOT NULL, + src BLOB NOT NULL, + key BLOB NOT NULL, + PRIMARY KEY (message_id, dst), + FOREIGN KEY (message_id) REFERENCES message_data(id) ON DELETE CASCADE + )""") + db.exec(sql"CREATE INDEX message_recipient_dst ON message_recipient(dst)") + # Ensure only one message per (dst, src, key) when key is not empty + # Different senders can send messages with the same key to the same recipient + db.exec(sql"""CREATE UNIQUE INDEX message_recipient_dst_src_key + ON message_recipient(dst, src, key) + WHERE key != x'' + """) + + # Automatically delete message_data when last recipient is removed + db.exec(sql"""CREATE TRIGGER cleanup_message_data + AFTER DELETE ON message_recipient + WHEN NOT EXISTS ( + SELECT 1 FROM message_recipient WHERE message_id = OLD.message_id + ) + BEGIN + DELETE FROM message_data WHERE id = OLD.message_id; + END + """) + + # stats + db.exec(sql"""CREATE TABLE stats_transfer ( + period TEXT NOT NULL DEFAULT(strftime('%Y-%W')), + ip TEXT NOT NULL, + pubkey TEXT NOT NULL, + data_in INTEGER DEFAULT 0, + data_out INTEGER DEFAULT 0, + PRIMARY KEY (period, ip, pubkey) + )""") + db.patch(applied, "connects"): + db.exec(sql"""CREATE TABLE stats_event ( + period TEXT NOT NULL DEFAULT(strftime('%Y-%W')), + ip TEXT NOT NULL, + pubkey TEXT NOT NULL, + connect INTEGER DEFAULT 0, + publish INTEGER DEFAULT 0, + send INTEGER DEFAULT 0, + PRIMARY KEY (period, ip, pubkey) + )""") + + +#------------------------------------------------------------------- +# Relay code +#------------------------------------------------------------------- + +proc `$`*[T](conn: RelayConnection[T]): string = + result = "RelayConnectiong(" + result &= &"pubkey={conn.pubkey.abbr} " + result &= &"sender={conn.sender}" + if conn.challenge.isSome: + result &= " cha=" & base64.encode(conn.challenge.get()) + result &= ")" + +proc `$`*[T](tab: TableRef[SignPublicKey, RelayConnection[T]]): string = + result = "TableRef(" + for key in tab.keys(): + let val = tab[key] + result.add &"{key}: {val}, " + result &= ")" + +proc newRelay*[T](db: DbConn): Relay[T] = + when TESTMODE: + resetSkew() + result.db = db + result.clients = newTable[SignPublicKey, RelayConnection[T]]() + db.updateSchema() + +template sendError*[T](conn: RelayConnection[T], cmd: RelayCommand, msg: string, code: ErrorCode) = + conn.sendMessage(RelayMessage( + kind: Error, + resp_id: cmd.resp_id, + err_code: code, + err_message: msg, + err_cmd: cmd.kind, + )) + +template sendOkay*[T](conn: RelayConnection[T], cmd: RelayCommand) = + conn.sendMessage(RelayMessage( + kind: Okay, + resp_id: cmd.resp_id, + ok_cmd: cmd.kind, + )) + +proc is_valid*(x: SignPublicKey): bool = + ## Return true if it looks like a valid public key + if x.string.len == 32: + return true + return false + +proc any_invalid(x: seq[SignPublicKey]): bool = + ## Return true if any of the public keys are invalid + for pk in x: + if not pk.is_valid(): + return true + return false + +proc initAuth*[T](relay: Relay[T], client: T): RelayConnection[T] = + new(result) + result.sender = client + result.challenge = some(generateChallenge()) + result.sendMessage(RelayMessage( + kind: Who, + resp_id: 0, # Who messages are not triggered by a command + who_challenge: result.challenge.get(), + )) + result.relay = relay + +proc disconnect*[T](relay: Relay[T], conn: RelayConnection[T]) = + if conn.pubkey.isSome: + let pubkey = conn.pubkey.get() + relay.clients.del(pubkey) + info &"[{conn.pubkey.abbr}] disconnected" + +#------------------------------------------------------------------- +# stats +#------------------------------------------------------------------- +type + TransferTotal* = tuple + data_in: int + data_out: int + ip: string + pubkey: SignPublicKey + period: string + + PeriodRange* = tuple + a: string + b: string + +proc record_transfer_stat*(db: DbConn, ip: string, pubkey = "".SignPublicKey, data_in = 0, data_out = 0) = + db.exec(sql""" + INSERT INTO stats_transfer (ip, pubkey, data_in, data_out) + VALUES (?, ?, ?, ?) + ON CONFLICT(period, ip, pubkey) DO UPDATE SET + data_in = data_in + excluded.data_in, + data_out = data_out + excluded.data_out; + """, ip, pubkey, data_in, data_out) + +when TESTMODE: + proc record_transfer_stat_period*(db: DbConn, ip: string, pubkey = "".SignPublicKey, period = "", data_in = 0, data_out = 0) = + db.exec(sql""" + INSERT INTO stats_transfer (ip, pubkey, period, data_in, data_out) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(period, ip, pubkey) DO UPDATE SET + data_in = data_in + excluded.data_in, + data_out = data_out + excluded.data_out; + """, ip, pubkey, period, data_in, data_out) + +proc record_event_stat*(db: DbConn, ip: string, pubkey: SignPublicKey, connect = 0, publish = 0, send = 0) = + db.exec(sql""" + INSERT INTO stats_event (ip, pubkey, connect, publish, send) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(period, ip, pubkey) DO UPDATE SET + connect = connect + excluded.connect, + publish = publish + excluded.publish, + send = send + excluded.send + """, ip, pubkey, connect, publish, send) + +proc current_data_in*(db: DbConn, pubkey: SignPublicKey): int = + ## Return the amount of data that has been transferred in by the given + ## public key for the current time period + db.getRow(sql""" + SELECT coalesce(sum(data_in), 0) FROM stats_transfer + WHERE + pubkey = ? + AND period = strftime('%Y-%W') + """, pubkey).get()[0].i.int + +proc stats_transfer_total*(db: DbConn, ip = "", pubkey = "".SignPublicKey, period = ""): TransferTotal = + var query = "SELECT sum(data_in), sum(data_out) FROM stats_transfer" + var whereparts: seq[string] + var params: seq[DbValue] + var groupby: seq[string] + if period != "": + whereparts.add "period=?" + params.add(period.dbValue()) + if ip != "": + whereparts.add "ip=?" + params.add(ip.dbValue()) + if pubkey.string != "": + whereparts.add "pubkey=?" + params.add(pubkey.dbValue()) + if whereparts.len > 0: + query &= " WHERE " & whereparts.join(" AND ") + let orow = db.getRow(sql(query), params) + if orow.isSome(): + let row = orow.get() + return (row[0].i.int, row[1].i.int, ip, pubkey, period) + +#------------------------------------------------------------------- +# pub/sub notes +#------------------------------------------------------------------- + +proc delExpiredNotes(relay: Relay) = + let offset = when TESTMODE: + -RELAY_NOTE_DURATION + TIME_SKEW + else: + -RELAY_NOTE_DURATION + let offstring = &"{offset} seconds" + relay.db.exec(sql"DELETE FROM note WHERE created <= datetime('now', ?)", offstring) + +proc popNote(relay: Relay, topic: string): Option[string] = + let db = relay.db + relay.delExpiredNotes() + db.exec(sql"BEGIN") + try: + let orow = db.getRow(sql"SELECT data FROM note WHERE topic=?", topic.DbBlob) + if orow.isSome: + let row = orow.get() + result = some(row[0].b.string) + info &"[note] pop {topic}" + db.exec(sql"DELETE FROM note WHERE topic=?", topic.DbBlob) + else: + debug &"[note] dne {topic}" + db.exec(sql"COMMIT") + except CatchableError: + warn &"[note] error " & getCurrentExceptionMsg() + db.exec(sql"ROLLBACK") + +proc noteCount(relay: Relay, pubkey: SignPublicKey): int = + ## Return the number of notes currently published by this ip + relay.db.getRow(sql"SELECT count(*) FROM note WHERE src = ?", pubkey).get()[0].i.int + +#------------------------------------------------------------------- +# send/receive data +#------------------------------------------------------------------- + +proc delExpiredMessages(relay: Relay) = + let offset = when TESTMODE: + -RELAY_MESSAGE_DURATION + TIME_SKEW + else: + -RELAY_MESSAGE_DURATION + let offstring = &"{offset} seconds" + # Delete expired message data; CASCADE will automatically delete associated recipients + relay.db.exec(sql"DELETE FROM message_data WHERE created <= datetime('now', ?)", offstring) + +proc nextMessage(relay: Relay, dst: SignPublicKey): Option[RelayMessage] = + let orow = relay.db.getRow(sql""" + SELECT md.key, md.src, md.data, md.id + FROM message_recipient mr + JOIN message_data md ON mr.message_id = md.id + WHERE + mr.dst = ? + ORDER BY + md.created ASC, + md.id ASC + LIMIT 1""", dst) + if orow.isSome: + let row = orow.get() + let message_id = row[3].i + result = some(RelayMessage( + kind: Data, + resp_id: 0, # Data messages are not triggered by recipient's command + data_key: row[0].b.string, + data_src: SignPublicKey.fromDB(row[1].b), + data_val: row[2].b.string, + )) + + # Delete this recipient entry (trigger will auto-cleanup message_data if this was the last recipient) + relay.db.exec(sql"DELETE FROM message_recipient WHERE message_id = ? AND dst = ?", message_id, dst) + +#------------------------------------------------------------------- +# relay command handling +#------------------------------------------------------------------- + +proc handleCommand*[T](relay: Relay[T], conn: var RelayConnection[T], cmd: RelayCommand) = + when LOG_COMMS: + info "[" & conn.pubkey.abbr & "] DO " & $cmd + if conn.pubkey.isNone and cmd.kind != Iam: + conn.sendError(cmd, "Not allowed", NotAllowed) + return + + case cmd.kind + of Iam: + if conn.challenge.isNone: + conn.sendError(cmd, "Already authenticated", Generic) + return + let challenge = conn.challenge.get() + conn.challenge = none[Challenge]() # disable future authentication attempts + + try: + if not is_valid_answer(cmd.iam_pubkey, challenge, cmd.iam_answer): + conn.sendError(cmd, "Invalid answer", Generic) + return + except CatchableError: + conn.sendError(cmd, "Invalid answer", Generic) + return + + # successful connection + let pubkey = cmd.iam_pubkey + conn.pubkey = some(pubkey) + relay.clients[pubkey] = conn + info &"[{conn.pubkey.abbr}] connected" + conn.sendOkay(cmd) + relay.db.record_event_stat( + ip = conn.ip, + pubkey = pubkey, + connect = 1, + ) + + # send all queued messages + relay.delExpiredMessages() + while true: + let nexto = relay.nextMessage(pubkey) + if nexto.isSome: + let msg = nexto.get() + conn.sendMessage(msg) + if msg.kind == Data: + relay.db.record_transfer_stat( + ip = conn.ip, + pubkey = pubkey, + data_out = msg.data_val.len + msg.data_key.len, + ) + else: + break + of PublishNote: + if cmd.pub_topic.len > RELAY_MAX_TOPIC_SIZE: + conn.sendError(cmd, "Topic too long", TooLarge) + elif cmd.pub_data.len > RELAY_MAX_NOTE_SIZE: + conn.sendError(cmd, "Data too long", TooLarge) + else: + let pubkey = conn.pubkey.get() + if relay.noteCount(pubkey) >= RELAY_MAX_NOTES: + conn.sendError(cmd, "Too many notes", StorageLimitExceeded) + else: + relay.db.record_transfer_stat( + ip = conn.ip, + pubkey = pubkey, + data_in = cmd.pub_data.len, + ) + relay.db.record_event_stat( + ip = conn.ip, + pubkey = pubkey, + publish = 1, + ) + try: + relay.db.exec(sql"INSERT INTO note (topic, data, src) VALUES (?, ?, ?)", + cmd.pub_topic.DbBlob, + cmd.pub_data.DbBlob, + pubkey, + ) + conn.sendOkay(cmd) + except CatchableError: + conn.sendError(cmd, "Duplicate topic", Generic) + of FetchNote: + if cmd.fetch_topic.len > RELAY_MAX_TOPIC_SIZE: + conn.sendError(cmd, "Topic too long", TooLarge) + else: + let odata = relay.popNote(cmd.fetch_topic) + if odata.isSome(): + # the note is already here + let data = odata.get() + conn.sendMessage(RelayMessage( + kind: Note, + resp_id: cmd.resp_id, # Response to FetchNote command + note_data: data, + note_topic: cmd.fetch_topic, + )) + relay.db.record_transfer_stat( + ip = conn.ip, + pubkey = conn.pubkey.get(), + data_out = data.len, + ) + else: + # the note doesn't exist + conn.sendError(cmd, "Topic not found", NotFound) + of SendData: + if cmd.send_val.len > RELAY_MAX_MESSAGE_SIZE: + conn.sendError(cmd, "Data too long", TooLarge) + elif cmd.send_key.len > RELAY_MAX_KEY_SIZE: + conn.sendError(cmd, "Key too long", TooLarge) + elif cmd.send_dst.any_invalid(): + conn.sendError(cmd, "Invalid pubkey", InvalidParams) + else: + let pubkey = conn.pubkey.get() + if relay.max_transfer_rate != 0 and relay.db.current_data_in(pubkey) > relay.max_transfer_rate: + conn.sendError(cmd, "Rate limit exceeded", TransferLimitExceeeded) + else: + relay.db.record_transfer_stat( + ip = conn.ip, + pubkey = pubkey, + data_in = cmd.send_val.len + cmd.send_key.len, + ) + relay.db.record_event_stat( + ip = conn.ip, + pubkey = pubkey, + send = 1, + ) + + # Collect offline recipients + var offline_dsts: seq[SignPublicKey] + for dst_pubkey in cmd.send_dst: + if relay.clients.hasKey(dst_pubkey): + # dst is online - send directly + var other_conn = relay.clients[dst_pubkey] + other_conn.sendMessage(RelayMessage( + kind: Data, + resp_id: 0, # Not triggered by other_conn's command + data_key: cmd.send_key, + data_src: pubkey, + data_val: cmd.send_val, + )) + relay.db.record_transfer_stat( + ip = other_conn.ip, + pubkey = other_conn.pubkey.get(), + data_out = cmd.send_val.len + cmd.send_key.len, + ) + else: + # dst is offline - collect for batch storage + offline_dsts.add(dst_pubkey) + + # Store message data once for all offline recipients + if offline_dsts.len > 0: + relay.db.exec(sql"BEGIN") + try: + # Insert message data once and get the ID + let message_id = relay.db.insertID(sql""" + INSERT INTO message_data (key, src, data) + VALUES (?, ?, ?)""", + cmd.send_key.DbBlob, pubkey, cmd.send_val.DbBlob) + + # Add recipient entries for each offline destination + # Use INSERT OR REPLACE to handle duplicate (dst, src, key) - database-enforced + for dst_pubkey in offline_dsts: + relay.db.exec(sql""" + INSERT OR REPLACE INTO message_recipient (message_id, dst, src, key) + VALUES (?, ?, ?, ?)""", + message_id, dst_pubkey, pubkey, cmd.send_key.DbBlob) + + relay.db.exec(sql"COMMIT") + except CatchableError: + relay.db.exec(sql"ROLLBACK") + raise diff --git a/src/bucketsrelay/v2/sampleclient.nim b/src/bucketsrelay/v2/sampleclient.nim new file mode 100644 index 0000000..4589901 --- /dev/null +++ b/src/bucketsrelay/v2/sampleclient.nim @@ -0,0 +1,92 @@ +import std/asyncdispatch +import std/logging +import std/options + +import ws + +import ./objs +import ./proto2 + +type + NetstringClient* = ref object + buf: string + socket: WebSocket + +proc newNetstringClient*(sock: WebSocket): NetstringClient = + new(result) + result.socket = sock + +proc receiveString*(ns: NetstringClient): Future[string] {.async.} = + while true: + try: + return ns.buf.nschop() + except IncompleteNetstring: + discard + let packet = await ns.socket.receiveStrPacket() + ns.buf &= packet + +proc sendString*(ns: NetstringClient, msg: string): Future[void] {.async.} = + await ns.socket.send(nsencode(msg)) + +proc sendCommand*(ns: NetstringClient, cmd: RelayCommand): Future[void] {.async.} = + when LOG_COMMS: + info "[client] -> " & $cmd + await ns.sendString(cmd.serialize()) + +proc receiveMessage*(ns: NetstringClient): Future[RelayMessage] {.async.} = + let s = await ns.receiveString() + let res = RelayMessage.deserialize(s) + when LOG_COMMS: + info "[client] <- " & $res + return res + +proc newNetstringClient*(url: string): NetstringClient = + let ws = waitFor newWebSocket(url) + return newNetstringClient(ws) + +proc newRelayClient*(url: string, keys: KeyPair): NetstringClient = + var ns = newNetstringClient(url) + let who = waitFor ns.receiveMessage() + let answer = who.who_challenge.answer(keys.sk) + waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: keys.pk)) + let ok = waitFor ns.receiveMessage() + doAssert ok.kind == Okay, $ok + return ns + +proc publishNote*(ns: NetstringClient, topic: string, data: string) {.async.} = + await ns.sendCommand(RelayCommand( + kind: PublishNote, + pub_topic: topic, + pub_data: data, + )) + let res = await ns.receiveMessage() + if res.kind == Okay: + discard + elif res.kind == Error: + raise ValueError.newException("Error publishing note: " & $res.err_code & " " & res.err_message) + +proc fetchNote*(ns: NetstringClient, topic: string): Future[string] {.async.} = + await ns.sendCommand(RelayCommand( + kind: FetchNote, + fetch_topic: topic, + )) + let res = await ns.receiveMessage() + if res.kind == Note: + return res.note_data + else: + raise ValueError.newException("No such note: " & topic) + +proc sendData*(ns: NetstringClient, dst: seq[SignPublicKey], val: string, key: string) {.async.} = + await ns.sendCommand(RelayCommand( + kind: SendData, + send_key: key, + send_dst: dst, + send_val: val, + )) + +proc getData*(ns: NetstringClient): Future[tuple[key: string, val: string]] {.async.} = + let res = await ns.receiveMessage() + if res.kind == Data: + return (res.data_key, res.data_val) + else: + raise ValueError.newException("Expecting Data but got: " & $res) diff --git a/src/bucketsrelay/v2/server2.nim b/src/bucketsrelay/v2/server2.nim new file mode 100644 index 0000000..a8cd936 --- /dev/null +++ b/src/bucketsrelay/v2/server2.nim @@ -0,0 +1,389 @@ +import std/asyncdispatch +import std/base64 +import std/deques +import std/httpcore +import std/logging +import std/os +import std/strformat +import std/strutils + +import jester +import nimja +import ws +import ws/jester_extra +import lowdb/sqlite +import libsodium/sodium + +import ./proto2 +import ./objs + +type + NetstringSocket = ref object + buf: string + socket: WebSocket + ip: string + pubkey: Option[SignPublicKey] + + QueuedMessage = tuple + socket: NetstringSocket + msg: RelayMessage + +const + RELAY_VERSION = slurp"../../../CHANGELOG.md".split(" ")[1] + logo_png = slurp"./static/logo.png" + favicon_png = slurp"./static/favicon.png" + +echo "RELAY_VERSION: ", RELAY_VERSION + +let ADMIN_USERNAME = getEnv("ADMIN_USERNAME", "admin") +let ADMIN_PWHASH = when defined(release): + getEnv("ADMIN_PWHASH", "") + else: + # the password is 'admin' + getEnv("ADMIN_PWHASH", "$argon2id$v=19$m=262144,t=3,p=1$AxXWW9mRuyJjWWbxa4WYoQ$xHAyhzWgKGFH+amM4D1GMuNsPSjGNNp40MueB9dJkgA") + +var relay: Relay[NetstringSocket] +var message_queue = initDeque[QueuedMessage]() + +proc trueClientIP(request: Request): string = + ## Return the true, originating client IP of a request + # CF-Connecting-IP (cloudflare) + result = request.headers.getOrDefault("cf-connecting-ip") + if result != "": + return result + # True-Client-IP (cloudflare) + result = request.headers.getOrDefault("true-client-ip") + if result != "": + return result + # X-Real-IP (nginx) + result = request.headers.getOrDefault("x-real-ip") + if result != "": + return result + result = request.ip + +proc isAdmin(request: Request): bool = + if not request.headers.hasKey("Authorization"): + return false + let authHeader = request.headers["Authorization"] + let encodedCreds = authHeader[("Basic ".len)..^1] + + try: + let decodedCreds = base64.decode(encodedCreds) + let parts = decodedCreds.split(":", 1) + if parts.len == 2: + let username = parts[0] + let password = parts[1] + return sodium.memcmp(username, ADMIN_USERNAME) and crypto_pwhash_str_verify(ADMIN_PWHASH, password) + except CatchableError: + return false + + return false + +proc wcommas(x: int): string = insertSep($x, sep = ',') + +proc newNetstringSocket(sock: WebSocket, ip: string): NetstringSocket = + new(result) + result.socket = sock + result.ip = ip + +proc receiveString(ns: NetstringSocket): Future[string] {.async.} = + while true: + try: + return ns.buf.nschop() + except IncompleteNetstring: + discard + let packet = await ns.socket.receiveStrPacket() + ns.buf &= packet + +proc sendString(ns: NetstringSocket, msg: string): Future[void] {.async.} = + let tosend = nsencode(msg) + await ns.socket.send(tosend) + +proc receiveCommand(relay: Relay, ns: NetstringSocket): Future[RelayCommand] {.async.} = + let s = await ns.receiveString() + return RelayCommand.deserialize(s) + +proc sendMessage(ns: NetstringSocket, msg: RelayMessage) {.async.} = + await ns.sendString(msg.serialize()) + +proc sendMessage(conn: RelayConnection[NetstringSocket], msg: RelayMessage) = + message_queue.addLast((conn.sender, msg)) + +proc sendQueuedMessages() {.async.} = + while message_queue.len > 0: + let (sock, msg) = message_queue.popFirst() + await sock.sendMessage(msg) + +proc handleWebsocket(req: Request) {.async, gcsafe.} = + var ws = await newWebSocket(req) + var ns = newNetstringSocket(ws, req.trueClientIP()) + var conn = relay.initAuth(ns) + conn.ip = req.trueClientIP() + await sendQueuedMessages() + while ns.socket.readyState == Open: + let cmd = try: + await relay.receiveCommand(ns) + except WebSocketClosedError: + break + except WebSocketProtocolMismatchError: + warn "Socket tried to use an unknown protocol: ", getCurrentExceptionMsg() + break + except WebSocketError: + warn "Unexpected socket error: ", getCurrentExceptionMsg() + break + except CatchableError: + warn "CatchableError: ", getCurrentExceptionMsg() + break + relay.handleCommand(conn, cmd) + if cmd.kind == Iam: + ns.pubkey = conn.pubkey + await sendQueuedMessages() + relay.disconnect(conn) + +type + StorageStat = tuple + pubkey: SignPublicKey + size: int + + PubkeyEventStat = tuple + pubkey: SignPublicKey + count: int + + IPEventStat = tuple + ip: string + count: int + +router myrouter: + get "/ws": + await request.handleWebsocket() + result[0] = TCActionRaw + + get "/": + var html = "" + compileTemplateFile("index.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") + resp html + + get "/static/logo.png": + resp logo_png + + get "/static/favicon.png": + resp favicon_png + + get "/stats": + if not request.isAdmin(): + responseHeaders.setHeader("WWW-Authenticate", "Basic realm=\"Relay Admin\"") + resp Http401, "Unauthorized" + + let days_back = "-28 days" + let datarange: PeriodRange = block: + let row = relay.db.getRow(sql"""SELECT + strftime('%Y-%W', datetime('now', ?)) AS a, + strftime('%Y-%W') AS b""", days_back).get() + (row[0].s, row[1].s) + + # total users + let total_ips = relay.db.getRow(sql"SELECT coalesce(count(distinct ip), 0) FROM stats_event").get()[0].i + let total_pubkeys = relay.db.getRow(sql"SELECT coalesce(count(distinct pubkey), 0) FROM stats_event").get()[0].i + + # total transfer + let row = relay.db.getRow(sql""" + SELECT + coalesce(sum(data_in), 0) AS din, + coalesce(sum(data_out), 0) AS dout + FROM + stats_transfer + WHERE + period >= ? + """, datarange.a).get() + let total_data_in = row[0].i.int + let total_data_out = row[1].i.int + + # total stored + let total_stored_note = relay.db.getRow(sql"SELECT coalesce(sum(length(data)), 0) FROM note").get()[0].i + let total_stored_message = relay.db.getRow(sql""" + SELECT + COALESCE(SUM(LENGTH(md.data) + LENGTH(md.key)), 0) + FROM message_data md + """).get()[0].i + let total_stored = total_stored_note + total_stored_message + let num_note = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM note").get()[0].i + let num_message = relay.db.getRow(sql"SELECT coalesce(count(*), 0) FROM message_recipient").get()[0].i + + # top traffic by ip + var traffic_by_ip: seq[TransferTotal] + for row in relay.db.getAllRows(sql""" + SELECT + sum(coalesce(data_in, 0)) AS din, + sum(coalesce(data_out, 0)) AS dout, + sum(coalesce(data_in, 0)) + sum(coalesce(data_out, 0)) AS total, + ip + FROM + stats_transfer + WHERE + period >= ? + GROUP BY ip + ORDER BY total DESC + LIMIT 10 + """, datarange.a): + traffic_by_ip.add(( + data_in: row[0].i.int, + data_out: row[1].i.int, + ip: row[3].s, + pubkey: default(SignPublicKey), + period: "", + )) + + # top traffic by pubkey + var traffic_by_pubkey: seq[TransferTotal] + for row in relay.db.getAllRows(sql""" + SELECT + sum(coalesce(data_in, 0)) AS din, + sum(coalesce(data_out, 0)) AS dout, + sum(coalesce(data_in, 0)) + sum(coalesce(data_out, 0)) AS total, + pubkey + FROM + stats_transfer + WHERE + period >= ? + AND pubkey <> '' + GROUP BY pubkey + ORDER BY total DESC + LIMIT 10 + """, datarange.a): + traffic_by_pubkey.add(( + data_in: row[0].i.int, + data_out: row[1].i.int, + ip: "", + pubkey: SignPublicKey.fromDB(row[3].b), + period: "", + )) + + # top storage by pubkey + var storage_by_pubkey: seq[StorageStat] + for row in relay.db.getAllRows(sql""" + SELECT + src, + SUM(total_bytes) AS total_bytes + FROM ( + -- Note storage + SELECT + src, + SUM(LENGTH(data)) AS total_bytes + FROM note + GROUP BY src + + UNION ALL + + -- Message data storage + SELECT + md.src, + SUM(LENGTH(md.data) + LENGTH(md.key)) AS total_bytes + FROM message_data md + GROUP BY md.src + ) + GROUP BY src + ORDER BY 2 DESC + LIMIT 10 + """): + storage_by_pubkey.add(( + pubkey: SignPublicKey.fromDb(row[0].b), + size: row[1].i.int, + )) + + # top events by pubkey + var connects_by_pubkey: seq[PubkeyEventStat] + for row in relay.db.getAllRows(sql""" + SELECT + pubkey, + COALESCE(SUM(connect), 0) + FROM + stats_event + WHERE + period >= ? + AND pubkey <> '' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 10 + """, datarange.a): + connects_by_pubkey.add(( + pubkey: SignPublicKey.fromDb(row[0].b), + count: row[1].i.int, + )) + + var publish_by_pubkey: seq[PubkeyEventStat] + for row in relay.db.getAllRows(sql""" + SELECT + pubkey, + COALESCE(SUM(publish), 0) + FROM + stats_event + WHERE + period >= ? + AND pubkey <> '' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 10 + """, datarange.a): + publish_by_pubkey.add(( + pubkey: SignPublicKey.fromDb(row[0].b), + count: row[1].i.int, + )) + + var send_by_pubkey: seq[PubkeyEventStat] + for row in relay.db.getAllRows(sql""" + SELECT + pubkey, + COALESCE(SUM(send), 0) + FROM + stats_event + WHERE + period >= ? + AND pubkey <> '' + GROUP BY 1 + ORDER BY 2 DESC + LIMIT 10 + """, datarange.a): + send_by_pubkey.add(( + pubkey: SignPublicKey.fromDb(row[0].b), + count: row[1].i.int, + )) + + var html = "" + compileTemplateFile("stats.nimja", baseDir = getScriptDir() / "templates", autoEscape = true, varname = "html") + resp html + +proc main(database: string, port: Port, address = "127.0.0.1") = + var L = newConsoleLogger() + addHandler(L) + info "Database: ", database + var db = open(database, "", "", "") + db.exec(sql"PRAGMA journal_mode=PERSIST") + db.exec(sql"PRAGMA busy_timeout = 5000") + db.exec(sql"PRAGMA synchronous = FULL") + relay = newRelay[NetstringSocket](db) + info &"Serving on {address}:{port.int}" + let settings = newSettings(port=port, bindAddr=address) + var jester = initJester(myrouter, settings=settings) + jester.serve() + +when isMainModule: + import argparse + var p = newParser: + option("-d", "--database", default=some("brelay.sqlite"), help="Database") + command("hashpassword"): + help("Generate a hash for a password given on stdin") + run: + let password = stdin.readAll().strip() + echo crypto_pwhash_str(password) + command("server"): + option("-p", "--port", default=some("9000")) + option("-a", "--address", default=some("127.0.0.1")) + run: + main(opts.parentOpts.database, opts.port.parseInt.Port, opts.address) + + try: + p.run() + except UsageError as e: + stderr.writeLine getCurrentExceptionMsg() + quit(1) + \ No newline at end of file diff --git a/src/static/favicon.png b/src/bucketsrelay/v2/static/favicon.png similarity index 100% rename from src/static/favicon.png rename to src/bucketsrelay/v2/static/favicon.png diff --git a/src/static/logo.png b/src/bucketsrelay/v2/static/logo.png similarity index 100% rename from src/static/logo.png rename to src/bucketsrelay/v2/static/logo.png diff --git a/src/bucketsrelay/v2/templates/index.nimja b/src/bucketsrelay/v2/templates/index.nimja new file mode 100644 index 0000000..3f9ff1f --- /dev/null +++ b/src/bucketsrelay/v2/templates/index.nimja @@ -0,0 +1,60 @@ + + + Buckets Relay + + + + + +
+
+ +
+ +

+ Buckets Relay +

+
+ {{ RELAY_VERSION }} +
+ +

+ If you use Buckets, this relay lets you securely share your budget among your devices. Data is stored until recipients receive it or it expires. All data passing through this relay is encrypted end-to-end. +

+ +

+ Use of this service may be revoked at any time for any reason. Also, for now it's free to use, but that could change depending on the cost to run it. +

+ +

+ The code for this is open source if you'd like to run your own instance. +

+
+ + \ No newline at end of file diff --git a/src/bucketsrelay/v2/templates/stats.nimja b/src/bucketsrelay/v2/templates/stats.nimja new file mode 100644 index 0000000..825def6 --- /dev/null +++ b/src/bucketsrelay/v2/templates/stats.nimja @@ -0,0 +1,177 @@ + + + Buckets Relay Stats + + + + + +

Stats [{{ datarange.a }}, {{ datarange.b }}]

+ +

Users

+ + + + + + + + + +
IPsPubkeys
{{ total_ips.wcommas }}{{ total_pubkeys.wcommas }}
+ +

Transfer

+ + + + + + + + + + + +
Bytes inBytes outTotal
{{ total_data_in.wcommas }}{{ total_data_out.wcommas }}{{ (total_data_in + total_data_out).wcommas }}
+ +

Storage total

+ + + + + + + + + + + + + + + + + + + +
NotesMessagesTotal
Bytes{{ total_stored_note.wcommas }}{{ total_stored_message.wcommas }}{{ total_stored.wcommas }}
Count{{ num_note.wcommas }}{{ num_message.wcommas }}
+ +

Top Traffic by IP

+ + + + + + + + {% for tot in traffic_by_ip %} + + + + + + + {% endfor %} +
IPBytes inBytes outTotal
{{ tot.ip }}{{ tot.data_in.wcommas }}{{ tot.data_out.wcommas }}{{ (tot.data_in + tot.data_out).wcommas }}
+ +

Top Traffic by Pubkey

+ + + + + + + + {% for tot in traffic_by_pubkey %} + + + + + + + {% endfor %} +
PubkeyBytes inBytes outTotal
{{ tot.pubkey.abbr }}{{ tot.data_in.wcommas }}{{ tot.data_out.wcommas }}{{ (tot.data_in + tot.data_out).wcommas }}
+ +

Top Storage by Pubkey

+ + + + + + {% for tot in storage_by_pubkey %} + + + + + {% endfor %} +
PubkeyBytes
{{ tot.pubkey.abbr }}{{ tot.size.wcommas }}
+ +

Top Events by Pubkey

+
+
+ + + + + + {% for st in connects_by_pubkey %} + + + + + {% endfor %} +
PubkeyConnect
{{ st.pubkey.abbr }}{{ st.count.wcommas }}
+
+ +
+ + + + + + {% for st in publish_by_pubkey %} + + + + + {% endfor %} +
PubkeyPublish
{{ st.pubkey.abbr }}{{ st.count.wcommas }}
+
+ +
+ + + + + + {% for st in send_by_pubkey %} + + + + + {% endfor %} +
PubkeySend
{{ st.pubkey.abbr }}{{ st.count.wcommas }}
+
+
+ + \ No newline at end of file diff --git a/src/partials/index.mustache b/src/partials/index.mustache deleted file mode 100644 index ea36c26..0000000 --- a/src/partials/index.mustache +++ /dev/null @@ -1,384 +0,0 @@ - - - Buckets Relay - - - - -
-

- - Buckets Relay -

- -

- If you use Buckets, this relay lets you securely share your budget among your devices. This relay doesn't store any budget info. Instead, think of it like a satellite in the sky that can bounce your data from your computer to your phone. -

- -

- Use of this service may be revoked at any time for any reason. -

- -

- The code for this is Open Source if you'd like to run your own instance. -

- - {{#openregistration}} -
-

Register

-
- - -
- -
- - -
- - -
-
- -
-

Verify email address

-
- - -
- -
- - -
- - -
-
- If you didn't receive a verification code, you can - -
- -
-

Forgot password?

-
- - -
- - -
-
- -
-

Change password

-
- - -
-
- - -
- - -
-
- {{/openregistration}} - -
- {{#openregistration}} - - {{/openregistration}} - - \ No newline at end of file diff --git a/tests/all.sh b/tests/all.sh new file mode 100755 index 0000000..521a554 --- /dev/null +++ b/tests/all.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +RC=0 +FAILED= +PASSED= +for filename in $(ls tests/t*.nim); do + echo $filename + nim r "$filename" + rc1=$? + if [ ! "$rc1" == "0" ]; then + RC="$rc1" + FAILED="$filename $FAILED" + else + PASSED="$filename $PASSED" + fi +done + +for filename in $PASSED; do + echo "PASSED: $filename" +done +for filename in $FAILED; do + echo "FAILED: $filename" +done + +if [ "$RC" == "0" ]; then + echo "OK" +else + echo "exit=$RC" +fi + +exit "$RC" diff --git a/tests/func1.sh b/tests/func1.sh deleted file mode 100755 index c52febe..0000000 --- a/tests/func1.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -waitForOpenPort() { - - PORT=$1 - HOSTTOCHECK="127.0.0.1" - TIMEOUT=5 - echo "Waiting for $HOSTTOCHECK:$PORT to open" - sleep "$TIMEOUT" & - SLEEPPID=$! - while ! nc -z "$HOSTTOCHECK" "$PORT"; do - sleep 0.1 - if ! kill -0 "$SLEEPPID" 2>/dev/null; then - echo "Timed out waiting for $HOSTTOCHECK:$PORT to open" - return 1 - fi - done - kill "$SLEEPPID" 2>/dev/null - echo "Port $HOSTTOCHECK:$PORT is open!" - return 0 -} - -dotest() { - echo "Adding a user ..." - printf 'foobar' | brelay adduser me@me.com --password-stdin - - echo "Generating keys ..." - (mkdir -p client1 && cd client1 && bclient genkeys) - (mkdir -p client2 && cd client2 && bclient genkeys) - echo "hello, world" > testfile - - echo "Starting the server ..." - brelay server --port 8080 & - CHILDPID=$! - trap "kill $CHILDPID" exit - - waitForOpenPort 8080 - - echo "Starting the clients ..." - printf "hello, world" > client2/testfile - (cd client1 && bclient receive -u me@me.com -p foobar http://127.0.0.1:8080/v1/relay "$(cat ../client2/relay.key.public)" > output) & - CLIENT1PID=$! - (cd client2 && cat testfile | bclient send -u me@me.com -p foobar http://127.0.0.1:8080/v1/relay "$(cat ../client1/relay.key.public)") - wait $CLIENT1PID - cat client1/output - - if [ "$(cat client2/testfile)" != "$(cat client1/output)" ]; then - echo "input != output" - exit 1 - fi - - echo "Showing some stats ..." - echo '.timeout 1000' | sqlite3 buckets_relay.sqlite - brelay stats -} - -rm -r _tests -set -xe -mkdir -p _tests -(cd _tests && dotest) diff --git a/tests/tbrelay.nim b/tests/tbrelay.nim deleted file mode 100644 index e934cb2..0000000 --- a/tests/tbrelay.nim +++ /dev/null @@ -1,75 +0,0 @@ -import std/unittest -import std/logging -import ./util - -import chronos - -import brelay -import bclient - -import bucketsrelay/common -import bucketsrelay/proto - -proc tlog(msg: string) = - debug "TEST: " & msg - -when multiusermode: - test "copy": - withinTmpDir: - tlog "Adding users ..." - addverifieduser("data.sqlite", "alice", "alice") - addverifieduser("data.sqlite", "bob", "bob") - let relayurl = "http://127.0.0.1:9001/v1/relay" - tlog "Starting relay ..." - let server = startRelay("data.sqlite", 9001.Port, "127.0.0.1") - tlog "Generating keys ..." - let akeys = genkeys() - let bkeys = genkeys() - tlog "Sending from sender to receiver ..." - let sendres = relaySend("hello", bkeys.pk, - relayurl = relayurl, - mykeys = akeys, - username = "alice", - password = "alice", - ) - tlog "Receiving from sender ..." - let recvres = relayReceive(akeys.pk, - relayurl = relayurl, - mykeys = bkeys, - username = "bob", - password = "bob", - ) - tlog "Waiting for send to resolve" - waitFor sendres - tlog "Waiting for recv to resolve" - let res = waitFor recvres - check res == "hello" - -when singleusermode: - test "copy": - withinTmpDir: - let relayurl = "http://127.0.0.1:9001/v1/relay" - tlog "Starting relay ..." - let server = startRelaySingleUser("alice", "password", 9001.Port, "127.0.0.1") - tlog "Generating keys ..." - let akeys = genkeys() - let bkeys = genkeys() - tlog "Sending from sender to receiver ..." - let sendres = relaySend("hello", bkeys.pk, - relayurl = relayurl, - mykeys = akeys, - username = "alice", - password = "password", - ) - tlog "Receiving from sender ..." - let recvres = relayReceive(akeys.pk, - relayurl = relayurl, - mykeys = bkeys, - username = "alice", - password = "password", - ) - tlog "Waiting for send to resolve" - waitFor sendres - tlog "Waiting for recv to resolve" - let res = waitFor recvres - check res == "hello" diff --git a/tests/tclient.nim b/tests/tclient.nim deleted file mode 100644 index d9929ec..0000000 --- a/tests/tclient.nim +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import std/unittest -import std/strutils - -import ./util - -import bucketsrelay/common -import bucketsrelay/client -import bucketsrelay/server - -type - ClientHandler = ref object - events: seq[RelayEvent] - lifeEvents: seq[ClientLifeEvent] - -proc handleEvent(handler: ClientHandler, ev: RelayEvent, remote: RelayClient) {.async.} = - handler.events.add(ev) - -proc handleLifeEvent(handler: ClientHandler, ev: ClientLifeEvent, remote: RelayClient) {.async.} = - handler.lifeEvents.add(ev) - -proc newClientHandler(): ClientHandler = - new(result) - -proc popEvent(client: ClientHandler, k: EventKind): Future[RelayEvent] {.async, gcsafe.} = - ## Wait for and remove particular event type from the queue - # Since this is just for tests, this does dumb polling - var res: RelayEvent - var delay = 10 - while true: - var idx = -1 - for i,ev in client.events: - if ev.kind == k: - idx = i - res = ev - break - if idx >= 0: - client.events.del(idx) - return res - else: - if delay > 1000: - echo "Waiting for event: " & $k - await sleepAsync(delay.milliseconds) - delay += 100 - -proc popEvent(client: ClientHandler, k: ClientLifeEventKind): Future[ClientLifeEvent] {.async.} = - var delay = 10 - while true: - var idx = -1 - for i,ev in client.lifeEvents: - if ev.kind == k: - idx = i - result = ev - break - if idx >= 0: - client.lifeEvents.del(idx) - return result - else: - if delay > 1000: - echo "Waiting for event: " & $k - await sleepAsync(delay.milliseconds) - delay += 100 - -when multiusermode: - - proc verified_user(rs: RelayServer, email: string, password = ""): int64 = - result = rs.register_user(email, password) - let token = rs.generate_email_verification_token(result) - assert rs.use_email_verification_token(result, token) == true - - test "basic": - withinTmpDir: - var server = newRelayServer(":memory:") - server.start(initTAddress("127.0.0.1", 9001)) - defer: - waitFor server.finish() - let user1 = server.verified_user("alice", "password") - let user2 = server.verified_user("bob", "password") - - var c1h = newClientHandler() - var keys1 = genkeys() - var client1 = newRelayClient(keys1, c1h, "alice", "password") - waitFor client1.connect("ws://127.0.0.1:9001/v1/relay") - discard waitFor c1h.popEvent(ConnectedToServer) - - var c2h = newClientHandler() - var keys2 = genkeys() - var client2 = newRelayClient(keys2, c2h, "bob", "password") - waitFor client2.connect("ws://127.0.0.1:9001/v1/relay") - discard waitFor c2h.popEvent(ConnectedToServer) - - waitFor client1.connect(keys2.pk) - waitFor client2.connect(keys1.pk) - - var atob = (waitFor c1h.popEvent(Connected)).conn_pubkey - var btoa = (waitFor c2h.popEvent(Connected)).conn_pubkey - check atob.string != "" - check btoa.string != "" - - waitFor client1.sendData(atob, "hello") - check (waitFor c2h.popEvent(Data)).data == "hello" - waitFor client2.sendData(btoa, "a".repeat(4096)) - check (waitFor c1h.popEvent(Data)).data == "a".repeat(4096) - - waitFor client1.disconnect(keys2.pk) - waitFor client2.disconnect(keys2.pk) - - check (waitFor c1h.popEvent(Disconnected)).dcon_pubkey == keys2.pk - check (waitFor c2h.popEvent(Disconnected)).dcon_pubkey == keys1.pk - - test "NotConnected": - withinTmpDir: - var server = newRelayServer(":memory:") - server.start(initTAddress("127.0.0.1", 9002)) - let user1 = server.verified_user("alice", "password") - - var ch = newClientHandler() - var keys1 = genkeys() - var client1 = newRelayClient(keys1, ch, "alice", "password") - waitFor client1.connect("ws://127.0.0.1:9002/v1/relay") - echo "Stopping relay server ..." - waitFor server.finish() - echo "Relay server stopped" - for req in allHttpRequests: - # req.stream.writer.tsource.close() - req.stream.reader.tsource.close() - echo "Closed stream" - discard waitFor ch.popEvent(DisconnectedFromServer) - expect RelayNotConnected: - waitFor client1.connect("foobar".PublicKey) - expect RelayNotConnected: - waitFor client1.sendData("foobar".PublicKey, "some data") - - test "server goes down": - withinTmpDir: - var server = newRelayServer(":memory:") - server.start(initTAddress("127.0.0.1", 9002)) - let user1 = server.verified_user("alice", "password") - - var ch = newClientHandler() - var keys1 = genkeys() - var client1 = newRelayClient(keys1, ch, "alice", "password") - waitFor client1.connect("ws://127.0.0.1:9002/v1/relay") - discard waitFor ch.popEvent(ConnectedToServer) - echo "Stopping relay server ..." - waitFor server.finish() - echo "Relay server stopped" - for req in allHttpRequests: - # req.stream.writer.tsource.close() - req.stream.reader.tsource.close() - echo "Closed stream" - discard waitFor ch.popEvent(DisconnectedFromServer) - expect RelayNotConnected: - waitFor client1.connect("foobar".PublicKey) - expect RelayNotConnected: - waitFor client1.sendData("foobar".PublicKey, "some data") - - test "wrong credentials": - var server = newRelayServer(":memory:") - server.start(initTAddress("127.0.0.1", 9003)) - defer: - waitFor server.finish() - let user1 = server.verified_user("alice", "password") - - var ch = newClientHandler() - var keys1 = genkeys() - var client1 = newRelayClient(keys1, ch, "alice", "wrongpassword") - expect RelayErrLoginFailed: - waitFor client1.connect("ws://127.0.0.1:9003/v1/relay") diff --git a/tests/tfunctional.nim b/tests/tfunctional.nim new file mode 100644 index 0000000..d97297a --- /dev/null +++ b/tests/tfunctional.nim @@ -0,0 +1,170 @@ +import std/asyncdispatch +import std/net +import std/options +import std/os +import std/osproc +import std/streams +import std/unittest + +import ./util + +import bucketsrelay/v2/sampleclient +import bucketsrelay/v2/proto2 + +import ws + +const TESTPORT = 12222.Port + +proc startServer(port: Port): Process = + let database = absolutePath(currentSourcePath().parentDir() / "func.sqlite") + if database.fileExists: + echo "removing ", database.relativePath(".") + removeFile(database) + + let bin = absolutePath(currentSourcePath().parentDir() / "bin" / "server2") + bin.parentDir.createDir() + echo "compiling ", bin.relativePath(".") + let compileProcess = startProcess( + "nim", + workingDir = currentSourcePath().parentDir().parentDir(), + args = ["c", "-d:testmode", "-o:" & bin, "src/bucketsrelay/v2/server2.nim"], + options = {poStdErrToStdOut, poUsePath} + ) + let output = compileProcess.outputStream.readAll() + let exitCode = compileProcess.waitForExit() + echo output + if exitCode != 0: + raise newException(OSError, "Compilation failed with exit code " & $exitCode) + echo "compiled ", bin.relativePath(".") + + startProcess(bin, + workingDir = currentSourcePath().parentDir(), + args = [ + "--database", database, + "server", + "--port", $port, + ], options = {poStdErrToStdOut, poUsePath, poParentStreams} + ) + +proc isPortOpen(port: Port): bool = + discard + +proc waitForPort(port: Port) = + while true: + sleep(100) + try: + let socket = newSocket() + socket.connect("127.0.0.1", port) + socket.close() + break + except: + echo "waiting for port ", $port + echo "port open: ", $port + +proc stop(p: Process) = + p.terminate() + +var server = startServer(TESTPORT) +waitForPort(TESTPORT) + +proc serverURL(): string = + "ws://127.0.0.1:" & $TESTPORT & "/ws" + +proc testClient(keys: KeyPair): NetstringClient = + let url = serverURL() + newRelayClient(url, keys) + +proc testClient(): NetstringClient = + testClient(genkeys()) + +suite "auth": + + test "same key auth": + var keys = genkeys() + var alice = testClient(keys) + var alice2 = testClient(keys) + var bob = testClient() + waitFor bob.sendData(@[keys.pk], "this is bob", "") + check (waitFor alice2.getData()) == ("", "this is bob") + +suite "publishnote": + + test "basic": + var alice = testClient() + var bob = testClient() + waitFor alice.publishNote("basic", "data") + check (waitFor bob.fetchNote("basic")) == "data" + var p = bob.fetchNote("basic") + check p.finished == false + waitFor alice.publishNote("basic", "again") + check (waitFor p) == "again" + + test "duplicate": + var alice = testClient() + waitFor alice.publishNote("dupe", "data") + expect(CatchableError): + waitFor alice.publishNote("dupe", "data again") + +suite "data": + + test "basic": + var akeys = genkeys() + var bkeys = genkeys() + var alice = testClient(akeys) + var bob = testClient(bkeys) + waitFor alice.sendData(@[bkeys.pk], "hey, bob?", "") + check (waitFor bob.getData()) == ("", "hey, bob?") + waitFor bob.sendData(@[akeys.pk], "hi, alice!", "hey") + check (waitFor alice.getData()) == ("hey", "hi, alice!") + + test "offline": + var akeys = genkeys() + var bkeys = genkeys() + var alice = testClient(akeys) + waitFor alice.sendData(@[bkeys.pk], "message \x01", "") + waitFor alice.sendData(@[bkeys.pk], "message \x02", "") + waitFor alice.sendData(@[bkeys.pk], "message \x00null", "") + + var bob = testClient(bkeys) + check (waitFor bob.getData()) == ("", "message \x01") + check (waitFor bob.getData()) == ("", "message \x02") + check (waitFor bob.getData()) == ("", "message \x00null") + + +suite "invalid": + + test "malformed": + let ws = waitFor newWebSocket(serverURL()) + waitFor ws.send("garbage") + waitForPort(TESTPORT) + + test "too big": + let ws = waitFor newWebSocket(serverURL()) + waitFor ws.send("123456789:foooooo") + waitForPort(TESTPORT) + + test "RelayMessage": + var keys = genkeys() + let ws = waitFor newWebSocket(serverURL()) + let ns = newNetstringClient(ws) + let who = waitFor ns.receiveMessage() + let answer = who.who_challenge.answer(keys.sk) + waitFor ns.sendCommand(RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: keys.pk)) + let ok = waitFor ns.receiveMessage() + checkpoint $ok + check ok.kind == Okay + + waitFor ws.send(nsencode(serialize(RelayMessage( + kind: Note, + note_topic: "hey", + note_data: "data", + )))) + waitFor sleepAsync(1000) + + var legit = testClient() + waitFor legit.publishNote("something", "here") + check (waitFor legit.fetchNote("something")) == "here" + check server.running() + waitForPort(TESTPORT) + +server.terminate() diff --git a/tests/tlicenses.nim b/tests/tlicenses.nim deleted file mode 100644 index 177edf6..0000000 --- a/tests/tlicenses.nim +++ /dev/null @@ -1,89 +0,0 @@ -import std/strutils -import std/unittest - -import bucketsrelay/licenses - -const PRIVATEKEY1 = """ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAkFVXBWA85bBdFOpdwusXL5hELbGh9u7cg/ZeoV1ToDD02Tw2 -BEetGBUSzXsp3fKPbx89wigTjGAJNHAXVGcdbAbBCve+ARhTJTHIrXZ3lXNxvl0j -KfXNa0VqV79WPeZaRtlvC0e8G9A8sL1wjvZn0nL2DG3gGBLeyAeSYiSCE8ROx5op -oDylJRj5RVTWDtCsQFU4j5h7+Jk2nFCfIsaLyDDKquiycIRcXAt8f32RaMEZn0qh -OXnqjHRAEF8V8hvDVfvVwx4iJXcAdnbDHKIl/aD7ssk2fYqeh0kFRH0zyLmPgFoJ -UYOb+opA1NsWBcZCmaeyLC+RWjJDDXQW9H4NgwIDAQABAoIBACEBkwvkrShtg2vE -CLsJXd0Beh3k8D/y8bSvw4YtPHF2oJeJAGVMKtZGA23AC5v42zozL8FVvtqsH47B -T2R6zCynAsBKVUYU1Pa9gsHARKqFou5AiEkRL++nCSGV3Nf89IodMRqoRekqXqag -O7xFtwpWRdQj0EpRDmc57AzLgn+YWrdhwy/2IklJpkmbXiE/lr2Hmgt1eLPb+F5Q -zJ3JGpLKyFmgQZEuShhSVFJnqnFdJGhpK6DDI9XTEufxoBOhEbgJyJrc3FKqjQ4s -Fro4GGNBOjFzOM8nAVWjAeMTMDh/6DSFDP0DDhbQlCHvKfv78UK7oIDEylGOwSha -ODaTVWECgYEA+6gRIR5M9hy8E4/09ZwWjCgOqOSEEL4dluZWSS8a3nMLLTz/Q18u -disfJVNP/rFPO40eRP5FdNHXtXDVbOFclCm0tERcWzQNFYNi54vT9yGQrMpdqJsd -a5/vX3vztvr0Kw7O3jPwzCOkMne04BGZKW/TJelguEBN6d0wNNYh1dMCgYEAktMS -CLM+tyf/tULVAkapYr6kr/fi5ZyNn7S6YkZQx8soH54JfhMqbBCzRwSeF2NKnj7D -XBzjp4FFQoFCoqNwo0G9nTAoOoY6y3S9lLTZr4LjTthW4Zgo1JpgD6/jIEL/v2mC -zpEpvfUWWKjVCn77QBj9Zxoda9v0DRa40DiAq5ECgYEAvXsJEree2Pw/vDb7COcy -rusGRrJwoa6T1uetdkMKZw2WD8TKqi6DbCQBunflVm6oqr0RWn9dSp0pXosLl4SD -0WcpkUWbiGxDobwgfxjwSzYxmXhxVp8cYsm0UV+h3FdN+xGWPwY6u2nmmr05KjD1 -8pYpFHWJBpIcWAbb4hyMs1MCgYA0tnTODNRiW4jxocnp5Eah/gIQbzXV68vo37De -4ZHU+Toxh8KuseDUJXbH839ytCIxCCWJZ5HQLJgaFWBAFd+1rT+PNJ/syw5Gx2Xd -AsT4v0wunXsryT43fiko2KP5jDRXm2DsGq/a1CgusoayGv7Hd3Fa18RiWfiXzmWR -1AdWEQKBgQCGjvSH/6cVzf5L8qpdhGcZ1jIalc5K0eO5//qvCYv8HysdXB8mUfdh -63oK3QONPrYql3KKLgWjQxaffPEWshm4c02tuJQanSGa2yTQwhSJsk4hg1iY+7lX -ihlOePC/fSmBJCmr9f0n0DNn1MxUL6GuIU7peFGm5Q1TJ0toimWG/w== ------END RSA PRIVATE KEY----- -""".strip() - -const PUBLICKEY1 = """ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAkFVXBWA85bBdFOpdwusX -L5hELbGh9u7cg/ZeoV1ToDD02Tw2BEetGBUSzXsp3fKPbx89wigTjGAJNHAXVGcd -bAbBCve+ARhTJTHIrXZ3lXNxvl0jKfXNa0VqV79WPeZaRtlvC0e8G9A8sL1wjvZn -0nL2DG3gGBLeyAeSYiSCE8ROx5opoDylJRj5RVTWDtCsQFU4j5h7+Jk2nFCfIsaL -yDDKquiycIRcXAt8f32RaMEZn0qhOXnqjHRAEF8V8hvDVfvVwx4iJXcAdnbDHKIl -/aD7ssk2fYqeh0kFRH0zyLmPgFoJUYOb+opA1NsWBcZCmaeyLC+RWjJDDXQW9H4N -gwIDAQAB ------END PUBLIC KEY----- -""".strip() - -const PRIVATEKEY2 = """ ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQDCqQMftQvDX2B2oJl1t7eXRSMhviklJx00olcqI/4okB2WLX18 -3wNUM+O+DZiMAkOlMk96Z6y1Rs03CmV4wJmu4fwrOGFrcS1nsOky8z9KLPENmzxp -0FAL2xwdG6TEhGOlHSRloDQQN58CEjegPYGcLwiysL30fmK69GbVE6f1ZwIDAQAB -AoGBAJCtVzIIuH5z89kXUhdo/V3Dt/HLSP9hC9bj1Y7vg2YYfrTwiHT3t5ysmFbX -+goNYMN2GhYq2fU9cya2ZmaSF2XR9fD5zGINSFltSSOQTUtokhUUx6pVDk06CmjJ -vetu7//nhVp1xP4T2IHXIOuaOB1FxfMlUk8LV+TNsmhsXHgxAkEA/WbYpDp8ukLw -ryhpOaqZiZW06aTe8seLNS2U7cGlTe+VsA9uGwS1HHIvAiOQ9/4f5rjM3XZtNwZD -NrH+2BambwJBAMSn+bNuoFUtwVGzKSaAMGOg/IERQN8uH73iSCSaFnfM7Kwzl4o7 -u96nEYi0B2R7UMa/UwbgpBDplnvZ8QRXXIkCPw/WXbPl8+WwSVqpK+puvynaMXRo -2YZS8mBgeO5jK/GzB6f5TuhhYvBkMovvrR/SwiupYSR2Ql0uBwVkGolm4QJANFHs -YQyho4fU0wOzgwa/2QHPrBcHB1miIEa/ot1L9PuUTAw92Q0jYo1YYOJkxRr51qa4 -VDAX9lfvLWxCb0E+4QJBAJlhxvujrPotY6/rXMVAY6Zt+MmQiUiYNDVm4eEaH6t5 -jMiVvR+d8aAzTRTV1U8jg+LwhM7t0lyN5gIC8NeuHuU= ------END RSA PRIVATE KEY----- -""".strip() - -const PUBLICKEY2 {.used.} = """ ------BEGIN PUBLIC KEY----- -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDCqQMftQvDX2B2oJl1t7eXRSMh -viklJx00olcqI/4okB2WLX183wNUM+O+DZiMAkOlMk96Z6y1Rs03CmV4wJmu4fwr -OGFrcS1nsOky8z9KLPENmzxp0FAL2xwdG6TEhGOlHSRloDQQN58CEjegPYGcLwiy -sL30fmK69GbVE6f1ZwIDAQAB ------END PUBLIC KEY----- -""".strip() - -suite "BucketsV1RSALicense": - - test "works": - let license = createV1License(PRIVATEKEY1, "foo@foo.com") - check $license is string - checkpoint $license - check verify(license, PUBLICKEY1) == true - check extractEmail(license) == "foo@foo.com" - - test "wrong key": - let license = createV1License(PRIVATEKEY2, "bad@foo.com") - check $license is string - checkpoint $license - check verify(license, PUBLICKEY1) == false - check extractEmail(license) == "bad@foo.com" diff --git a/tests/tnetstring.nim b/tests/tnetstring.nim index db57520..8e9136b 100644 --- a/tests/tnetstring.nim +++ b/tests/tnetstring.nim @@ -1,88 +1,120 @@ import std/unittest -import bucketsrelay/netstring +import bucketsrelay/v2/objs -test "nsencode": - check nsencode("apple") == "5:apple," - check nsencode("") == "0:," - check nsencode("banana\x00,") == "8:banana\x00,," +suite "encode": + test "nsencode": + check nsencode("apple") == "5:apple," + check nsencode("") == "0:," + check nsencode("banana\x00,") == "8:banana\x00,," -test "nsencode newline allowed instead of comma": - check nsencode("apple", '\n') == "5:apple\n" - check nsencode("", '\n') == "0:\n" - check nsencode("banana\x00\n", '\n') == "8:banana\x00\n\n" + test "nsencode newline allowed instead of comma": + check nsencode("apple", '\n') == "5:apple\n" + check nsencode("", '\n') == "0:\n" + check nsencode("banana\x00\n", '\n') == "8:banana\x00\n\n" -suite "NetstringDecoder": +suite "decode": - test "netstring in, message out": - var ns = newNetstringDecoder() - ns.consume("5:apple,") - check ns.len == 1 - ns.consume("7:bana") - check ns.len == 1 - ns.consume("na\x00,3:foo,3:bar") - check ns.len == 3 - ns.consume(",") - check ns.len == 4 - check ns.nextMessage() == "apple" - check ns.nextMessage() == "banana\x00" - check ns.nextMessage() == "foo" - check ns.nextMessage() == "bar" + test "basic": + check nsdecode("5:apple,") == "apple" + + test "incomplete": + expect(IncompleteNetstring): + discard nsdecode("7:bana") + expect(IncompleteNetstring): + discard nsdecode("") + expect(IncompleteNetstring): + discard nsdecode("1") + expect(IncompleteNetstring): + discard nsdecode("10:") + expect(IncompleteNetstring): + discard nsdecode("10:1234567890") + + test "2 strings": + var idx = 0 + check nsdecode("5:apple,3:f\x00o,", idx) == "apple" + check nsdecode("5:apple,3:f\x00o,", idx) == "f\x00o" test "newline delimiter": - var ns = newNetstringDecoder('\n') - ns.consume("5:apple\n") - check ns.len == 1 - ns.consume("7:bana") - check ns.len == 1 - ns.consume("na\x00\n3:foo\n3:bar") - check ns.len == 3 - ns.consume("\n") - check ns.len == 4 - check ns.nextMessage() == "apple" - check ns.nextMessage() == "banana\x00" - check ns.nextMessage() == "foo" - check ns.nextMessage() == "bar" + check nsdecode(nsencode("apple", '\n')) == "apple" test "empty string": - var ns = newNetstringDecoder() - ns.consume("0:,") - check ns.nextMessage() == "" + check nsdecode("0:,") == "" test "can't start with 0": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume("01:,") + expect(NetstringError): + discard nsdecode("01:a,") test "can't include non-numerics": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume("1a:,") + expect(NetstringError): + discard nsdecode("1a:,") test ": required": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume("1f,") + expect(NetstringError): + discard nsdecode("1f,") test ", required": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume("1:a2:ab,") + expect(NetstringError): + discard nsdecode("1:a2:ab,") test "len required": - var ns = newNetstringDecoder() - expect(Exception): - ns.consume(":s,") + expect(NetstringError): + discard nsdecode(":s,") test "max message length": - var ns = newNetstringDecoder() - - ns.maxlen = 4 - ns.consume("4:fooa,") - expect(Exception): - ns.consume("5:") - ns.reset() + check nsdecode("4:boom,", maxlen=4) == "boom" + expect(NetstringError): + discard nsdecode("5:apple,", maxlen=4) + expect(NetstringError): + discard nsdecode("200:a", maxlen=100) + +suite "chop": + + test "basic": + var s = "5:apple," + check nschop(s) == "apple" + check s == "" + + test "incomplete": + var s = "" + expect(IncompleteNetstring): + s = "7:bana" + discard nschop(s) + check s == "7:bana" + expect(IncompleteNetstring): + s = "" + discard nschop(s) + check s == "" + expect(IncompleteNetstring): + s = "1" + discard nschop(s) + check s == "1" + expect(IncompleteNetstring): + s = "10:" + discard nschop(s) + check s == "10:" + expect(IncompleteNetstring): + s = "10:1234567890" + discard nschop(s) + check s == "10:1234567890" + + test "2 strings": + var s = "5:apple,3:f\x00o," + check nschop(s) == "apple" + check nschop(s) == "f\x00o" + check s == "" + + test "leftover": + var s = "3:foo,2:ba" + check nschop(s) == "foo" + check s == "2:ba" + + test "newline delimiter": + var s = "5:apple\n" + check nschop(s) == "apple" + check s == "" - ns.maxlen = 10000 - expect(Exception): - ns.consume("100000") + test "empty string": + var s = "0:," + check s.nschop() == "" + check s == "" diff --git a/tests/tproto.nim b/tests/tproto.nim deleted file mode 100644 index f31461b..0000000 --- a/tests/tproto.nim +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) 2022 One Part Rain, LLC. All rights reserved. -# -# This work is licensed under the terms of the MIT license. -# For a copy, see LICENSE.md in this repository. - -import unittest -import os -import options -import tables -import sets -import logging - -import bucketsrelay/proto -import libsodium/sodium -import ./util - -type - KeyPair = tuple - pk: PublicKey - sk: SecretKey - StringClient = ref object - id: int - received: seq[RelayEvent] - pk: PublicKey - sk: SecretKey - -proc newClient(): StringClient = - new(result) - result.received = newSeq[RelayEvent]() - -proc popEvent(client: StringClient): RelayEvent = - doAssert client.received.len > 0, "Expected an event" - result = client.received[0] - client.received.del(0) - -proc popEvent(client: StringClient, kind: EventKind): RelayEvent = - result = client.popEvent() - doAssert result.kind == kind, "Expected " & $kind & " but found " & $result - -proc sendEvent(client: StringClient, ev: RelayEvent) = - client.received.add(ev) - -proc popEvent(conn: RelayConnection[StringClient]): RelayEvent = - conn.sender.popEvent() - -proc popEvent(conn: RelayConnection[StringClient], kind: EventKind): RelayEvent = - conn.sender.popEvent(kind) - -proc mkConnection(relay: var Relay, keys = none[KeyPair](), channel = ""): RelayConnection[StringClient] = - var keys = keys - if keys.isNone: - keys = some(genkeys()) - var client = newClient() - client.pk = keys.get().pk - client.sk = keys.get().sk - var conn = relay.initAuth(client, channel = channel) - let who = client.popEvent() - let signature = sign(client.sk, who.who_challenge) - relay.handleCommand(conn, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: client.pk)) - let ok = client.popEvent() - result = conn - -template sendData*(relay: var Relay, src: RelayConnection, dst: PublicKey, data: string) = - relay.handleCommand(src, RelayCommand(kind: SendData, send_data: data, dest_pubkey: dst)) - -test "basic": - var relay = newRelay[StringClient]() - let (pk, sk) = genkeys() - var aclient = newClient() - aclient.pk = pk - aclient.sk = sk - - checkpoint "who?" - var alice = relay.initAuth(aclient) - let who = alice.popEvent() - check who.kind == Who - check who.who_challenge != "" - - checkpoint "iam" - let signature = sign(sk, who.who_challenge) - relay.handleCommand(alice, RelayCommand(kind: Iam, iam_signature: signature, iam_pubkey: pk)) - let ok = alice.popEvent() - check ok.kind == Authenticated - - checkpoint "connect" - let bob = relay.mkConnection() - check bob.pubkey != alice.pubkey - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - block: - let ev = bob.popEvent() - check ev.kind == Connected - check ev.conn_pubkey == alice.pubkey - - block: - let ev = alice.popEvent() - check ev.kind == Connected - check ev.conn_pubkey == bob.pubkey - - checkpoint "data" - relay.handleCommand(bob, RelayCommand(kind: SendData, send_data: "hello, alice!", dest_pubkey: alice.pubkey)) - let adata = alice.popEvent() - check adata.kind == Data - check adata.data == "hello, alice!" - check adata.sender_pubkey == bob.pubkey - - relay.handleCommand(alice, RelayCommand(kind: SendData, send_data: "hello, bob!", dest_pubkey: bob.pubkey)) - let bdata = bob.popEvent() - check bdata.kind == Data - check bdata.data == "hello, bob!" - check bdata.sender_pubkey == alice.pubkey - -test "multiple conns to same pubkey": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - check bob.sender.received.len == 0 - check alice.sender.received.len == 0 - -test "no crosstalk": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - var cathy = relay.mkConnection() - var dave = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - check cathy.sender.received.len == 0 - check dave.sender.received.len == 0 - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: dave.pubkey)) - relay.handleCommand(dave, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard dave.popEvent(Connected) - relay.sendData(alice, bob.pubkey, "hi, bob") - check bob.popEvent(Data).data == "hi, bob" - check cathy.sender.received.len == 0 - check dave.sender.received.len == 0 - -test "disconnect multiple times": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - relay.removeConnection(alice) - relay.removeConnection(alice) - -test "disconnect, remove from remote client.connections": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - relay.removeConnection(alice) - let edcon = bob.popEvent(Disconnected) - check edcon.dcon_pubkey == alice.pubkey - let bobclient = relay.testmode_conns()[bob.pubkey] - check bobclient.testmode_conns.len == 0 - -test "send data to invalid id": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - relay.sendData(alice, "goober".PublicKey, "testing?") - discard alice.popEvent(ErrorEvent) - relay.sendData(alice, alice.pubkey, "feedback") - discard alice.popEvent(ErrorEvent) - -test "send data to unconnected id": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.sendData(alice, bob.pubkey, "hello") - discard alice.popEvent(ErrorEvent) - check bob.sender.received.len == 0 - -test "connect to self": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(ErrorEvent) - -test "not authenticated": - var relay = newRelay[StringClient]() - let (pk, sk) = genkeys() - let aclient = newClient() - - checkpoint "who?" - var alice = relay.initAuth(aclient) - discard alice.popEvent(Who) - - let bob = relay.mkConnection() - - checkpoint "connect" - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - discard alice.popEvent(ErrorEvent) - check bob.sender.received.len == 0 - - checkpoint "send" - relay.sendData(alice, bob.pubkey, "something") - discard alice.popEvent(ErrorEvent) - check bob.sender.received.len == 0 - -test "disconnect command": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - - relay.handleCommand(alice, RelayCommand(kind: Disconnect, dcon_pubkey: bob.pubkey)) - check bob.popEvent(Disconnected).dcon_pubkey == alice.pubkey - check alice.popEvent(Disconnected).dcon_pubkey == bob.pubkey - -test "remember connection requests": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - - relay.handleCommand(alice, RelayCommand(kind: Disconnect, dcon_pubkey: bob.pubkey)) - check bob.popEvent(Disconnected).dcon_pubkey == alice.pubkey - check alice.popEvent(Disconnected).dcon_pubkey == bob.pubkey - - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - -test "forget connection requests on disconnect": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection() - var bob = relay.mkConnection() - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.handleCommand(bob, RelayCommand(kind: Connect, conn_pubkey: alice.pubkey)) - discard alice.popEvent(Connected) - discard bob.popEvent(Connected) - - relay.handleCommand(alice, RelayCommand(kind: Disconnect, dcon_pubkey: bob.pubkey)) - check bob.popEvent(Disconnected).dcon_pubkey == alice.pubkey - check alice.popEvent(Disconnected).dcon_pubkey == bob.pubkey - - relay.handleCommand(bob, RelayCommand(kind: Disconnect, dcon_pubkey: alice.pubkey)) - relay.handleCommand(alice, RelayCommand(kind: Connect, conn_pubkey: bob.pubkey)) - relay.sendData(alice, bob.pubkey, "something") - discard alice.popEvent(ErrorEvent) - check bob.sender.received.len == 0 - -test "pub/sub": - var relay = newRelay[StringClient]() - var alice = relay.mkConnection(channel = "alicenbob") - var bob = relay.mkConnection(channel = "alicenbob") - block: - let ev = alice.popEvent(Entered) - check ev.entered_pubkey == bob.pubkey - block: - let ev = bob.popEvent(Entered) - check ev.entered_pubkey == alice.pubkey - relay.removeConnection(alice) - block: - let ev = bob.popEvent(Exited) - check ev.exited_pubkey == alice.pubkey diff --git a/tests/tproto2.nim b/tests/tproto2.nim new file mode 100644 index 0000000..4fca179 --- /dev/null +++ b/tests/tproto2.nim @@ -0,0 +1,918 @@ +import std/deques +import std/logging +import std/options +import std/os +import std/strutils +import std/unittest + +import lowdb/sqlite +import bucketsrelay/v2/proto2 + +if getEnv("SHOW_LOGS") != "": + var L = newConsoleLogger() + addHandler(L) +else: + echo "set SHOW_LOGS=something to see logs" + + +#--------------------------------- +# In-memory TestClient +#--------------------------------- +type + TestClient* = ref object + received: Deque[RelayMessage] + pk: SignPublicKey + sk: SignSecretKey + +proc `$`*(tc: TestClient): string = $tc[] + +proc newTestClient*(): TestClient = + new(result) + result.received = initDeque[RelayMessage]() + +proc newTestClient*(keys: KeyPair): TestClient = + new(result) + result = newTestClient() + result.pk = keys.pk + result.sk = keys.sk + +proc sendMessage*(conn: RelayConnection[TestClient], msg: RelayMessage) = + when LOG_COMMS: + info "[" & conn.pubkey.abbr & "] <- " & $msg + conn.sender.received.addLast(msg) + +proc pop*(c: var TestClient): RelayMessage = + c.received.popFirst() + +#--------------------------------- +# Test utilities +#--------------------------------- +proc testRelay(): Relay[TestClient] = + newRelay[TestClient](open(":memory:", "", "", "")) + +proc pop(conn: var RelayConnection[TestClient]): RelayMessage = + conn.sender.pop() + +proc pop(conn: var RelayConnection[TestClient], expected: MessageKind): RelayMessage = + try: + result = conn.sender.pop() + except IndexDefect: + raise IndexDefect.newException("No message found while expecting kind: " & $expected) + doAssert result.kind == expected, "Expected " & $expected & " but got " & $result + +proc msgCount(conn: var RelayConnection[TestClient]): int = + conn.sender.received.len + +proc pk(conn: var RelayConnection[TestClient]): SignPublicKey = conn.sender.pk +proc sk(conn: var RelayConnection[TestClient]): SignSecretKey = conn.sender.sk +proc keys(conn: var RelayConnection[TestClient]): KeyPair = (conn.sender.pk, conn.sender.sk) + +proc anonConn(relay: Relay): RelayConnection[TestClient] = + let client = newTestClient(genkeys()) + var conn = relay.initAuth(client) + return conn + +proc authenticatedConn(relay: Relay, keys: KeyPair): RelayConnection[TestClient] = + let client = newTestClient(keys) + var conn = relay.initAuth(client) + let who = conn.pop() + doAssert who.kind == Who + let answer = who.who_challenge.answer(client.sk) + relay.handleCommand(conn, RelayCommand( + kind: Iam, + resp_id: 1, + iam_answer: answer, + iam_pubkey: client.pk + )) + let ok = conn.pop() + doAssert ok.kind == Okay + doAssert ok.ok_cmd == Iam + doAssert ok.resp_id == 1 + return conn + +proc authenticatedConn(relay: Relay): RelayConnection[TestClient] = + relay.authenticatedConn(genkeys()) + +#--------------------------------- +# End of TestClient +#--------------------------------- + +suite "auth": + test "basic": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + checkpoint "who?" + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + check who.who_challenge != default(Challenge) + checkpoint $who + + checkpoint "iam" + let answer = who.who_challenge.answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + discard alice.pop(Okay) + + test "iam twice": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + checkpoint "who?" + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + check who.who_challenge != default(Challenge) + + checkpoint "iam" + let answer = who.who_challenge.answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + discard alice.pop(Okay) + + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + check alice.pop().kind == Error + + test "iam invalid answer": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + let answer = generateChallenge().answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + + test "iam invalid signature": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + var bobkeys = genkeys() + let answer = generateChallenge().answer(bobkeys.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + + test "invalid opslimit": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + let answer = generateChallenge(opslimit = who.who_challenge.opslimit - 1).answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + + test "invalid memlimit": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + let answer = generateChallenge(memlimit = who.who_challenge.memlimit - 32).answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + + test "invalid bits": + let relay = testRelay() + let aclient = newTestClient(genkeys()) + + var alice = relay.initAuth(aclient) + let who = alice.pop(Who) + + let answer = generateChallenge(bits = who.who_challenge.bits - 1).answer(aclient.sk) + relay.handleCommand(alice, RelayCommand(kind: Iam, iam_answer: answer, iam_pubkey: aclient.pk)) + let err = alice.pop(Error) + check err.err_cmd == Iam + +suite "notes": + + test "basic": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "somedata", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + let data = bob.pop(Note) + check data.note_data == "somedata" + check data.note_topic == "sometopic" + + test "same topic": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "somedata", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "new data", + )) + let err = alice.pop(Error) + check err.err_cmd == PublishNote + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + let data = bob.pop(Note) + check data.note_data == "somedata" + check data.note_topic == "sometopic" + + test "fetch first": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + resp_id: 34, + fetch_topic: "heyo", + )) + let err = alice.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotFound + check err.resp_id == 34 + + test "publish max size topic": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "h".repeat(RELAY_MAX_TOPIC_SIZE + 1), + pub_data: "foo", + )) + block: + let err = alice.pop(Error) + check err.err_code == TooLarge + check err.err_cmd == PublishNote + + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "a".repeat(RELAY_MAX_TOPIC_SIZE + 1), + )) + block: + let err = alice.pop(Error) + check err.err_code == TooLarge + check err.err_cmd == FetchNote + + test "publish max size data": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "topic", + pub_data: "a".repeat(RELAY_MAX_NOTE_SIZE + 1), + )) + let err = alice.pop(Error) + check err.err_code == TooLarge + check err.err_cmd == PublishNote + + when not defined(release): + test "expiration": + let relay = testRelay() + var alice = relay.authenticatedConn() + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "topic", + pub_data: "a", + )) + check alice.pop(Okay).ok_cmd == PublishNote + + skewTime(RELAY_NOTE_DURATION) + skewTime(1) + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "topic", + )) + let err = alice.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotFound + + test "fetch note again": + let relay = testRelay() + var alice = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "sometopic", + pub_data: "somedata", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + let data = alice.pop(Note) + check data.note_data == "somedata" + + relay.handleCommand(alice, RelayCommand( + kind: FetchNote, + fetch_topic: "sometopic", + )) + let err = alice.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotFound + + test "sub then disconnect, the pub": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "foo", + )) + let err = bob.pop(Error) + check err.err_cmd == FetchNote + check err.err_code == NotFound + relay.disconnect(bob) + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "foo", + pub_data: "bar", + )) + + var bob2 = relay.authenticatedConn(bob.keys) + check bob2.msgCount == 0 + relay.handleCommand(bob2, RelayCommand( + kind: FetchNote, + fetch_topic: "foo" + )) + let data = bob2.pop(Note) + check data.note_data == "bar" + check data.note_topic == "foo" + + test "topic null byte": + let relay = testRelay() + var alice = relay.authenticatedConn() + var bob = relay.authenticatedConn() + + relay.handleCommand(alice, RelayCommand( + kind: PublishNote, + pub_topic: "a\x00b", + pub_data: "c\x00d", + )) + let ok = alice.pop(Okay) + check ok.ok_cmd == PublishNote + + relay.handleCommand(bob, RelayCommand( + kind: FetchNote, + fetch_topic: "a\x00b", + )) + let data = bob.pop(Note) + check data.note_data == "c\x00d" + check data.note_topic == "a\x00b" + + test "max notes": + var relay = testRelay() + var alice = relay.authenticatedConn() + for i in 0..