Replace rust-sdk bindings with ones that work (#236)

* First pass of updating to new rust-sdk bindings

* Fix index exports

* Remove conflicting function

* Update for `OlmMachine#sign` support

* Emit decrypted to_device events

* Clean up linter complaints

* Update tests for new bindings

* Appease the linter

* Update for new mock http lib

* Remove unused crypto models

* Fix room tracker test

* Use now-mostly-available attachment functions

* Remove old bindings dependency

* Use the OlmMachine sequencing correctly

* Add required locks to rust handling

* Wire up appservices to new crypto

* Fix tests for new event emitter flow

* Add dependency

* Fix lint

* Update rust-sdk

examples/encryption_appservice.ts

@@ -36,7 +36,7 @@ const worksImage = fs.readFileSync("./examples/static/it-works.png");
 const registration: IAppserviceRegistration = {
     "as_token": creds?.['asToken'] ?? "change_me",
     "hs_token": creds?.['hsToken'] ?? "change_me",
-    "sender_localpart": "crypto_test_appservice_rust3",
+    "sender_localpart": "crypto_main_bot_user",
     "namespaces": {
         users: [{
             regex: "@crypto.*:localhost",
@@ -65,8 +65,8 @@ const options: IAppserviceOptions = {
 };
 
 const appservice = new Appservice(options);
-// const bot = appservice.botIntent;
-const bot = appservice.getIntentForUserId("@crypto_nondefault_test3:localhost");
+const bot = appservice.botIntent;
+// const bot = appservice.getIntentForUserId("@crypto_bot1:localhost");
 
 (async function() {
     await bot.enableEncryption();

package.json

@@ -51,9 +51,10 @@
     "tsconfig.json"
   ],
   "dependencies": {
-    "@turt2live/matrix-sdk-crypto-nodejs": "^0.1.0-beta.10",
+    "@matrix-org/matrix-sdk-crypto-nodejs": "^0.1.0-beta.1",
     "@types/express": "^4.17.13",
     "another-json": "^0.2.0",
+    "async-lock": "^1.3.2",
     "chalk": "^4",
     "express": "^4.18.1",
     "glob-to-regexp": "^0.4.1",

src/MatrixClient.ts

@@ -26,9 +26,6 @@ import { Space, SpaceCreateOptions } from "./models/Spaces";
 import { PowerLevelAction } from "./models/PowerLevelAction";
 import { CryptoClient } from "./e2ee/CryptoClient";
 import {
-    DeviceKeyAlgorithm,
-    DeviceKeyLabel,
-    EncryptionAlgorithm,
     FallbackKey,
     IToDeviceMessage,
     MultiUserDeviceListResponse,
@@ -133,6 +130,14 @@ export class MatrixClient extends EventEmitter {
                 throw new Error("Cannot support custom encryption stores: Use a RustSdkCryptoStorageProvider");
             }
             this.crypto = new CryptoClient(this);
+            this.on("room.event", (roomId, event) => {
+                // noinspection JSIgnoredPromiseFromCall
+                this.crypto.onRoomEvent(roomId, event);
+            });
+            this.on("room.join", (roomId) => {
+                // noinspection JSIgnoredPromiseFromCall
+                this.crypto.onRoomJoin(roomId);
+            });
             LogService.debug("MatrixClientLite", "End-to-end encryption client created");
         } else {
             // LogService.trace("MatrixClientLite", "Not setting up encryption");
@@ -1762,27 +1767,6 @@ export class MatrixClient extends EventEmitter {
         return new Space(roomId, this);
     }
 
-    /**
-     * Uploads new identity keys for the current device.
-     * @param {EncryptionAlgorithm[]} algorithms The supported algorithms.
-     * @param {Record<DeviceKeyLabel<DeviceKeyAlgorithm, string>, string>} keys The keys for the device.
-     * @returns {Promise<OTKCounts>} Resolves to the current One Time Key counts when complete.
-     */
-    @timedMatrixClientFunctionCall()
-    @requiresCrypto()
-    public async uploadDeviceKeys(algorithms: EncryptionAlgorithm[], keys: Record<DeviceKeyLabel<DeviceKeyAlgorithm, string>, string>): Promise<OTKCounts> {
-        const obj = {
-            user_id: await this.getUserId(),
-            device_id: this.crypto.clientDeviceId,
-            algorithms: algorithms,
-            keys: keys,
-        };
-        obj['signatures'] = await this.crypto.sign(obj);
-        return this.doRequest("POST", "/_matrix/client/v3/keys/upload", null, {
-            device_keys: obj,
-        }).then(r => r['one_time_key_counts']);
-    }
-
     /**
      * Uploads One Time Keys for the current device.
      * @param {OTKs} keys The keys to upload.

src/appservice/Appservice.ts

@@ -612,11 +612,13 @@ export class Appservice extends EventEmitter {
         if (!event["content"]) return;
 
         // Update the target intent's joined rooms (fixes transition errors with the cache, like join->kick->join)
-        await this.getIntentForUserId(event['state_key']).refreshJoinedRooms();
+        const intent = this.getIntentForUserId(event['state_key']);
+        await intent.refreshJoinedRooms();
 
         const targetMembership = event["content"]["membership"];
         if (targetMembership === "join") {
             this.emit("room.join", event["room_id"], event);
+            await intent.underlyingClient.crypto?.onRoomJoin(event["room_id"]);
         } else if (targetMembership === "ban" || targetMembership === "leave") {
             this.emit("room.leave", event["room_id"], event);
         } else if (targetMembership === "invite") {
@@ -729,6 +731,9 @@ export class Appservice extends EventEmitter {
                     removed: [],
                 };
 
+                if (!deviceLists.changed) deviceLists.changed = [];
+                if (!deviceLists.removed) deviceLists.removed = [];
+
                 const otks = req.body["org.matrix.msc3202.device_one_time_key_counts"];
                 if (otks) {
                     for (const userId of Object.keys(otks)) {

src/appservice/Intent.ts

@@ -170,6 +170,12 @@ export class Intent {
 
                     // Now set up crypto
                     await this.client.crypto.prepare(await this.client.getJoinedRooms());
+
+                    this.appservice.on("room.event", (roomId, event) => {
+                        if (!this.knownJoinedRooms.includes(roomId)) return;
+                        this.client.crypto.onRoomEvent(roomId, event);
+                    });
+
                     resolve();
                 } catch (e) {
                     reject(e);

src/e2ee/CryptoClient.ts

@@ -1,13 +1,16 @@
 import {
-    decryptFile as rustDecryptFile,
-    encryptFile as rustEncryptFile,
+    DeviceId,
     OlmMachine,
-} from "@turt2live/matrix-sdk-crypto-nodejs";
+    UserId,
+    DeviceLists,
+    RoomId,
+    Attachment,
+    EncryptedAttachment,
+} from "@matrix-org/matrix-sdk-crypto-nodejs";
 
 import { MatrixClient } from "../MatrixClient";
 import { LogService } from "../logging/LogService";
 import {
-    DeviceKeyAlgorithm,
     IMegolmEncrypted,
     IOlmEncrypted,
     IToDeviceMessage,
@@ -21,8 +24,8 @@ import { EncryptedRoomEvent } from "../models/events/EncryptedRoomEvent";
 import { RoomEvent } from "../models/events/RoomEvent";
 import { EncryptedFile } from "../models/events/MessageEvent";
 import { RustSdkCryptoStorageProvider } from "../storage/RustSdkCryptoStorageProvider";
-import { SdkOlmEngine } from "./SdkOlmEngine";
-import { InternalOlmMachineFactory } from "./InternalOlmMachineFactory";
+import { RustEngine, SYNC_LOCK_NAME } from "./RustEngine";
+import { MembershipEvent } from "../models/events/MembershipEvent";
 
 /**
  * Manages encryption for a MatrixClient. Get an instance from a MatrixClient directly
@@ -35,7 +38,7 @@ export class CryptoClient {
     private deviceEd25519: string;
     private deviceCurve25519: string;
     private roomTracker: RoomTracker;
-    private machine: OlmMachine;
+    private engine: RustEngine;
 
     public constructor(private client: MatrixClient) {
         this.roomTracker = new RoomTracker(this.client);
@@ -83,16 +86,45 @@ export class CryptoClient {
 
         LogService.debug("CryptoClient", "Starting with device ID:", this.deviceId);
 
-        this.machine = new InternalOlmMachineFactory(await this.client.getUserId(), this.deviceId, new SdkOlmEngine(this.client), this.storage.storagePath).build();
-        await this.machine.runEngine();
+        const machine = await OlmMachine.initialize(new UserId(await this.client.getUserId()), new DeviceId(this.deviceId), this.storage.storagePath);
+        this.engine = new RustEngine(machine, this.client);
+        await this.engine.run();
 
-        const identity = this.machine.identityKeys;
-        this.deviceCurve25519 = identity[DeviceKeyAlgorithm.Curve25519];
-        this.deviceEd25519 = identity[DeviceKeyAlgorithm.Ed25519];
+        const identity = this.engine.machine.identityKeys;
+        this.deviceCurve25519 = identity.curve25519.toBase64();
+        this.deviceEd25519 = identity.ed25519.toBase64();
 
         this.ready = true;
     }
 
+    /**
+     * Handles a room event.
+     * @internal
+     * @param roomId The room ID.
+     * @param event The event.
+     */
+    public async onRoomEvent(roomId: string, event: any) {
+        await this.roomTracker.onRoomEvent(roomId, event);
+        if (typeof event['state_key'] !== 'string') return;
+        if (event['type'] === 'm.room.member') {
+            const membership = new MembershipEvent(event);
+            if (membership.effectiveMembership !== 'join' && membership.effectiveMembership !== 'invite') return;
+            await this.engine.addTrackedUsers([membership.membershipFor]);
+        } else if (event['type'] === 'm.room.encryption') {
+            const members = await this.client.getRoomMembers(roomId, null, ['join', 'invite']);
+            await this.engine.addTrackedUsers(members.map(e => e.membershipFor));
+        }
+    }
+
+    /**
+     * Handles a room join.
+     * @internal
+     * @param roomId The room ID.
+     */
+    public async onRoomJoin(roomId: string) {
+        await this.roomTracker.onRoomJoin(roomId);
+    }
+
     /**
      * Checks if a room is encrypted.
      * @param {string} roomId The room ID to check.
@@ -121,10 +153,22 @@ export class CryptoClient {
         changedDeviceLists: string[],
         leftDeviceLists: string[],
     ): Promise<void> {
-        await this.machine.pushSync(toDeviceMessages, {
-            changed: changedDeviceLists,
-            left: leftDeviceLists,
-        }, otkCounts, unusedFallbackKeyAlgs);
+        const deviceMessages = JSON.stringify({ events: toDeviceMessages });
+        const deviceLists = new DeviceLists(
+            changedDeviceLists.map(u => new UserId(u)),
+            leftDeviceLists.map(u => new UserId(u)));
+
+        await this.engine.lock.acquire(SYNC_LOCK_NAME, async () => {
+            const syncResp = await this.engine.machine.receiveSyncChanges(deviceMessages, deviceLists, otkCounts, unusedFallbackKeyAlgs);
+            const decryptedToDeviceMessages = JSON.parse(syncResp);
+            if (Array.isArray(decryptedToDeviceMessages?.events)) {
+                for (const msg of decryptedToDeviceMessages.events) {
+                    this.client.emit("to_device.decrypted", msg);
+                }
+            }
+
+            await this.engine.run();
+        });
     }
 
     /**
@@ -140,7 +184,16 @@ export class CryptoClient {
         delete obj['signatures'];
         delete obj['unsigned'];
 
-        const sig = await this.machine.sign(obj);
+        const container = await this.engine.machine.sign(JSON.stringify(obj));
+        const userSignature = container.get(new UserId(await this.client.getUserId()));
+        const sig: Signatures = {
+            [await this.client.getUserId()]: {},
+        };
+        for (const [key, maybeSignature] of Object.entries(userSignature)) {
+            if (maybeSignature.isValid) {
+                sig[await this.client.getUserId()][key] = maybeSignature.signature.toBase64();
+            }
+        }
         return {
             ...sig,
             ...existingSignatures,
@@ -164,7 +217,10 @@ export class CryptoClient {
             throw new Error("Room is not encrypted");
         }
 
-        const encrypted = await this.machine.encryptRoomEvent(roomId, eventType, content);
+        await this.engine.prepareEncrypt(roomId, await this.roomTracker.getRoomCryptoConfig(roomId));
+
+        const encrypted = JSON.parse(await this.engine.machine.encryptRoomEvent(new RoomId(roomId), eventType, JSON.stringify(content)));
+        await this.engine.run();
         return encrypted as IMegolmEncrypted;
     }
 
@@ -177,12 +233,13 @@ export class CryptoClient {
      */
     @requiresReady()
     public async decryptRoomEvent(event: EncryptedRoomEvent, roomId: string): Promise<RoomEvent<unknown>> {
-        const decrypted = await this.machine.decryptRoomEvent(roomId, event.raw);
+        const decrypted = await this.engine.machine.decryptRoomEvent(JSON.stringify(event.raw), new RoomId(roomId));
+        const clearEvent = JSON.parse(decrypted.event);
 
         return new RoomEvent<unknown>({
             ...event.raw,
-            type: decrypted.clearEvent.type || "io.t2bot.unknown",
-            content: (typeof (decrypted.clearEvent.content) === 'object') ? decrypted.clearEvent.content : {},
+            type: clearEvent.type || "io.t2bot.unknown",
+            content: (typeof (clearEvent.content) === 'object') ? clearEvent.content : {},
         });
     }
 
@@ -195,15 +252,11 @@ export class CryptoClient {
      */
     @requiresReady()
     public async encryptMedia(file: Buffer): Promise<{ buffer: Buffer, file: Omit<EncryptedFile, "url"> }> {
-        const encrypted = rustEncryptFile(file);
+        const encrypted = Attachment.encrypt(file);
+        const info = JSON.parse(encrypted.mediaEncryptionInfo);
         return {
-            buffer: encrypted.data,
-            file: {
-                iv: encrypted.file.iv,
-                key: encrypted.file.web_key,
-                v: encrypted.file.v,
-                hashes: encrypted.file.hashes as { sha256: string },
-            },
+            buffer: Buffer.from(encrypted.encryptedData),
+            file: info,
         };
     }
 
@@ -214,9 +267,12 @@ export class CryptoClient {
      */
     @requiresReady()
     public async decryptMedia(file: EncryptedFile): Promise<Buffer> {
-        return rustDecryptFile((await this.client.downloadContent(file.url)).data, {
-            ...file,
-            web_key: file.key as any, // we know it is compatible
-        });
+        const contents = (await this.client.downloadContent(file.url)).data;
+        const encrypted = new EncryptedAttachment(
+            contents,
+            JSON.stringify(file),
+        );
+        const decrypted = Attachment.decrypt(encrypted);
+        return Buffer.from(decrypted);
     }
 }

src/e2ee/ICryptoRoomInformation.ts

@@ -0,0 +1,9 @@
+import { EncryptionEventContent } from "../models/events/EncryptionEvent";
+
+/**
+ * Information about a room for the purposes of crypto.
+ * @category Encryption
+ */
+export interface ICryptoRoomInformation extends Partial<EncryptionEventContent> {
+    historyVisibility?: string;
+}