- Upgraded to rust crypto but old libolm data will not migrate

- Change how matrix-js-sdk is loaded with it now being an ESM module

package.json

@@ -3,18 +3,18 @@
   "version": "0.8.0",
   "description": "Matrix chat server client for Node-RED",
   "dependencies": {
+    "@matrix-org/matrix-sdk-crypto-nodejs": "^0.2.0-beta.1",
     "abort-controller": "^3.0.0",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^11.1.0",
     "got": "^12.0.2",
     "image-size": "^1.0.2",
     "isomorphic-webcrypto": "^2.3.8",
-    "matrix-js-sdk": "^28.0.0",
+    "matrix-js-sdk": "^34.5.0",
     "mime": "^3.0.0",
     "node-fetch": "^3.3.0",
     "node-localstorage": "^2.2.1",
     "olm": "https://gitlab.matrix.org/matrix-org/olm/-/package_files/2572/download",
-    "request": "^2.88.2",
     "sharp": "^0.33.4",
     "tmp": "^0.2.1",
     "utf8": "^3.0.0"

src/matrix-event-relations.js

@@ -1,5 +1,3 @@
-const {RelationType, EventType, Direction} = require("matrix-js-sdk");
-
 module.exports = function(RED) {
     function MatrixFetchRelations(n) {
         RED.nodes.createNode(this, n);
@@ -43,6 +41,8 @@ module.exports = function(RED) {
         });
 
         node.on("input", async function(msg) {
+            const {Direction} = await import("matrix-js-sdk");
+
             if (!node.server || !node.server.matrixClient) {
                 node.error("No matrix server selected", msg);
                 return;

src/matrix-mark-read.js

@@ -1,5 +1,3 @@
-const {TimelineWindow, RelationType, Filter} = require("matrix-js-sdk");
-const crypto = require('crypto');
 module.exports = function(RED) {
     function MatrixReceiveMessage(n) {
         RED.nodes.createNode(this, n);

src/matrix-paginate-room.js

@@ -1,5 +1,3 @@
-const {TimelineWindow, RelationType, Filter} = require("matrix-js-sdk");
-const crypto = require('crypto');
 module.exports = function(RED) {
     function MatrixReceiveMessage(n) {
         RED.nodes.createNode(this, n);
@@ -34,6 +32,9 @@ module.exports = function(RED) {
         });
 
         node.on("input", async function (msg) {
+            const {TimelineWindow, RelationType, Filter} = await import("matrix-js-sdk");
+            const crypto = await import('crypto');
+
             if (! node.server || ! node.server.matrixClient) {
                 node.error("No matrix server selected", msg);
                 return;

src/matrix-receive.js

@@ -1,4 +1,3 @@
-const {RelationType} = require("matrix-js-sdk");
 module.exports = function(RED) {
     function MatrixReceiveMessage(n) {
         RED.nodes.createNode(this, n);

src/matrix-send-message.js

@@ -1,5 +1,3 @@
-const {RelationType} = require("matrix-js-sdk");
-
 module.exports = function(RED) {
     function MatrixSendImage(n) {
         RED.nodes.createNode(this, n);
@@ -68,7 +66,8 @@ module.exports = function(RED) {
             node.status({ fill: "green", shape: "ring", text: "connected" });
         });
 
-        node.on("input", function (msg) {
+        node.on("input", async function (msg) {
+            const {RelationType} = await import("matrix-js-sdk");
             function getToValue(msg, type, property) {
                 let value = property;
                 if (type === "msg") {

src/matrix-server-config.js

@@ -1,14 +1,10 @@
-const {RelationType, TimelineWindow} = require("matrix-js-sdk");
-
 global.Olm = require('olm');
 const fs = require("fs-extra");
-const sdk = require("matrix-js-sdk");
 const { resolve } = require('path');
 const { LocalStorage } = require('node-localstorage');
-const { LocalStorageCryptoStore } = require('matrix-js-sdk/lib/crypto/store/localStorage-crypto-store');
-const {RoomEvent, RoomMemberEvent, HttpApiEvent, ClientEvent, MemoryStore} = require("matrix-js-sdk");
-const request = require("request");
+globalThis.crypto = require('crypto');
 require("abort-controller/polyfill"); // polyfill abort-controller if we don't have it
+
 if (!globalThis.fetch) {
     // polyfill fetch if we don't have it
     if (!globalThis.fetch) {
@@ -19,16 +15,22 @@ if (!globalThis.fetch) {
 }
 
 module.exports = function(RED) {
+    // Prepare dynamic imports
+    const sdkPromise = import("matrix-js-sdk");
+    const LocalStorageCryptoStorePromise = import('matrix-js-sdk/lib/crypto/store/localStorage-crypto-store.js');
+    const loggerPromise = import('matrix-js-sdk/lib/logger.js');
+
     // disable logging if set to "off"
-    let loggingSettings = RED.settings.get('logging');
-    if(
-        typeof loggingSettings.console !== 'undefined' &&
-        typeof loggingSettings.console.level !== 'undefined' &&
-        ['info','debug','trace'].indexOf(loggingSettings.console.level.toLowerCase()) >= 0
-    ) {
-        const { logger } = require('matrix-js-sdk/lib/logger');
-        logger.disableAll();
-    }
+    // let loggingSettings = RED.settings.get('logging');
+    // if(
+    //     typeof loggingSettings.console !== 'undefined' &&
+    //     typeof loggingSettings.console.level !== 'undefined' &&
+    //     ['info','debug','trace'].indexOf(loggingSettings.console.level.toLowerCase()) >= 0
+    // ) {
+    //     loggerPromise.then(({ logger }) => {
+    //         logger.disableAll();
+    //     });
+    // }
 
     function MatrixFolderNameFromUserId(name) {
         return name.replace(/[^a-z0-9]/gi, '_').toLowerCase();
@@ -83,7 +85,7 @@ module.exports = function(RED) {
         } else if(!this.url) {
             node.error("Matrix connection failed: missing server URL in configuration.", {});
         } else {
-            node.setConnected = async function(connected, cb) {
+            node.setConnected = function(connected, cb) {
                 if (node.connected !== connected) {
                     node.connected = connected;
                     if(typeof cb === 'function') {
@@ -145,16 +147,21 @@ module.exports = function(RED) {
 
             fs.ensureDirSync(storageDir); // create storage directory if it doesn't exist
             upgradeDirectoryIfNecessary(node, storageDir);
+
+            // Wait for the dynamic imports to resolve
+            Promise.all([sdkPromise, LocalStorageCryptoStorePromise]).then(([sdkModule, LocalStorageCryptoStoreModule]) => {
+                const sdk = sdkModule.default || sdkModule;
+                const { LocalStorageCryptoStore } = LocalStorageCryptoStoreModule;
+
                 node.matrixClient = sdk.createClient({
                     baseUrl: this.url,
                     accessToken: this.credentials.accessToken,
                     cryptoStore: new LocalStorageCryptoStore(localStorage),
-                store: new MemoryStore({
+                    store: new sdk.MemoryStore({
                         localStorage: localStorage,
                     }),
                     userId: this.userId,
-                deviceId: (this.deviceId || getStoredDeviceId(localStorage)) || undefined,
-                request
+                    deviceId: (this.deviceId || getStoredDeviceId(localStorage)) || undefined
                     // verificationMethods: ["m.sas.v1"]
                 });
 
@@ -192,6 +199,8 @@ module.exports = function(RED) {
                     return node.connected;
                 };
 
+                const { RelationType, RoomEvent, RoomMemberEvent, HttpApiEvent, ClientEvent } = sdk;
+
                 node.matrixClient.on(RoomEvent.Timeline, async function(event, room, toStartOfTimeline, removed, data) {
                     if (toStartOfTimeline) {
                         node.log("Ignoring" + (event.isEncrypted() ? ' encrypted' : '') +" timeline event [" + (event.getContent()['msgtype'] || event.getType()) + "]: (" + room.name + ") " + event.getId() + " for reason: paginated result");
@@ -245,42 +254,18 @@ module.exports = function(RED) {
                     };
 
                     // remove keys from user property that start with an underscore
+                    if (msg.user) {
                         Object.keys(msg.user).forEach(function (key) {
                             if (/^_/.test(key)) {
                                 delete msg.user[key];
                             }
                         });
+                    }
 
                     node.log(`Received ${msg.encrypted ? 'encrypted ' : ''}timeline event [${msg.type}]: (${room.name}) ${event.getSender()} :: ${msg.content.body} ${toStartOfTimeline ? ' [PAGINATED]' : ''}`);
                     node.emit("Room.timeline", event, room, toStartOfTimeline, removed, data, msg);
                 });
 
-            /**
-             * Fires when we want to suggest to the user that they restore their megolm keys
-             * from backup or by cross-signing the device.
-             *
-             * @event module:client~MatrixClient#"crypto.suggestKeyRestore"
-             */
-            // node.matrixClient.on("crypto.suggestKeyRestore", function(){
-            //
-            // });
-
-            // node.matrixClient.on("RoomMember.typing", async function(event, member) {
-            //     let isTyping = member.typing;
-            //     let roomId = member.roomId;
-            // });
-
-            // node.matrixClient.on("RoomMember.powerLevel", async function(event, member) {
-            //     let newPowerLevel = member.powerLevel;
-            //     let newNormPowerLevel = member.powerLevelNorm;
-            //     let roomId = member.roomId;
-            // });
-
-            // node.matrixClient.on("RoomMember.name", async function(event, member) {
-            //     let newName = member.name;
-            //     let roomId = member.roomId;
-            // });
-
                 // handle auto-joining rooms
 
                 node.matrixClient.on(RoomMemberEvent.Membership, async function(event, member) {
@@ -380,17 +365,6 @@ module.exports = function(RED) {
 
 
                 node.matrixClient.on(HttpApiEvent.SessionLoggedOut, async function(errorObj){
-                // Example if user auth token incorrect:
-                // {
-                //     errcode: 'M_UNKNOWN_TOKEN',
-                //     data: {
-                //         errcode: 'M_UNKNOWN_TOKEN',
-                //         error: 'Invalid macaroon passed.',
-                //         soft_logout: false
-                //     },
-                //     httpStatus: 401
-                // }
-
                     node.error("Authentication failure: " + errorObj, {});
                     stopClient();
                 });
@@ -398,8 +372,13 @@ module.exports = function(RED) {
                 async function run() {
                     try {
                         if(node.e2ee){
-                        node.log("Initializing crypto...");
-                        await node.matrixClient.initCrypto();
+                            node.matrixClient.on("crypto.legacyCryptoStoreMigrationProgress", function(progress, total){
+                                node.log(`Migrating from legacy crypto to rust crypto. ${progress}/${total}`);
+                            });
+                            await node.matrixClient.initRustCrypto({
+                                useIndexedDB: false
+                            });
+                            console.log(`CRYPTO VERSION: ${node.matrixClient.getCrypto()?.getVersion()}`);
                             node.matrixClient.getCrypto().globalBlacklistUnverifiedDevices = false; // prevent errors from unverified devices
                         }
                         node.log("Connecting to Matrix server...");
@@ -407,6 +386,7 @@ module.exports = function(RED) {
                             initialSyncLimit: node.initialSyncLimit
                         });
                     } catch(error) {
+                        node.error(error);
                         node.error(error, {});
                     }
                 }
@@ -452,6 +432,9 @@ module.exports = function(RED) {
                             }
                         )
                 })();
+            }).catch((error) => {
+                node.error("Failed to load Matrix SDK modules: " + error, {});
+            });
         }
     }
 
@@ -475,6 +458,9 @@ module.exports = function(RED) {
                 deviceId = req.body.deviceId || undefined,
                 displayName = req.body.displayName || undefined;
 
+            sdkPromise.then((sdkModule) => {
+                const sdk = sdkModule.default || sdk;
+
                 const matrixClient = sdk.createClient({
                     baseUrl: baseUrl,
                     deviceId: deviceId,
@@ -483,8 +469,6 @@ module.exports = function(RED) {
                     request
                 });
 
-            new TimelineWindow()
-
                 matrixClient.timelineSupport = true;
 
                 matrixClient.login(
@@ -512,6 +496,12 @@ module.exports = function(RED) {
                             });
                         }
                     );
+            }).catch((error) => {
+                res.json({
+                    'result': 'error',
+                    'message': "Failed to load Matrix SDK: " + error
+                });
+            });
         });
 
     function upgradeDirectoryIfNecessary(node, storageDir) {