- Add image thumbnail generation support for file upload node (and improved logging) #102

- Fix Received images missing thumbnail kills Node Red #65
- Trim rooms provided in receive node's Room ID config value
This commit is contained in:
2024-06-13 20:21:20 -06:00
parent 5090e4fbb6
commit 351679ad77
5 changed files with 1085 additions and 359 deletions
+291 -228
View File
@@ -1,5 +1,6 @@
const crypto = require("isomorphic-webcrypto");
const ffmpeg = require('fluent-ffmpeg');
const sharp = require('sharp');
const getImageSize = require('image-size');
const tmp = require('tmp');
const fs = require('fs');
@@ -35,274 +36,336 @@ module.exports = function(RED) {
node.status({ fill: "green", shape: "ring", text: "connected" });
});
async function detectFileType(filename, bufferOrPath)
{
async function detectFileType(filename, bufferOrPath, msg) {
const Mime = require('mime');
let file = Buffer.isBuffer(bufferOrPath) ? filename : bufferOrPath;
if(file)
{
let type = Mime.getType(file);
let ext = Mime.getExtension(file);
if(type) {
return {ext: ext, mime: type}
try {
if(file) {
let type = Mime.getType(file);
let ext = Mime.getExtension(file);
if(type) {
return {ext: ext, mime: type};
}
}
} catch (error) {
node.error(`Error detecting file type for ${filename}: ${error.message}`, msg);
}
return null;
}
function getFileBuffer(data)
{
if(Buffer.isBuffer(data)) {
return data;
}
function getFileBuffer(data, msg) {
try {
if (Buffer.isBuffer(data)) {
return data;
}
if (data && RED.settings.fileWorkingDirectory && !path.isAbsolute(data)) {
return fs.readFileSync(path.resolve(path.join(RED.settings.fileWorkingDirectory,data)));
if (data && RED.settings.fileWorkingDirectory && !path.isAbsolute(data)) {
return fs.readFileSync(path.resolve(path.join(RED.settings.fileWorkingDirectory, data)));
}
return fs.readFileSync(data);
} catch (error) {
node.error(`Error reading file buffer: ${error.message}`, msg);
throw error;
}
return fs.readFileSync(data);
}
function getToValue(msg, type, property) {
let value = property;
if (type === "msg") {
value = RED.util.getMessageProperty(msg, property);
} else if ((type === 'flow') || (type === 'global')) {
try {
value = RED.util.evaluateNodeProperty(property, type, node, msg);
} catch(e2) {
throw new Error("Invalid value evaluation");
try {
if (type === "msg") {
value = RED.util.getMessageProperty(msg, property);
} else if ((type === 'flow') || (type === 'global')) {
try {
value = RED.util.evaluateNodeProperty(property, type, node, msg);
} catch(e2) {
throw new Error("Invalid value evaluation");
}
} else if(type === "bool") {
value = (property === 'true');
} else if(type === "num") {
value = Number(property);
}
} else if(type === "bool") {
value = (property === 'true');
} else if(type === "num") {
value = Number(property);
} catch (error) {
node.error(`Error evaluating value for type ${type}: ${error.message}`);
throw new Error("Invalid value evaluation");
}
return value;
}
node.on("input", onInput);
async function onInput(msg)
{
if (! node.server || ! node.server.matrixClient) {
node.warn("No matrix server selected");
return;
}
async function onInput(msg) {
try {
if (!node.server || !node.server.matrixClient) {
node.warn("No matrix server selected");
return;
}
if(!node.server.isConnected()) {
node.error("Matrix server connection is currently closed", msg);
msg.error = "Matrix server connection is currently closed";
node.send([null, msg]);
return;
}
let bufferOrPath = getToValue(msg, node.inputType, node.inputValue);
if(!bufferOrPath) {
node.error('Missing file path/buffer input', msg);
msg.error = 'Missing file path/buffer input';
node.send([null, msg]);
return;
}
let filename = getToValue(msg, node.fileNameType, node.fileNameValue);
if(!filename || typeof filename !== 'string') {
if(!Buffer.isBuffer(bufferOrPath)) {
filename = path.basename(bufferOrPath);
} else {
node.error('Missing filename, this is required if input is a file buffer', msg);
msg.error = 'Missing filename, this is required if input is a file buffer';
if (!node.server.isConnected()) {
node.error("Matrix server connection is currently closed", msg);
msg.error = "Matrix server connection is currently closed";
node.send([null, msg]);
return;
}
}
msg.contentType = node.contentType || msg.contentType || null;
let detectedFileType = await detectFileType(filename, bufferOrPath);
node.log("Detected file type " + JSON.stringify(detectedFileType) + " for " + (Buffer.isBuffer(bufferOrPath) ? 'buffer' : `file ${bufferOrPath}`), msg);
let contentType = msg.contentType || detectedFileType?.mime || null,
msgtype = msg.msgtype || null;
if(!contentType) {
node.warn("Content-type failed to detect, falling back to text/plain", msg);
contentType = 'text/plain';
}
if(!msgtype) {
msgtype = autoDetectMatrixMessageType(detectedFileType);
}
let encryptedFile = null;
if(msg.encrypted) {
encryptedFile = await encryptAttachment(getFileBuffer(bufferOrPath));
}
node.log("Uploading file ", msg);
let file;
try {
file = await node.server.matrixClient.uploadContent(
encryptedFile?.data || getFileBuffer(bufferOrPath),
{
name: filename, // Name to give the file on the server.
rawResponse: false, // Return the raw body, rather than parsing the JSON.
type: contentType, // Content-type for the upload. Defaults to file.type, or applicaton/octet-stream.
onlyContentUri: false // Just return the content URI, rather than the whole body. Defaults to false. Ignored if opts.rawResponse is true.
});
} catch(e) {
node.error("Upload content error " + e);
msg.error = e;
node.send([null, msg]);
return;
}
// we call this method when we need a file and cannot use the buffer
// so if we get passed a buffer we write it to a tmp file and return that
// otherwise we just return the string because it's already a file
let tempFile = null;
function getFile(bufferOrFile) {
if(!Buffer.isBuffer(bufferOrFile)) {
return bufferOrFile; // already a file
let bufferOrPath = getToValue(msg, node.inputType, node.inputValue);
if (!bufferOrPath) {
node.error('Missing file path/buffer input', msg);
msg.error = 'Missing file path/buffer input';
node.send([null, msg]);
return;
}
if(tempFile) {
return tempFile;
let filename = getToValue(msg, node.fileNameType, node.fileNameValue);
if (!filename || typeof filename !== 'string') {
if (!Buffer.isBuffer(bufferOrPath)) {
filename = path.basename(bufferOrPath);
} else {
node.error('Missing filename, this is required if input is a file buffer', msg);
msg.error = 'Missing filename, this is required if input is a file buffer';
node.send([null, msg]);
return;
}
}
// write buffer to tmp file and return path
let tmpObj = tmp.fileSync({ postfix: `.${detectedFileType.ext}` });
fs.writeFileSync(tmpObj.name, bufferOrFile);
tempFile = tmpObj.name;
return tmpObj.name;
}
msg.contentType = node.contentType || msg.contentType || null;
let detectedFileType = await detectFileType(filename, bufferOrPath, msg);
node.log("Detected file type " + JSON.stringify(detectedFileType) + " for " + (Buffer.isBuffer(bufferOrPath) ? 'buffer' : `file ${bufferOrPath}`), msg);
function deleteTempFile() {
if(!tempFile) return null;
fs.rmSync(tempFile);
}
// get size of a buffer or file in bytes
function getFileSize(bufferOrPath) {
if(Buffer.isBuffer(bufferOrPath)) {
return Buffer.byteLength(bufferOrPath);
let contentType = msg.contentType || detectedFileType?.mime || null,
msgtype = msg.msgtype || null;
if (!contentType) {
node.warn("Content-type failed to detect, falling back to text/plain", msg);
contentType = 'text/plain';
}
if (!msgtype) {
msgtype = autoDetectMatrixMessageType(detectedFileType);
}
return fs.statSync(bufferOrPath).size;
}
async function addThumbnail(buffer) {
let imageSize = getImageSize(Buffer.isBuffer(buffer) ? buffer : buffer.data);
msg.payload.info.thumbnail_info = {
w: imageSize.width,
h: imageSize.height,
size: getFileSize(Buffer.isBuffer(buffer) ? buffer : buffer.data)
let encryptedFile = null;
if (msg.encrypted) {
encryptedFile = await encryptAttachment(getFileBuffer(bufferOrPath, msg));
}
let uploadedThumbnail = await node.server.matrixClient.uploadContent(
Buffer.isBuffer(buffer) ? buffer : buffer.data,
{
name: "thumbnail.png", // Name to give the file on the server.
rawResponse: false, // Return the raw body, rather than parsing the JSON.
type: "image/png", // Content-type for the upload. Defaults to file.type, or applicaton/octet-stream.
onlyContentUri: false // Just return the content URI, rather than the whole body. Defaults to false. Ignored if opts.rawResponse is true.
});
// delete local file
if(msg.encrypted) {
msg.payload.info.thumbnail_file.url = uploadedThumbnail.content_uri;
} else {
msg.payload.info.thumbnail_url = uploadedThumbnail.content_uri;
}
}
function _ffmpegVideoThumbnail(filepath){
return new Promise((resolve,reject) => {
let filename = `${msg._msgid}-screenshot.png`;
ffmpeg(filepath)
.on('end', async function() {
let path = `/tmp/${filename}`;
let buffer = getFileBuffer(path);
let encryptedThumbnail = null;
if(msg.encrypted) {
encryptedThumbnail = await encryptAttachment(buffer);
msg.payload.info.thumbnail_file = encryptedFile.info;
}
try {
await addThumbnail(encryptedThumbnail || buffer);
fs.rmSync(path); // delete temporary thumbnail file
resolve();
} catch(e) {
return reject(new Error("Thumbnail upload failure: " + e));
}
})
.on('error', function(err) {
return reject(err);
})
.screenshots({
timestamps: [0],
filename: filename,
folder: '/tmp',
size: '320x?'
node.log("Uploading file ", msg);
let file;
try {
file = await node.server.matrixClient.uploadContent(
encryptedFile?.data || getFileBuffer(bufferOrPath, msg),
{
name: filename, // Name to give the file on the server.
rawResponse: false, // Return the raw body, rather than parsing the JSON.
type: contentType, // Content-type for the upload. Defaults to file.type, or applicaton/octet-stream.
onlyContentUri: false // Just return the content URI, rather than the whole body. Defaults to false. Ignored if opts.rawResponse is true.
});
});
}
msg.payload = {};
if(msg.encrypted) {
msg.payload.file = encryptedFile?.info || {};
msg.payload.file.url = file.content_uri;
} else {
msg.payload.url = file.content_uri;
}
msg.payload.msgtype = msgtype;
msg.payload.body = msg.body || msg.filename || "";
msg.payload.info = {
"mimetype": contentType,
"size": getFileSize(bufferOrPath),
};
if(msgtype === 'm.image') {
// detect size of image
try {
let imageSize = getImageSize(buffer);
msg.payload.info.h = imageSize.height;
msg.payload.info.w = imageSize.width;
} catch(e) {
node.error("Failed to get image size: " + e, msg);
}
} else if(msgtype === 'm.audio' && detectedFileType) {
try {
// detect duration of audio clip
let filepath = getFile(bufferOrPath);
let metadata = await _ffprobe(filepath);
let audioStream = metadata?.streams.filter(function(stream){return stream.codec_type === "audio" || false;})[0];
if(audioStream?.duration) {
msg.payload.info.duration = audioStream?.duration * 1000;
}
} catch(e) {
node.error(e, msg);
}
deleteTempFile();
} else if(msgtype === 'm.video' && detectedFileType) {
let filepath = getFile(bufferOrPath);
try {
// detect duration & width/height of video clip
let metadata = await _ffprobe(filepath);
let videoStream = metadata?.streams.filter(function(stream){return stream.codec_type === "video" || false;})[0];
if(videoStream) {
msg.payload.info.duration = videoStream.duration * 1000;
msg.payload.info.w = videoStream.width;
msg.payload.info.h = videoStream.height;
}
} catch(e) {
node.error("ffprobe error: " + e);
} catch (e) {
node.error("Upload content error " + e, msg);
msg.error = e;
node.send([null, msg]);
return;
}
if(node.generateThumbnails) {
// we call this method when we need a file and cannot use the buffer
// so if we get passed a buffer we write it to a tmp file and return that
// otherwise we just return the string because it's already a file
let tempFile = null;
function getFile(bufferOrFile) {
try {
await _ffmpegVideoThumbnail(filepath);
} catch(e) {
node.error("Screenshot generation error: " + e);
if (!Buffer.isBuffer(bufferOrFile)) {
return bufferOrFile; // already a file
}
if (tempFile) {
return tempFile;
}
// write buffer to tmp file and return path
let tmpObj = tmp.fileSync({postfix: `.${detectedFileType.ext}`});
fs.writeFileSync(tmpObj.name, bufferOrFile);
tempFile = tmpObj.name;
return tmpObj.name;
} catch (error) {
node.error(`Error creating temp file: ${error.message}`, msg);
throw error;
}
}
deleteTempFile();
}
node.send(msg, null);
function deleteTempFile() {
if (!tempFile) return null;
try {
fs.rmSync(tempFile);
} catch (error) {
node.error(`Error deleting temp file: ${error.message}`, msg);
}
}
// get size of a buffer or file in bytes
function getFileSize(bufferOrPath) {
try {
if (Buffer.isBuffer(bufferOrPath)) {
return Buffer.byteLength(bufferOrPath);
}
return fs.statSync(bufferOrPath).size;
} catch (error) {
node.error(`Error getting file size: ${error.message}`, msg);
throw error;
}
}
async function addThumbnail(buffer) {
try {
let imageSize = getImageSize(Buffer.isBuffer(buffer) ? buffer : buffer.data);
msg.payload.info.thumbnail_info = {
w: imageSize.width,
h: imageSize.height,
size: getFileSize(Buffer.isBuffer(buffer) ? buffer : buffer.data)
}
let uploadedThumbnail = await node.server.matrixClient.uploadContent(
Buffer.isBuffer(buffer) ? buffer : buffer.data,
{
name: "thumbnail.png", // Name to give the file on the server.
rawResponse: false, // Return the raw body, rather than parsing the JSON.
type: "image/png", // Content-type for the upload. Defaults to file.type, or applicaton/octet-stream.
onlyContentUri: false // Just return the content URI, rather than the whole body. Defaults to false. Ignored if opts.rawResponse is true.
});
// delete local file
if (msg.encrypted) {
msg.payload.info.thumbnail_file.url = uploadedThumbnail.content_uri;
} else {
msg.payload.info.thumbnail_url = uploadedThumbnail.content_uri;
}
} catch (error) {
node.error(`Error adding thumbnail: ${error.message}`, msg);
}
}
async function createImageThumbnail(bufferOrPath) {
try {
if (Buffer.isBuffer(bufferOrPath)) {
return sharp(bufferOrPath).resize({width: 320}).toBuffer();
}
return sharp(fs.readFileSync(bufferOrPath)).resize({width: 320}).toBuffer();
} catch (error) {
node.error(`Error creating image thumbnail: ${error.message}`);
throw error;
}
}
function _ffmpegVideoThumbnail(filepath) {
return new Promise((resolve, reject) => {
let filename = `${msg._msgid}-screenshot.png`;
ffmpeg(filepath)
.on('end', async function () {
let path = `/tmp/${filename}`;
let buffer = getFileBuffer(path, msg);
let encryptedThumbnail = null;
if (msg.encrypted) {
encryptedThumbnail = await encryptAttachment(buffer);
msg.payload.info.thumbnail_file = encryptedFile.info;
}
try {
await addThumbnail(encryptedThumbnail || buffer);
fs.rmSync(path); // delete temporary thumbnail file
resolve();
} catch (e) {
return reject(new Error("Thumbnail upload failure: " + e));
}
})
.on('error', function (err) {
return reject(err);
})
.screenshots({
timestamps: [0],
filename: filename,
folder: '/tmp',
size: '320x?'
});
});
}
msg.payload = {};
if (msg.encrypted) {
msg.payload.file = encryptedFile?.info || {};
msg.payload.file.url = file.content_uri;
} else {
msg.payload.url = file.content_uri;
}
msg.payload.msgtype = msgtype;
msg.payload.body = msg.body || msg.filename || "";
msg.payload.info = {
"mimetype": contentType,
"size": getFileSize(bufferOrPath),
};
if (msgtype === 'm.image') {
// detect size of image
try {
let imageSize = getImageSize(bufferOrPath);
msg.payload.info.h = imageSize.height;
msg.payload.info.w = imageSize.width;
// Generate thumbnail for image
if (node.generateThumbnails) {
let thumbnailBuffer = await createImageThumbnail(bufferOrPath);
let encryptedThumbnail = null;
if (msg.encrypted) {
encryptedThumbnail = await encryptAttachment(thumbnailBuffer);
msg.payload.info.thumbnail_file = encryptedThumbnail.info;
}
await addThumbnail(encryptedThumbnail || thumbnailBuffer);
}
} catch (e) {
node.error("thumbnail error: " + e, msg);
}
} else if (msgtype === 'm.audio' && detectedFileType) {
try {
// detect duration of audio clip
let filepath = getFile(bufferOrPath);
let metadata = await _ffprobe(filepath);
let audioStream = metadata?.streams.filter(function (stream) {
return stream.codec_type === "audio" || false;
})[0];
if (audioStream?.duration) {
msg.payload.info.duration = audioStream?.duration * 1000;
}
} catch (e) {
node.error(e, msg);
}
deleteTempFile();
} else if (msgtype === 'm.video' && detectedFileType) {
let filepath = getFile(bufferOrPath);
try {
// detect duration & width/height of video clip
let metadata = await _ffprobe(filepath);
let videoStream = metadata?.streams.filter(function (stream) {
return stream.codec_type === "video" || false;
})[0];
if (videoStream) {
msg.payload.info.duration = videoStream.duration * 1000;
msg.payload.info.w = videoStream.width;
msg.payload.info.h = videoStream.height;
}
} catch (e) {
node.error("ffprobe error: " + e, msg);
}
if (node.generateThumbnails) {
try {
await _ffmpegVideoThumbnail(filepath);
} catch (e) {
node.error("Screenshot generation error: " + e, msg);
}
}
deleteTempFile();
}
node.send(msg, null);
} catch (error) {
node.error(`Unhandled error: ${error.message}`, msg);
node.log(error, msg);
}
}
node.on("close", function() {