oxmc/myDrive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added dep injection to image thumbnails

Browse Source
This commit is contained in:
subnub
2024-07-07 07:43:04 -04:00
parent 6fc4c56359
commit aca97d934c
6 changed files with 217 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/controllers/file.ts
+9 -10
View File
@@ -1,7 +1,5 @@
import { Request, Response } from "express";
import FileService from "../services/FileService";
import FileSystemService from "../services/ChunkService/FileSystemService";
import S3Service from "../services/ChunkService/S3Service";
import User, { UserInterface } from "../models/user";
import sendShareEmail from "../utils/sendShareEmail";
import {
@@ -15,10 +13,11 @@ import getFileSize from "../services/ChunkService/utils/getFileSize";
import File, { FileMetadateInterface } from "../models/file";
import imageChecker from "../utils/imageChecker";
import videoChecker from "../utils/videoChecker";
import { S3Actions } from "../services/ChunkService/S3Actions";
import { S3Actions } from "../services/ChunkService/Actions/S3Actions";
import { FilesystemActions } from "../services/ChunkService/Actions/FileSystemActions";
import createVideoThumbnail from "../services/ChunkService/utils/createVideoThumbnail";
import NotAuthorizedError from "../utils/NotAuthorizedError";
import createThumbnail from "../services/ChunkService/utils/createImageThumbnail";
const fileService = new FileService();
const storageActions =
@@ -184,7 +183,6 @@ class FileController {
const videoCheck = videoChecker(currentFile.filename);
if (videoCheck) {
console.log("is vidoe");
const updatedFile = await createVideoThumbnail(
currentFile,
filename,
@@ -193,12 +191,13 @@ class FileController {
res.send(updatedFile);
} else if (currentFile.length < 15728640 && imageCheck) {
// const updatedFile = await createThumbnailAny(
// currentFile,
// filename,
// user
// );
// return updatedFile;
const updatedFile = await createThumbnail(
currentFile,
filename,
user
);
res.send(updatedFile);
} else {
res.send(currentFile);
}
backend/db/utils/fileUtils/index.ts
+29 -70
View File
@@ -1,73 +1,59 @@
import mongoose from "../../mongoose";
import { ObjectId } from "mongodb";
import File, { FileInterface } from "../../../models/file";
import File from "../../../models/file";
import { UserInterface } from "../../../models/user";
import { QueryInterface } from "../../../utils/createQuery";
const conn = mongoose.connection;
class DbUtil {
constructor() {}
getPublicFile = async (fileID: string) => {
let file = (await conn.db
.collection("fs.files")
.findOne({ _id: new ObjectId(fileID) })) as FileInterface;
const file = await File.findOne({ _id: new ObjectId(fileID) });
return file;
};
removeOneTimePublicLink = async (
fileID: string | mongoose.Types.ObjectId
) => {
mongoose;
const file = (await conn.db.collection("fs.files").findOneAndUpdate(
const file = await File.findOneAndUpdate(
{ _id: new ObjectId(fileID) },
{
$unset: { "metadata.linkType": "", "metadata.link": "" },
}
)) as FileInterface;
return file;
);
};
removeLink = async (fileID: string, userID: string) => {
const file = (await conn.db
.collection("fs.files")
.findOneAndUpdate(
{ _id: new ObjectId(fileID), "metadata.owner": userID },
{ $unset: { "metadata.linkType": "", "metadata.link": "" } }
)) as FileInterface;
const file = await File.findOneAndUpdate(
{ _id: new ObjectId(fileID), "metadata.owner": userID },
{ $unset: { "metadata.linkType": "", "metadata.link": "" } }
);
return file;
};
makePublic = async (fileID: string, userID: string, token: string) => {
const file = (await conn.db
.collection("fs.files")
.findOneAndUpdate(
{ _id: new ObjectId(fileID), "metadata.owner": userID },
{ $set: { "metadata.linkType": "public", "metadata.link": token } }
)) as FileInterface;
const file = await File.findOneAndUpdate(
{ _id: new ObjectId(fileID), "metadata.owner": userID },
{ $set: { "metadata.linkType": "public", "metadata.link": token } }
);
return file;
};
getPublicInfo = async (fileID: string, tempToken: string) => {
const file = (await conn.db.collection("fs.files").findOne({
const file = await File.findOne({
_id: new ObjectId(fileID),
"metadata.link": tempToken,
})) as FileInterface;
});
return file;
};
makeOneTimePublic = async (fileID: string, userID: string, token: string) => {
const file = (await conn.db
.collection("fs.files")
.findOneAndUpdate(
{ _id: new ObjectId(fileID), "metadata.owner": userID },
{ $set: { "metadata.linkType": "one", "metadata.link": token } }
)) as FileInterface;
const file = await File.findOneAndUpdate(
{ _id: new ObjectId(fileID), "metadata.owner": userID },
{ $set: { "metadata.linkType": "one", "metadata.link": token } }
);
return file;
};
@@ -100,12 +86,7 @@ class DbUtil {
};
getFileInfo = async (fileID: string, userID: string) => {
//TODO: Using mongoose like this causes the object to be returned in raw form
// const file = await File.findOne({
// "metadata.owner": userID,
// _id: new ObjectId(fileID),
// });
const file = await conn.db.collection("fs.files").findOne({
const file = await File.findOne({
"metadata.owner": userID,
_id: new ObjectId(fileID),
});
@@ -116,23 +97,15 @@ class DbUtil {
getQuickList = async (userID: string, limit: number) => {
let query: any = { "metadata.owner": userID, "metadata.trashed": null };
const fileList = (await conn.db
.collection("fs.files")
.find(query)
const fileList = await File.find(query)
.sort({ uploadDate: -1 })
.limit(limit)
.toArray()) as FileInterface[];
.limit(limit);
return fileList;
};
getList = async (queryObj: QueryInterface, sortBy: string, limit: number) => {
const fileList = (await conn.db
.collection("fs.files")
.find(queryObj)
.sort(sortBy)
.limit(limit)
.toArray()) as FileInterface[];
const fileList = await File.find(queryObj).sort(sortBy).limit(limit);
return fileList;
};
@@ -159,11 +132,7 @@ class DbUtil {
if (mediaMode) query = { ...query, "metadata.hasThumbnail": true };
const fileList = (await conn.db
.collection("fs.files")
.find(query)
.limit(10)
.toArray()) as FileInterface[];
const fileList = await File.find(query).limit(10);
return fileList;
};
@@ -212,7 +181,7 @@ class DbUtil {
parent: string,
parentList: string
) => {
const file = await conn.db.collection("fs.files").findOneAndUpdate(
const file = await File.findOneAndUpdate(
{ _id: new ObjectId(fileID), "metadata.owner": userID },
{
$set: {
@@ -229,29 +198,19 @@ class DbUtil {
userID: string | mongoose.Types.ObjectId,
parentListString: string
) => {
const fileList = (await conn.db
.collection("fs.files")
.find({
"metadata.owner": userID,
"metadata.parentList": { $regex: `.*${parentListString}.*` },
})
.toArray()) as FileInterface[];
const fileList = await File.find({
"metadata.owner": userID,
"metadata.parentList": { $regex: `.*${parentListString}.*` },
});
return fileList;
};
getFileListByOwner = async (userID: string) => {
const fileList = (await conn.db
.collection("fs.files")
.find({ "metadata.owner": userID })
.toArray()) as FileInterface[];
const fileList = await File.find({ "metadata.owner": userID });
return fileList;
};
removeChunksByID = async (fileID: string) => {
await conn.db.collection("fs.chunks").deleteMany({ files_id: fileID });
};
}
export default DbUtil;
backend/db/utils/folderUtils/index.ts
+9 -15
View File
@@ -15,18 +15,16 @@ class DbUtil {
trashed: trashMode ? true : null,
};
const folderList = (await Folder.find(query).limit(
10
)) as FolderInterface[];
const folderList = await Folder.find(query).limit(10);
return folderList;
};
getFolderInfo = async (folderID: string, userID: string) => {
const folder = (await Folder.findOne({
const folder = await Folder.findOne({
owner: userID,
_id: new ObjectId(folderID),
})) as FolderInterface;
});
return folder;
};
@@ -67,9 +65,7 @@ class DbUtil {
query = { ...query, trashed: null };
}
const folderList = (await Folder.find(query).sort(
sortBy
)) as FolderInterface[];
const folderList = await Folder.find(query).sort(sortBy);
return folderList;
};
@@ -120,9 +116,7 @@ class DbUtil {
query = { ...query, trashed: null };
}
const folderList = (await Folder.find(query).sort(
sortBy
)) as FolderInterface[];
const folderList = await Folder.find(query).sort(sortBy);
return folderList;
};
@@ -142,19 +136,19 @@ class DbUtil {
};
renameFolder = async (folderID: string, userID: string, title: string) => {
const folder = (await Folder.findOneAndUpdate(
const folder = await Folder.findOneAndUpdate(
{ _id: new ObjectId(folderID), owner: userID },
{ $set: { name: title } }
)) as FolderInterface;
);
return folder;
};
findAllFoldersByParent = async (parentID: string, userID: string) => {
const folderList = (await Folder.find({
const folderList = await Folder.find({
parentList: parentID,
owner: userID,
})) as FolderInterface[];
});
return folderList;
};
backend/models/thumbnail.ts
+31 -26
View File
@@ -1,46 +1,51 @@
import mongoose, {Document} from "mongoose";
import mongoose, { Document } from "mongoose";
const thumbnailSchema = new mongoose.Schema({
const thumbnailSchema = new mongoose.Schema(
{
name: {
type: String,
required: true,
type: String,
required: true,
},
owner: {
type: String,
required: true
type: String,
required: true,
},
data: {
type: Buffer,
type: Buffer,
},
path: {
type: String
type: String,
},
IV: {
type: Buffer,
type: Buffer,
},
s3ID: String,
personalFile: String,
}, {
timestamps: true
})
},
{
timestamps: true,
}
);
export interface ThumbnailInterface extends Document {
_id: any,
name: string,
owner: string,
data?: any,
path?: string,
IV?: Buffer,
s3ID? : string,
personalFile?: boolean,
createdAt: Date,
updatedAt: Date
_id: any;
name: string;
owner: string;
data?: any;
path?: string;
IV: Buffer;
s3ID?: string;
personalFile?: boolean;
createdAt: Date;
updatedAt: Date;
}
const Thumbnail = mongoose.model<ThumbnailInterface>("Thumbnail", thumbnailSchema);
const Thumbnail = mongoose.model<ThumbnailInterface>(
"Thumbnail",
thumbnailSchema
);
export default Thumbnail;
module.exports = Thumbnail;
module.exports = Thumbnail;
backend/services/ChunkService/index.ts
+7 -24
View File
@@ -56,6 +56,7 @@ class StorageService {
parent,
user._id.toString()
);
if (!parentFolder) throw new NotFoundError("Parent Folder Not Found");
parentList.push(...parentFolder.parentList, parentFolder._id);
} else {
parentList.push("/");
@@ -106,12 +107,7 @@ class StorageService {
if (!password) throw new ForbiddenError("Invalid Encryption Key");
// TODO: I believe this has to do with using conn.db instead of the mongoose driver
// We should switch to the mongoose driver
let IV = currentFile.metadata.IV;
if (!(IV instanceof Buffer)) {
IV = IV.buffer;
}
const IV = currentFile.metadata.IV;
const readStreamParams = createGenericParams({
filePath: currentFile.metadata.filePath,
@@ -144,11 +140,7 @@ class StorageService {
throw new ForbiddenError("Thumbnail Unauthorized Error");
}
// TODO: Fix this type
let iv = thumbnail.IV as any;
if (!(iv instanceof Buffer)) {
iv = iv.buffer;
}
const iv = thumbnail.IV;
const CIPHER_KEY = crypto.createHash("sha256").update(password).digest();
@@ -176,11 +168,7 @@ class StorageService {
const password = user.getEncryptionKey();
// TODO: Fix this type
let IV = file.metadata.IV as any;
if (!(IV instanceof Buffer)) {
IV = IV.buffer;
}
const IV = file.metadata.IV;
if (!password) throw new ForbiddenError("Invalid Encryption Key");
@@ -221,7 +209,7 @@ class StorageService {
const parts = range.replace(/bytes=/, "").split("-");
let start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1;
const IV = currentFile.metadata.IV.buffer as Buffer;
const IV = currentFile.metadata.IV;
const chunksize = end - start + 1;
let head = {
@@ -280,7 +268,7 @@ class StorageService {
};
getPublicDownload = async (fileID: string, tempToken: any, res: Response) => {
const file: FileInterface = await dbUtilsFile.getPublicFile(fileID);
const file = await dbUtilsFile.getPublicFile(fileID);
if (!file || !file.metadata.link || file.metadata.link !== tempToken) {
throw new NotAuthorizedError("File Not Public");
@@ -294,12 +282,7 @@ class StorageService {
if (!password) throw new ForbiddenError("Invalid Encryption Key");
// TODO: I believe this has to do with using conn.db instead of the mongoose driver
// We should switch to the mongoose driver
let IV = file.metadata.IV as any;
if (!(IV instanceof Buffer)) {
IV = IV.buffer;
}
const IV = file.metadata.IV;
const readStreamParams = createGenericParams({
filePath: file.metadata.filePath,
backend/services/ChunkService/utils/createImageThumbnail.ts
+132
View File
@@ -0,0 +1,132 @@
import mongoose from "../../../db/mongoose";
import crypto from "crypto";
import Thumbnail from "../../../models/thumbnail";
import sharp from "sharp";
import { FileInterface } from "../../../models/file";
import { UserInterface } from "../../../models/user";
import fs from "fs";
import uuid from "uuid";
import env from "../../../enviroment/env";
import { ObjectId } from "mongodb";
import File from "../../../models/file";
import { createGenericParams } from "./storageHelper";
import { S3Actions } from "../Actions/S3Actions";
import { FilesystemActions } from "../Actions/FileSystemActions";
const conn = mongoose.connection;
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const createThumbnail = (
file: FileInterface,
filename: string,
user: UserInterface
) => {
return new Promise<FileInterface>((resolve, reject) => {
try {
const password = user.getEncryptionKey();
let CIPHER_KEY = crypto.createHash("sha256").update(password!).digest();
const thumbnailFilename = uuid.v4();
const thumbnailIV = crypto.randomBytes(16);
const params = createGenericParams({
filePath: file.metadata.filePath,
Key: file.metadata.s3ID,
});
const readStream = storageActions.createReadStream(params);
const writeStream = storageActions.createWriteStream(
params,
readStream,
thumbnailFilename
);
const decipher = crypto.createDecipheriv(
"aes256",
CIPHER_KEY,
file.metadata.IV
);
const imageResize = sharp().resize(300);
const handleFinish = async () => {
try {
const thumbnailModel = new Thumbnail({
name: filename,
owner: user._id,
IV: thumbnailIV,
path: env.fsDirectory + thumbnailFilename,
});
await thumbnailModel.save();
if (!file._id) {
return reject();
}
const updateFileResponse = await File.updateOne(
{ _id: new ObjectId(file._id), "metadata.owner": user._id },
{
$set: {
"metadata.hasThumbnail": true,
"metadata.thumbnailID": thumbnailModel._id,
},
}
);
if (updateFileResponse.modifiedCount === 0) {
return reject();
}
const updatedFile = await File.findById({
_id: new ObjectId(file._id),
"metadata.owner": user._id,
});
if (!updatedFile) return reject();
resolve(updatedFile?.toObject());
} catch (e) {
console.log("thumbnail error", e);
resolve(file);
}
};
const handleError = (e: Error) => {
console.log("thumbnail stream error", e);
resolve(file);
};
readStream.on("error", handleError);
writeStream.on("error", handleError);
decipher.on("error", handleError);
imageResize.on("error", handleError);
const thumbnailCipher = crypto.createCipheriv(
"aes256",
CIPHER_KEY,
thumbnailIV
);
readStream
.pipe(decipher)
.pipe(imageResize)
.pipe(thumbnailCipher)
.pipe(writeStream);
writeStream.on("finish", async () => {
await handleFinish();
});
} catch (e) {
console.log("File service upload thumbnail error", e);
resolve(file);
}
});
};
export default createThumbnail;