From e466ec0d0728f0519fc0116b676c2ba44600b8a3 Mon Sep 17 00:00:00 2001 From: subnub Date: Sat, 7 Dec 2024 18:55:56 -0500 Subject: [PATCH] cleaned up file uploads --- backend/controllers/file-controller.ts | 116 +--------- backend/db/mongoDB/fileDB.ts | 21 ++ .../chunk-service/actions/helper-actions.ts | 11 + .../services/chunk-service/chunk-service.ts | 93 ++------ .../utils/createImageThumbnail.ts | 4 +- .../utils/createVideoThumbnail.ts | 4 +- .../chunk-service/utils/getBusboyData.ts | 204 ++++++++++++++++-- .../utils/getFolderUploadBusboyData.ts | 4 +- .../utils/tempCreateVideoThumbnail.ts | 6 +- .../utils/tempCreateVideoThumbnailFS.ts | 4 +- 10 files changed, 250 insertions(+), 217 deletions(-) create mode 100644 backend/services/chunk-service/actions/helper-actions.ts diff --git a/backend/controllers/file-controller.ts b/backend/controllers/file-controller.ts index 5b318ea..62efe11 100644 --- a/backend/controllers/file-controller.ts +++ b/backend/controllers/file-controller.ts @@ -131,126 +131,30 @@ class FileController { } }; - uploadFile = async (req: RequestTypeFullUser, res: Response) => { + uploadFile = async ( + req: RequestTypeFullUser, + res: Response, + next: NextFunction + ) => { if (!req.user) { return; } - let responseSent = false; - - const handleError = () => { - if (!responseSent) { - responseSent = true; - res.writeHead(500, { Connection: "close" }); - res.end(); - } - }; - - const handleFinish = async ( - filename: string, - metadata: FileMetadateInterface - ) => { - try { - const user = req.user; - - if (!user) throw new NotAuthorizedError("User Not Authorized"); - - const date = new Date(); - - let length = 0; - - if (env.dbType === "fs" && metadata.filePath) { - length = (await getFileSize(metadata.filePath)) as number; - } else { - // TODO: Fix this we should be using the encrypted file size - length = metadata.size; - } - - const currentFile = new File({ - filename, - uploadDate: date.toISOString(), - length, - metadata, - }); - - await currentFile.save(); - - const imageCheck = imageChecker(currentFile.filename); - const videoCheck = videoChecker(currentFile.filename); - - if (videoCheck) { - const updatedFile = await createVideoThumbnail( - currentFile, - filename, - user - ); - - res.send(updatedFile); - } else if (currentFile.length < 15728640 && imageCheck) { - const updatedFile = await createThumbnail( - currentFile, - filename, - user - ); - - res.send(updatedFile); - } else { - res.send(currentFile); - } - } catch (e: unknown) { - if (!responseSent) { - res.writeHead(500, { Connection: "close" }); - res.end(); - } - } - }; - try { const user = req.user; const busboy = req.busboy; - busboy.on("error", (e: Error) => { - console.log("busboy error", e); - handleError(); + req.on("error", (e: Error) => { + next(e); }); req.pipe(busboy); - const { cipher, fileWriteStream, emitter, metadata, filename } = - await this.chunkService.uploadFile(user, busboy, req); + const file = await this.chunkService.uploadFile(user, busboy, req); - cipher.on("error", (e: Error) => { - console.log("cipher error", e); - handleError(); - }); - - fileWriteStream.on("error", (e: Error) => { - console.log("file write stream error", e); - handleError(); - }); - - if (emitter) { - emitter.on("finish", async () => { - await handleFinish(filename, metadata); - }); - } else { - fileWriteStream.on("finish", async () => { - await handleFinish(filename, metadata); - }); - } - - cipher.pipe(fileWriteStream); + res.send(file); } catch (e: unknown) { - if (!responseSent) { - res.writeHead(500, { Connection: "close" }); - if (e instanceof Error) { - console.log("\nUploading File Error File Route:", e.message); - res.end(e.message); - } else { - console.log("\nUploading File Error File Route:", e); - res.end("Server error uploading file"); - } - } + next(e); } }; diff --git a/backend/db/mongoDB/fileDB.ts b/backend/db/mongoDB/fileDB.ts index 97029cb..6314018 100644 --- a/backend/db/mongoDB/fileDB.ts +++ b/backend/db/mongoDB/fileDB.ts @@ -120,6 +120,27 @@ class DbUtil { // UPDATE + updateFileUploadedFile = async ( + fileID: string, + userID: string, + parent: string, + parentList: string + ) => { + const file = await File.findOneAndUpdate( + { _id: new ObjectId(fileID), "metadata.owner": userID }, + { + $set: { + "metadata.parent": parent, + "metadata.parentList": parentList, + }, + $unset: { "metadata.processingFile": null }, + }, + { new: true } + ); + + return file; + }; + updateFolderUploadedFile = async ( fileID: string, userID: string, diff --git a/backend/services/chunk-service/actions/helper-actions.ts b/backend/services/chunk-service/actions/helper-actions.ts new file mode 100644 index 0000000..5493d3d --- /dev/null +++ b/backend/services/chunk-service/actions/helper-actions.ts @@ -0,0 +1,11 @@ +import { S3Actions } from "../actions/S3-actions"; +import { FilesystemActions } from "../actions/file-system-actions"; +import env from "../../../enviroment/env"; + +export const getStorageActions = () => { + if (env.dbType === "s3") { + return new S3Actions(); + } else { + return new FilesystemActions(); + } +}; diff --git a/backend/services/chunk-service/chunk-service.ts b/backend/services/chunk-service/chunk-service.ts index 25f0548..2f7af34 100644 --- a/backend/services/chunk-service/chunk-service.ts +++ b/backend/services/chunk-service/chunk-service.ts @@ -3,7 +3,7 @@ import { UserInterface } from "../../models/user-model"; import NotAuthorizedError from "../../utils/NotAuthorizedError"; import NotFoundError from "../../utils/NotFoundError"; import crypto from "crypto"; -import getBusboyData from "./utils/getBusboyData"; +import uploadFileToStorage from "./utils/getBusboyData"; import videoChecker from "../../utils/videoChecker"; import uuid from "uuid"; import { FileInterface, FileMetadateInterface } from "../../models/file-model"; @@ -23,65 +23,20 @@ import fixEndChunkLength from "./utils/fixEndChunkLength"; import archiver from "archiver"; import async from "async"; import getFolderBusboyData from "./utils/getFolderUploadBusboyData"; +import { getStorageActions } from "./actions/helper-actions"; const fileDB = new FileDB(); const folderDB = new FolderDB(); const thumbnailDB = new ThumbnailDB(); const userDB = new UserDB(); -const storageActions = - env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); +const storageActions = getStorageActions(); class StorageService { constructor() {} uploadFile = async (user: UserInterface, busboy: any, req: Request) => { - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const initVect = crypto.randomBytes(16); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect); - - const { file, filename: fileInfo, formData } = await getBusboyData(busboy); - - const filename = fileInfo.filename; - const parent = formData.get("parent") || "/"; - const size = +formData.get("size"); - const hasThumbnail = false; - const thumbnailID = ""; - const isVideo = videoChecker(filename); - - const errors = []; - - if (!filename || filename.length === 0 || filename.length > 255) { - errors.push({ - msg: "Filename is required and length must be greater than 0 and less than 255", - param: "filename", - }); - } - - if (!file) { - errors.push({ msg: "File is required", param: "file" }); - } - - // @prettier-ignore - if (!(file instanceof Readable)) { - errors.push({ msg: "File must be a readable stream", param: "file" }); - } - - if (!size || size < 0) { - errors.push({ msg: "Size must be a positive integer", param: "size" }); - } - - if (errors.length > 0) { - throw new Error( - `Invalid request parameters ${errors.map((e) => e.msg).join(", ")}` - ); - } + const { parent, file } = await uploadFileToStorage(busboy, user); const parentList = []; @@ -96,38 +51,14 @@ class StorageService { parentList.push("/"); } - const randomFilenameID = uuid.v4(); - - const metadata = { - owner: user._id.toString(), + await fileDB.updateFileUploadedFile( + file._id!.toString(), + user._id.toString(), parent, - parentList: parentList.toString(), - hasThumbnail, - thumbnailID, - isVideo, - size, - IV: initVect, - } as FileMetadateInterface; - - if (env.dbType === "fs") { - metadata.filePath = env.fsDirectory + randomFilenameID; - } else { - metadata.s3ID = randomFilenameID; - } - - const { writeStream, emitter } = storageActions.createWriteStream( - metadata, - file.pipe(cipher), - randomFilenameID + parentList.toString() ); - return { - cipher, - fileWriteStream: writeStream, - emitter, - metadata, - filename, - }; + return file; }; uploadFolder = async ( @@ -202,7 +133,7 @@ class StorageService { if (parentDirectory && foldersCreated[parentDirectory]) { tempParentList.push( ...foldersCreated[parentDirectory].parentList, - foldersCreated[parentDirectory]._id.toString() + foldersCreated[parentDirectory]._id!.toString() ); } else { tempParentList.push(...parentList); @@ -230,10 +161,10 @@ class StorageService { await fileDB.updateFolderUploadedFile( currentFile.uploadedFileId, user._id.toString(), - foldersCreated[parentDirectory]._id.toString(), + foldersCreated[parentDirectory]._id!.toString(), [ ...foldersCreated[parentDirectory].parentList, - foldersCreated[parentDirectory]._id.toString(), + foldersCreated[parentDirectory]._id!.toString(), ].toString() ); } else { diff --git a/backend/services/chunk-service/utils/createImageThumbnail.ts b/backend/services/chunk-service/utils/createImageThumbnail.ts index d93adf3..40ca2c0 100644 --- a/backend/services/chunk-service/utils/createImageThumbnail.ts +++ b/backend/services/chunk-service/utils/createImageThumbnail.ts @@ -10,11 +10,11 @@ import { S3Actions } from "../actions/S3-actions"; import { FilesystemActions } from "../actions/file-system-actions"; import { EventEmitter } from "stream"; import FileDB from "../../../db/mongoDB/fileDB"; +import { getStorageActions } from "../actions/helper-actions"; const fileDB = new FileDB(); -const storageActions = - env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); +const storageActions = getStorageActions(); const processData = ( file: FileInterface, diff --git a/backend/services/chunk-service/utils/createVideoThumbnail.ts b/backend/services/chunk-service/utils/createVideoThumbnail.ts index 37e7574..debb611 100644 --- a/backend/services/chunk-service/utils/createVideoThumbnail.ts +++ b/backend/services/chunk-service/utils/createVideoThumbnail.ts @@ -14,9 +14,9 @@ import tempCreateVideoThumbnailFS from "./tempCreateVideoThumbnailFS"; import { S3Actions } from "../actions/S3-actions"; import { FilesystemActions } from "../actions/file-system-actions"; import { createGenericParams } from "./storageHelper"; +import { getStorageActions } from "../actions/helper-actions"; -const storageActions = - env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); +const storageActions = getStorageActions(); const attemptToRemoveChunks = async ( file: FileInterface, diff --git a/backend/services/chunk-service/utils/getBusboyData.ts b/backend/services/chunk-service/utils/getBusboyData.ts index 9e70a21..72d88ef 100644 --- a/backend/services/chunk-service/utils/getBusboyData.ts +++ b/backend/services/chunk-service/utils/getBusboyData.ts @@ -1,20 +1,170 @@ -import { Stream } from "stream"; +import { EventEmitter, Stream } from "stream"; +import { UserInterface } from "../../../models/user-model"; +import File, { + FileInterface, + FileMetadateInterface, +} from "../../../models/file-model"; +import uuid from "uuid"; +import crypto from "crypto"; +import ForbiddenError from "../../../utils/ForbiddenError"; +import env from "../../../enviroment/env"; +import { getStorageActions } from "../actions/helper-actions"; +import getFileSize from "./getFileSize"; +import imageChecker from "../../../utils/imageChecker"; +import videoChecker from "../../../utils/videoChecker"; +import createVideoThumbnail from "./createVideoThumbnail"; +import createThumbnail from "./createImageThumbnail"; -type dataType = { - file: Stream; - filename: { - filename: string; - }; - formData: Map; +// TODO: We should stop using moongoose directly here, +// Also in our fileDB make sure we are actually using File instead +// Of just modifying data directly so we get validation + +const storageActions = getStorageActions(); + +type FileInfo = { + file: FileInterface; + parent: string; }; -const getBusboyData = (busboy: any) => { - return new Promise((resolve, reject) => { - const formData = new Map(); +const processData = (busboy: any, user: UserInterface) => { + const eventEmitter = new EventEmitter(); + + try { + let parent = ""; + let size = 0; + + const handleFinish = async ( + filename: string, + metadata: FileMetadateInterface + ) => { + const date = new Date(); + + let length = 0; + + if (env.dbType === "fs" && metadata.filePath) { + length = (await getFileSize(metadata.filePath)) as number; + } else { + // TODO: Fix this we should be using the encrypted file size + length = metadata.size; + } + + const currentFile = new File({ + filename, + uploadDate: date.toISOString(), + length, + metadata, + }); + + await currentFile.save(); + + const imageCheck = imageChecker(currentFile.filename); + const videoCheck = videoChecker(currentFile.filename); + + if (videoCheck) { + const updatedFile = await createVideoThumbnail( + currentFile, + filename, + user + ); + return updatedFile; + } else if (currentFile.length < 15728640 && imageCheck) { + const updatedFile = await createThumbnail(currentFile, filename, user); + return updatedFile; + } else { + return currentFile; + } + }; + + const uploadFile = (filename: string, fileStream: Stream) => { + return new Promise<{ filename: string; metadata: FileMetadateInterface }>( + (resolve, reject) => { + const randomFilenameID = uuid.v4(); + + const password = user.getEncryptionKey(); + + if (!password) throw new ForbiddenError("Invalid Encryption Key"); + + const initVect = crypto.randomBytes(16); + + const CIPHER_KEY = crypto + .createHash("sha256") + .update(password) + .digest(); + + const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect); + + const metadata = { + owner: user._id.toString(), + parent: "/", + parentList: ["/"].toString(), + hasThumbnail: false, + thumbnailID: "", + isVideo: false, + size, + IV: initVect, + processingFile: true, + } as FileMetadateInterface; + + if (env.dbType === "fs") { + metadata.filePath = env.fsDirectory + randomFilenameID; + } else { + metadata.s3ID = randomFilenameID; + } + + const { writeStream, emitter } = storageActions.createWriteStream( + metadata, + fileStream.pipe(cipher), + randomFilenameID + ); + + writeStream.on("error", (e: Error) => { + reject(e); + }); + + cipher.on("error", (e: Error) => { + reject(e); + }); + + fileStream.on("error", (e: Error) => { + reject(e); + }); + + cipher.pipe(writeStream); + + if (emitter) { + emitter.on("finish", () => { + resolve({ filename, metadata }); + }); + } else { + writeStream.on("finish", () => { + resolve({ filename, metadata }); + }); + } + } + ); + }; + + const processFile = async (filename: string, fileStream: Stream) => { + try { + const { filename: newFilename, metadata } = await uploadFile( + filename, + fileStream + ); + const file = await handleFinish(newFilename, metadata); + eventEmitter.emit("finish", { + file, + parent, + }); + } catch (e) { + eventEmitter.emit("error", e); + } + }; busboy.on("field", (field: any, val: any) => { - if (typeof val !== "string" || val !== "undefined") { - formData.set(field, val); + if (field === "parent") { + parent = val; + } else if (field === "size") { + size = +val; } }); @@ -23,18 +173,34 @@ const getBusboyData = (busboy: any) => { ( _: string, file: Stream, - filename: { + filedata: { filename: string; } ) => { - resolve({ - file, - filename, - formData, - }); + processFile(filedata.filename, file); } ); + + busboy.on("error", (e: Error) => { + eventEmitter.emit("error", e); + }); + } catch (e) { + eventEmitter.emit("error", e); + } + + return eventEmitter; +}; + +const uploadFileToStorage = (busboy: any, user: UserInterface) => { + return new Promise((resolve, reject) => { + const eventEmitter = processData(busboy, user); + eventEmitter.on("finish", (data) => { + resolve(data); + }); + eventEmitter.on("error", (e) => { + reject(e); + }); }); }; -export default getBusboyData; +export default uploadFileToStorage; diff --git a/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts b/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts index c381292..3b3a723 100644 --- a/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts +++ b/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts @@ -16,6 +16,7 @@ import videoChecker from "../../../utils/videoChecker"; import createVideoThumbnail from "./createVideoThumbnail"; import createThumbnail from "./createImageThumbnail"; import { EventEmitter } from "events"; +import { getStorageActions } from "../actions/helper-actions"; type FileDataType = { name: string; @@ -27,8 +28,7 @@ type FileDataType = { uploadedFileId: string; }; -const storageActions = - env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); +const storageActions = getStorageActions(); type dataType = Record; diff --git a/backend/services/chunk-service/utils/tempCreateVideoThumbnail.ts b/backend/services/chunk-service/utils/tempCreateVideoThumbnail.ts index 9994976..fdefdfb 100644 --- a/backend/services/chunk-service/utils/tempCreateVideoThumbnail.ts +++ b/backend/services/chunk-service/utils/tempCreateVideoThumbnail.ts @@ -13,9 +13,9 @@ import ffmpeg from "fluent-ffmpeg"; import { S3Actions } from "../actions/S3-actions"; import { FilesystemActions } from "../actions/file-system-actions"; import { createGenericParams } from "./storageHelper"; +import { getStorageActions } from "../actions/helper-actions"; -const storageActions = - env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); +const storageActions = getStorageActions(); const tempCreateVideoThumbnail = ( file: FileInterface, @@ -43,7 +43,7 @@ const tempCreateVideoThumbnail = ( const readStream = storageActions.createReadStream(readStreamParams); - const writeStream = storageActions.createWriteStream( + const { writeStream, emitter } = storageActions.createWriteStream( readStreamParams, readStream, thumbnailFilename diff --git a/backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts b/backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts index 3acb5fd..52bfa53 100644 --- a/backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts +++ b/backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts @@ -13,9 +13,9 @@ import ffmpeg from "fluent-ffmpeg"; import { S3Actions } from "../actions/S3-actions"; import { FilesystemActions } from "../actions/file-system-actions"; import { createGenericParams } from "./storageHelper"; +import { getStorageActions } from "../actions/helper-actions"; -const storageActions = - env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); +const storageActions = getStorageActions(); const tempCreateVideoThumbnailFS = ( file: FileInterface,