From 6fc4c56359463c1e4744a9328f24523130f5b24f Mon Sep 17 00:00:00 2001 From: subnub Date: Fri, 5 Jul 2024 14:22:36 -0400 Subject: [PATCH] cleanup --- backend/controllers/file.ts | 2 +- .../{ => Actions}/FileSystemActions.ts | 6 +- .../ChunkService/{ => Actions}/S3Actions.ts | 7 +- .../ChunkService/FileSystemService.ts | 666 ------------------ backend/services/ChunkService/S3Service.ts | 639 ----------------- backend/services/ChunkService/index.ts | 4 +- .../utils/createVideoThumbnail.ts | 4 +- .../utils/tempCreateVideoThumbnail.ts | 4 +- 8 files changed, 13 insertions(+), 1319 deletions(-) rename backend/services/ChunkService/{ => Actions}/FileSystemActions.ts (91%) rename backend/services/ChunkService/{ => Actions}/S3Actions.ts (92%) delete mode 100644 backend/services/ChunkService/FileSystemService.ts delete mode 100644 backend/services/ChunkService/S3Service.ts diff --git a/backend/controllers/file.ts b/backend/controllers/file.ts index da247fa..9a3da57 100644 --- a/backend/controllers/file.ts +++ b/backend/controllers/file.ts @@ -16,7 +16,7 @@ import File, { FileMetadateInterface } from "../models/file"; import imageChecker from "../utils/imageChecker"; import videoChecker from "../utils/videoChecker"; import { S3Actions } from "../services/ChunkService/S3Actions"; -import { FilesystemActions } from "../services/ChunkService/FileSystemActions"; +import { FilesystemActions } from "../services/ChunkService/Actions/FileSystemActions"; import createVideoThumbnail from "../services/ChunkService/utils/createVideoThumbnail"; import NotAuthorizedError from "../utils/NotAuthorizedError"; diff --git a/backend/services/ChunkService/FileSystemActions.ts b/backend/services/ChunkService/Actions/FileSystemActions.ts similarity index 91% rename from backend/services/ChunkService/FileSystemActions.ts rename to backend/services/ChunkService/Actions/FileSystemActions.ts index 8d311f7..a98890e 100644 --- a/backend/services/ChunkService/FileSystemActions.ts +++ b/backend/services/ChunkService/Actions/FileSystemActions.ts @@ -1,7 +1,7 @@ import fs from "fs"; -import { UserInterface } from "../../models/user"; -import { GenericParams, IStorageActions } from "./StoreTypes"; -import env from "../../enviroment/env"; +import { UserInterface } from "../../../models/user"; +import { GenericParams, IStorageActions } from "../StoreTypes"; +import env from "../../../enviroment/env"; class FilesystemActions implements IStorageActions { async getAuth() { diff --git a/backend/services/ChunkService/S3Actions.ts b/backend/services/ChunkService/Actions/S3Actions.ts similarity index 92% rename from backend/services/ChunkService/S3Actions.ts rename to backend/services/ChunkService/Actions/S3Actions.ts index b701eec..3060b7d 100644 --- a/backend/services/ChunkService/S3Actions.ts +++ b/backend/services/ChunkService/Actions/S3Actions.ts @@ -1,7 +1,6 @@ -import { UserInterface } from "../../models/user"; -import s3 from "../../db/s3"; -import env from "../../enviroment/env"; -import { GenericParams, IStorageActions } from "./StoreTypes"; +import s3 from "../../../db/s3"; +import env from "../../../enviroment/env"; +import { GenericParams, IStorageActions } from "./../StoreTypes"; import internal from "stream"; import { PassThrough } from "stream"; diff --git a/backend/services/ChunkService/FileSystemService.ts b/backend/services/ChunkService/FileSystemService.ts deleted file mode 100644 index 01f1455..0000000 --- a/backend/services/ChunkService/FileSystemService.ts +++ /dev/null @@ -1,666 +0,0 @@ -import { Response, Request } from "express"; -import ChunkInterface from "./utils/ChunkInterface"; -import { UserInterface } from "../../models/user"; -import NotAuthorizedError from "../../utils/NotAuthorizedError"; -import NotFoundError from "../../utils/NotFoundError"; -import crypto from "crypto"; -import getBusboyData from "./utils/getBusboyData"; -import videoChecker from "../../utils/videoChecker"; -import fs from "fs"; -import uuid from "uuid"; -import awaitUploadStreamFS from "./utils/awaitUploadStreamFS"; -import File, { FileInterface } from "../../models/file"; -import getFileSize from "./utils/getFileSize"; -import DbUtilFile from "../../db/utils/fileUtils/index"; -import DbUtilFolder from "../../db/utils/folderUtils/index"; -import awaitStream from "./utils/awaitStream"; -import createThumbnailAny from "./utils/createThumbailAny"; -import imageChecker from "../../utils/imageChecker"; -import Thumbnail, { ThumbnailInterface } from "../../models/thumbnail"; -import streamToBuffer from "../../utils/streamToBuffer"; -import User from "../../models/user"; -import env from "../../enviroment/env"; -import removeChunksFS from "./utils/removeChunksFS"; -import getPrevIVFS from "./utils/getPrevIVFS"; -import awaitStreamVideo from "./utils/awaitStreamVideo"; -import fixStartChunkLength from "./utils/fixStartChunkLength"; -import Folder, { FolderInterface } from "../../models/folder"; -import addToStoageSize from "./utils/addToStorageSize"; -import subtractFromStorageSize from "./utils/subtractFromStorageSize"; -import ForbiddenError from "../../utils/ForbiddenError"; -import { ObjectId } from "mongodb"; -import FolderService from "../FolderService"; -import MongoFileService from "../FileService"; -import createVideoThumbnailFS from "./utils/createVideoThumbnailFS"; - -const dbUtilsFile = new DbUtilFile(); -const dbUtilsFolder = new DbUtilFolder(); -const folderService = new FolderService(); -const fileService = new MongoFileService(); - -class FileSystemService implements ChunkInterface { - constructor() {} - - uploadFile = async (user: UserInterface, busboy: any, req: Request) => { - console.log("upeload file2"); - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const initVect = crypto.randomBytes(16); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect); - - const { file, filename: fileInfo, formData } = await getBusboyData(busboy); - - const filename = fileInfo.filename; - console.log("1"); - const parent = formData.get("parent") || "/"; - const size = formData.get("size") || ""; - const personalFile = formData.get("personal-file") ? true : false; - let hasThumbnail = false; - let thumbnailID = ""; - const isVideo = videoChecker(filename); - console.log("2", parent, typeof parent); - - const parentList = []; - - if (parent !== "/") { - const parentFolder = await dbUtilsFolder.getFolderInfo( - parent, - user._id.toString() - ); - parentList.push(...parentFolder.parentList, parentFolder._id); - } else { - parentList.push("/"); - } - - console.log("parent list", parentList); - - const systemFileName = uuid.v4(); - - const metadata = { - owner: user._id, - parent, - parentList: parentList.toString(), - hasThumbnail, - thumbnailID, - isVideo, - size, - IV: initVect, - filePath: env.fsDirectory + systemFileName, - }; - - const fileWriteStream = fs.createWriteStream(metadata.filePath); - - const totalStreamsToErrorCatch = [file, cipher, fileWriteStream]; - - await awaitUploadStreamFS( - file.pipe(cipher), - fileWriteStream, - req, - metadata.filePath, - totalStreamsToErrorCatch - ); - - const date = new Date(); - const encryptedFileSize = await getFileSize(metadata.filePath); - - const currentFile = new File({ - filename, - uploadDate: date.toISOString(), - length: encryptedFileSize, - metadata, - }); - - await currentFile.save(); - - await addToStoageSize(user, size, personalFile); - - const imageCheck = imageChecker(currentFile.filename); - const videoCheck = videoChecker(currentFile.filename); - - if (videoCheck) { - console.log("is vidoe"); - const updatedFile = await createVideoThumbnailFS( - currentFile, - filename, - user - ); - - return updatedFile; - } else if (currentFile.length < 15728640 && imageCheck) { - const updatedFile = await createThumbnailAny(currentFile, filename, user); - - return updatedFile; - } else { - return currentFile; - } - }; - - getFileWriteStream = async ( - user: UserInterface, - file: FileInterface, - parentFolder: FolderInterface - ) => { - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const initVect = crypto.randomBytes(16); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect); - - const filename = file.filename; - const parent = parentFolder._id; - const parentList = [...parentFolder.parentList, parentFolder._id]; - const size = file.metadata.size; - const personalFile = file.metadata.personalFile ? true : false; - let hasThumbnail = file.metadata.hasThumbnail; - let thumbnailID = file.metadata.thumbnailID; - const isVideo = file.metadata.isVideo; - - const systemFileName = uuid.v4(); - - const metadata = { - owner: user._id, - parent, - parentList, - hasThumbnail, - thumbnailID, - isVideo, - size, - IV: file.metadata.IV, - filePath: env.fsDirectory + systemFileName, - }; - - const fileWriteStream = fs.createWriteStream(metadata.filePath); - - return fileWriteStream; - }; - - downloadFile = async (user: UserInterface, fileID: string, res: Response) => { - const currentFile = await dbUtilsFile.getFileInfo( - fileID, - user._id.toString() - ); - console.log(fileID, user._id); - if (!currentFile) throw new NotFoundError("Download File Not Found"); - - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const filePath = currentFile.metadata.filePath!; - - // TODO: I believe this has to do with using conn.db instead of the mongoose driver - // We should switch to the mongoose driver - let IV = currentFile.metadata.IV; - if (!(IV instanceof Buffer)) { - IV = IV.buffer; - } - - const readStream = fs.createReadStream(filePath); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - - res.set("Content-Type", "binary/octet-stream"); - res.set( - "Content-Disposition", - 'attachment; filename="' + currentFile.filename + '"' - ); - res.set("Content-Length", currentFile.metadata.size.toString()); - - const allStreamsToErrorCatch = [readStream, decipher]; - - await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch); - }; - - getFileReadStream = async (user: UserInterface, fileID: string) => { - const currentFile = await dbUtilsFile.getFileInfo( - fileID, - user._id.toString() - ); - - if (!currentFile) throw new NotFoundError("Download File Not Found"); - - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const filePath = currentFile.metadata.filePath!; - - const IV = currentFile.metadata.IV.buffer as Buffer; - - const readStream = fs.createReadStream(filePath); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - - return readStream; - }; - - getThumbnail = async (user: UserInterface, id: string) => { - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const thumbnail = (await Thumbnail.findById( - new ObjectId(id) - )) as ThumbnailInterface; - - if (thumbnail.owner !== user._id.toString()) { - throw new ForbiddenError("Thumbnail Unauthorized Error"); - } - - // TODO: Fix this type - let iv = thumbnail.IV as any; - if (!(iv instanceof Buffer)) { - iv = iv.buffer; - } - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv); - - const readStream = fs.createReadStream(thumbnail.path!); - - const allStreamsToErrorCatch = [readStream, decipher]; - - const bufferData = await streamToBuffer( - readStream.pipe(decipher), - allStreamsToErrorCatch - ); - - return bufferData; - }; - - getFullThumbnail = async ( - user: UserInterface, - fileID: string, - res: Response - ) => { - const userID = user._id; - - const file = await dbUtilsFile.getFileInfo(fileID, userID.toString()); - - if (!file) throw new NotFoundError("File Thumbnail Not Found"); - - const password = user.getEncryptionKey(); - const IV = file.metadata.IV; - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const readStream = fs.createReadStream(file.metadata.filePath!); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - - res.set("Content-Type", "binary/octet-stream"); - res.set( - "Content-Disposition", - 'attachment; filename="' + file.filename + '"' - ); - res.set("Content-Length", file.metadata.size.toString()); - - const allStreamsToErrorCatch = [readStream, decipher]; - - await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch); - }; - - streamVideo = async ( - user: UserInterface, - fileID: string, - headers: any, - res: Response, - req: Request - ) => { - // To get this all working correctly with encryption and across - // All browsers took many days, tears, and some of my sanity. - // Shoutout to Tyzoid for helping me with the decryption - // And and helping me understand how the IVs work. - - // P.S I hate safari >:( - // Why do yall have to be weird with video streaming - // 90% of the issues with this are only in Safari - // Is safari going to be the next internet explorer? - - const userID = user._id; - const currentFile = await dbUtilsFile.getFileInfo( - fileID, - userID.toString() - ); - - if (!currentFile) throw new NotFoundError("Video File Not Found"); - - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const fileSize = currentFile.metadata.size; - - const range = headers.range; - const parts = range.replace(/bytes=/, "").split("-"); - let start = parseInt(parts[0], 10); - let end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1; - const IV = currentFile.metadata.IV.buffer as Buffer; - const chunksize = end - start + 1; - - let head = { - "Content-Range": "bytes " + start + "-" + end + "/" + fileSize, - "Accept-Ranges": "bytes", - "Content-Length": chunksize, - "Content-Type": "video/mp4", - }; - - let currentIV = IV; - - let fixedStart = 0; - let fixedEnd = currentFile.length; - - if (start === 0 && end === 1) { - // This is for Safari/iOS, Safari will request the first - // Byte before actually playing the video. Needs to be - // 16 bytes. - - fixedStart = 0; - fixedEnd = 15; - } else { - // If you're a normal browser, or this isn't Safari's first request - // We need to make it so start is divisible by 16, since AES256 - // Has a block size of 16 bytes. - - fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start); - } - - if (+start === 0) { - // This math will not work if the start is 0 - // So if it is we just change fixed start back - // To 0. - - fixedStart = 0; - } - - // We also need to calculate the difference between the start and the - // Fixed start position. Since there will be an offset if the original - // Request is not divisible by 16, it will not return the right part - // Of the file, you will see how we do this in the awaitStreamVideo - // code. - - const differenceStart = start - fixedStart; - - if (fixedStart !== 0 && start !== 0) { - // If this isn't the first request, the way AES256 works is when you try to - // Decrypt a certain part of the file that isn't the start, the IV will - // Actually be the 16 bytes ahead of where you are trying to - // Start the decryption. - - currentIV = (await getPrevIVFS( - fixedStart - 16, - currentFile.metadata.filePath! - )) as Buffer; - } - - const readStream = fs.createReadStream(currentFile.metadata.filePath!, { - start: fixedStart, - end: fixedEnd, - }); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, currentIV); - - decipher.setAutoPadding(false); - - res.writeHead(206, head); - - const allStreamsToErrorCatch = [readStream, decipher]; - - readStream.pipe(decipher); - - await awaitStreamVideo( - start, - end, - differenceStart, - decipher, - res, - req, - allStreamsToErrorCatch, - readStream - ); - - readStream.destroy(); - }; - - getPublicDownload = async (fileID: string, tempToken: any, res: Response) => { - const file: FileInterface = await dbUtilsFile.getPublicFile(fileID); - - if (!file || !file.metadata.link || file.metadata.link !== tempToken) { - throw new NotAuthorizedError("File Not Public"); - } - - const user = (await User.findById(file.metadata.owner)) as UserInterface; - - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const IV = file.metadata.IV.buffer as Buffer; - - const readStream = fs.createReadStream(file.metadata.filePath!); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - - res.set("Content-Type", "binary/octet-stream"); - res.set( - "Content-Disposition", - 'attachment; filename="' + file.filename + '"' - ); - res.set("Content-Length", file.metadata.size.toString()); - - const allStreamsToErrorCatch = [readStream, decipher]; - - await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch); - - if (file.metadata.linkType === "one") { - await dbUtilsFile.removeOneTimePublicLink(fileID); - } - }; - - trashMulti = async ( - userID: string, - items: { - type: "file" | "folder" | "quick-item"; - id: string; - file?: FileInterface; - folder?: FolderInterface; - }[] - ) => { - const fileList = items.filter( - (item) => item.type === "file" || item.type === "quick-item" - ); - const folderList = items - .filter((item) => item.type === "folder") - .sort((a, b) => { - if (!a.folder || !b.folder) return 0; - return b.folder.parentList.length - a.folder.parentList.length; - }); - - for (const file of fileList) { - await fileService.trashFile(userID, file.id); - } - for (const folder of folderList) { - await folderService.trashFolder(userID, folder.id); - } - }; - - restoreMulti = async ( - userID: string, - items: { - type: "file" | "folder" | "quick-item"; - id: string; - file?: FileInterface; - folder?: FolderInterface; - }[] - ) => { - const fileList = items.filter( - (item) => item.type === "file" || item.type === "quick-item" - ); - const folderList = items - .filter((item) => item.type === "folder") - .sort((a, b) => { - if (!a.folder || !b.folder) return 0; - return b.folder.parentList.length - a.folder.parentList.length; - }); - - for (const file of fileList) { - await fileService.restoreFile(userID, file.id); - } - for (const folder of folderList) { - await folderService.restoreFolder(userID, folder.id); - } - }; - - deleteMulti = async ( - userID: string, - items: { - type: "file" | "folder" | "quick-item"; - id: string; - file?: FileInterface; - folder?: FolderInterface; - }[] - ) => { - const fileList = items.filter( - (item) => item.type === "file" || item.type === "quick-item" - ); - const folderList = items - .filter((item) => item.type === "folder") - .sort((a, b) => { - if (!a.folder || !b.folder) return 0; - return b.folder.parentList.length - a.folder.parentList.length; - }); - - for (const file of fileList) { - await this.deleteFile(userID, file.id); - } - for (const folder of folderList) { - await this.deleteFolder(userID, folder.id); - } - }; - - deleteFile = async (userID: string, fileID: string) => { - const file = await dbUtilsFile.getFileInfo(fileID, userID); - - if (!file) throw new NotFoundError("Delete File Not Found Error"); - - if (file.metadata.thumbnailID) { - const thumbnail = (await Thumbnail.findById( - file.metadata.thumbnailID - )) as ThumbnailInterface; - const thumbnailPath = thumbnail.path!; - await removeChunksFS(thumbnailPath); - - await Thumbnail.deleteOne({ _id: file.metadata.thumbnailID }); - } - - await removeChunksFS(file.metadata.filePath!); - await File.deleteOne({ _id: file._id }); - await subtractFromStorageSize( - userID, - file.length, - file.metadata.personalFile! - ); - }; - - deleteFolder = async (userID: string, folderID: string) => { - const folder = await dbUtilsFolder.getFolderInfo(folderID, userID); - - if (!folder) throw new NotFoundError("Delete Folder Not Found Error"); - - const parentList = [...folder.parentList, folder._id]; - - await Folder.deleteMany({ - owner: userID, - parentList: { $all: parentList }, - }); - await Folder.deleteMany({ owner: userID, _id: folderID }); - - const fileList = await dbUtilsFile.getFileListByParent( - userID, - parentList.toString() - ); - - if (!fileList) throw new NotFoundError("Delete File List Not Found"); - - for (let i = 0; i < fileList.length; i++) { - const currentFile = fileList[i]; - - try { - if (currentFile.metadata.thumbnailID) { - const thumbnail = (await Thumbnail.findById( - currentFile.metadata.thumbnailID - )) as ThumbnailInterface; - const thumbnailPath = thumbnail.path!; - await removeChunksFS(thumbnailPath); - - await Thumbnail.deleteOne({ _id: currentFile.metadata.thumbnailID }); - } - - await removeChunksFS(currentFile.metadata.filePath!); - await File.deleteOne({ _id: currentFile._id }); - } catch (e) { - console.log( - "Could not delete file", - currentFile.filename, - currentFile._id - ); - } - } - }; - - deleteAll = async (userID: string) => { - await Folder.deleteMany({ owner: userID }); - - const fileList = await dbUtilsFile.getFileListByOwner(userID); - - if (!fileList) - throw new NotFoundError("Delete All File List Not Found Error"); - - for (let i = 0; i < fileList.length; i++) { - const currentFile = fileList[i]; - - try { - if (currentFile.metadata.thumbnailID) { - const thumbnail = (await Thumbnail.findById( - currentFile.metadata.thumbnailID - )) as ThumbnailInterface; - const thumbnailPath = thumbnail.path!; - await removeChunksFS(thumbnailPath); - - await Thumbnail.deleteOne({ _id: currentFile.metadata.thumbnailID }); - } - - await removeChunksFS(currentFile.metadata.filePath!); - await File.deleteOne({ _id: currentFile._id }); - } catch (e) { - console.log( - "Could Not Remove File", - currentFile.filename, - currentFile._id - ); - } - } - }; -} - -export default FileSystemService; diff --git a/backend/services/ChunkService/S3Service.ts b/backend/services/ChunkService/S3Service.ts deleted file mode 100644 index e3773a3..0000000 --- a/backend/services/ChunkService/S3Service.ts +++ /dev/null @@ -1,639 +0,0 @@ -import File, { FileInterface } from "../../models/file"; -import User, { UserInterface } from "../../models/user"; -import s3 from "../../db/s3"; -import env from "../../enviroment/env"; -import { Response, Request } from "express"; -import ChunkInterface from "./utils/ChunkInterface"; -import NotAuthorizedError from "../../utils/NotAuthorizedError"; -import NotFoundError from "../../utils/NotFoundError"; -import crypto from "crypto"; -import getBusboyData from "./utils/getBusboyData"; -import videoChecker from "../../utils/videoChecker"; -import uuid from "uuid"; -import awaitUploadStreamS3 from "./utils/awaitUploadStreamS3"; -import awaitStream from "./utils/awaitStream"; -import createThumbnailAny from "./utils/createThumbailAny"; -import imageChecker from "../../utils/imageChecker"; -import Thumbnail, { ThumbnailInterface } from "../../models/thumbnail"; -import streamToBuffer from "../../utils/streamToBuffer"; -import removeChunksS3 from "./utils/removeChunksS3"; -import fixStartChunkLength from "./utils/fixStartChunkLength"; -import fixEndChunkLength from "./utils/fixEndChunkLength"; -import getPrevIVS3 from "./utils/getPrevIVS3"; -import awaitStreamVideo from "./utils/awaitStreamVideo"; -import Folder, { FolderInterface } from "../../models/folder"; -import DbUtilFile from "../../db/utils/fileUtils/index"; -import addToStoageSize from "./utils/addToStorageSize"; -import subtractFromStorageSize from "./utils/subtractFromStorageSize"; -import ForbiddenError from "../../utils/ForbiddenError"; -import { ObjectId } from "mongodb"; - -const dbUtilsFile = new DbUtilFile(); - -class S3Service implements ChunkInterface { - constructor() {} - - uploadFile = async (user: UserInterface, busboy: any, req: Request) => { - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const initVect = crypto.randomBytes(16); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect); - - const { file, filename: fileInfo, formData } = await getBusboyData(busboy); - const filename = fileInfo.filename; - - const parent = formData.get("parent") || "/"; - const parentList = formData.get("parentList") || "/"; - const size = formData.get("size") || ""; - const personalFile = formData.get("personal-file") ? true : false; - let hasThumbnail = false; - let thumbnailID = ""; - const isVideo = videoChecker(filename); - - const randomS3ID = uuid.v4(); - - const s3Data: any = personalFile ? await user.decryptS3Data() : {}; - const bucketName = personalFile ? s3Data.bucket : env.s3Bucket; - - let metadata: any = { - owner: user._id, - parent, - parentList, - hasThumbnail, - thumbnailID, - isVideo, - size, - IV: initVect, - s3ID: randomS3ID, - }; - - if (personalFile) metadata = { ...metadata, personalFile: true }; - - const params = { - Bucket: bucketName, - Body: file.pipe(cipher), - Key: randomS3ID, - }; - - await awaitUploadStreamS3(params, personalFile, s3Data); - - const date = new Date(); - const encryptedFileSize = size; - - const currentFile = new File({ - filename, - uploadDate: date.toISOString(), - length: encryptedFileSize, - metadata, - }); - - await currentFile.save(); - - await addToStoageSize(user, size, personalFile); - - const imageCheck = imageChecker(currentFile.filename); - - if (currentFile.length < 15728640 && imageCheck) { - const updatedFile = await createThumbnailAny(currentFile, filename, user); - - return updatedFile; - } else { - return currentFile; - } - }; - - getFileWriteStream = async ( - user: UserInterface, - file: FileInterface, - parentFolder: FolderInterface, - readStream: any - ) => { - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const initVect = crypto.randomBytes(16); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect); - - const filename = file.filename; - const parent = parentFolder._id; - const parentList = [...parentFolder.parentList, parentFolder._id]; - const size = file.metadata.size; - const personalFile = file.metadata.personalFile ? true : false; - let hasThumbnail = file.metadata.hasThumbnail; - let thumbnailID = file.metadata.thumbnailID; - const isVideo = file.metadata.isVideo; - - const metadata = { - owner: user._id, - parent, - parentList, - hasThumbnail, - thumbnailID, - isVideo, - size, - IV: file.metadata.IV, - s3ID: file.metadata.s3ID, - personalFile, - }; - - const s3Data: any = personalFile ? await user.decryptS3Data() : {}; - const bucketName = personalFile ? s3Data.bucket : env.s3Bucket; - - const params = { - Bucket: bucketName, - Body: readStream, - Key: file.metadata.s3ID, - }; - }; - - getS3AuthThumbnail = async ( - thumbnail: ThumbnailInterface, - user: UserInterface - ) => { - return { s3Storage: s3, bucket: env.s3Bucket }; - }; - - getS3Auth = async (file: FileInterface, user: UserInterface) => { - return { s3Storage: s3, bucket: env.s3Bucket }; - }; - - downloadFile = async (user: UserInterface, fileID: string, res: Response) => { - const currentFile = await dbUtilsFile.getFileInfo( - fileID, - user._id.toString() - ); - - if (!currentFile) throw new NotFoundError("Download File Not Found"); - - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const { s3Storage, bucket } = await this.getS3Auth(currentFile, user); - - const IV = currentFile.metadata.IV.buffer as Buffer; - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - - res.set("Content-Type", "binary/octet-stream"); - res.set( - "Content-Disposition", - 'attachment; filename="' + currentFile.filename + '"' - ); - res.set("Content-Length", currentFile.metadata.size.toString()); - - const params: any = { Bucket: bucket, Key: currentFile.metadata.s3ID! }; - - const s3ReadStream = s3Storage.getObject(params).createReadStream(); - - const allStreamsToErrorCatch = [s3ReadStream, decipher]; - - await awaitStream(s3ReadStream.pipe(decipher), res, allStreamsToErrorCatch); - }; - - getFileReadStream = async (user: UserInterface, fileID: string) => { - const currentFile = await dbUtilsFile.getFileInfo( - fileID, - user._id.toString() - ); - - if (!currentFile) throw new NotFoundError("Download File Not Found"); - - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const { s3Storage, bucket } = await this.getS3Auth(currentFile, user); - - const IV = currentFile.metadata.IV.buffer as Buffer; - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - - const params: any = { Bucket: bucket, Key: currentFile.metadata.s3ID! }; - - const s3ReadStream = s3Storage.getObject(params).createReadStream(); - - return s3ReadStream; - }; - - streamVideo = async ( - user: UserInterface, - fileID: string, - headers: any, - res: Response, - req: Request - ) => { - // To get this all working correctly with encryption and across - // All browsers took many days, tears, and some of my sanity. - // Shoutout to Tyzoid for helping me with the decryption - // And and helping me understand how the IVs work. - - // P.S I hate safari >:( - // Why do yall have to be weird with video streaming - // 90% of the issues with this are only in Safari - // Is safari going to be the next internet explorer? - - const userID = user._id; - const currentFile = await dbUtilsFile.getFileInfo( - fileID, - userID.toString() - ); - - if (!currentFile) throw new NotFoundError("Video File Not Found"); - - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const fileSize = currentFile.metadata.size; - - const isPersonal = currentFile.metadata.personalFile!; - - const range = headers.range; - const parts = range.replace(/bytes=/, "").split("-"); - let start = parseInt(parts[0], 10); - let end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1; - const chunksize = end - start + 1; - const IV = currentFile.metadata.IV.buffer as Buffer; - - let head = { - "Content-Range": "bytes " + start + "-" + end + "/" + fileSize, - "Accept-Ranges": "bytes", - "Content-Length": chunksize, - "Content-Type": "video/mp4", - }; - - let currentIV = IV; - - let fixedStart = 0; - let fixedEnd = currentFile.length; - - if (start === 0 && end === 1) { - // This is for Safari/iOS, Safari will request the first - // Byte before actually playing the video. Needs to be - // 16 bytes. - - fixedStart = 0; - fixedEnd = 15; - } else { - // If you're a normal browser, or this isn't Safari's first request - // We need to make it so start is divisible by 16, since AES256 - // Has a block size of 16 bytes. - - fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start); - - // I goofed up and forgot to add the encrypted file size to S3 data, - // It just used the normal file size :( - // So you'll notice only on the S3 route do i need to fix the end length - // To, this is because the other 2 routes use the encrypted file size - // Which is always a multiple of 16. I cannot change this now - // Since previous versions will still have the issue, but - // This is a simple fix luckily. - - fixedEnd = fixedEnd % 16 === 0 ? fixedEnd : fixEndChunkLength(fixedEnd); - } - - if (+start === 0) { - // This math will not work if the start is 0 - // So if it is we just change fixed start back - // To 0. - - fixedStart = 0; - } - - // We also need to calculate the difference between the start and the - // Fixed start position. Since there will be an offset if the original - // Request is not divisible by 16, it will not return the right part - // Of the file, you will see how we do this in the awaitStreamVideo - // code. - - const differenceStart = start - fixedStart; - - if (fixedStart !== 0 && start !== 0) { - // If this isn't the first request, the way AES256 works is when you try to - // Decrypt a certain part of the file that isn't the start, the IV will - // Actually be the 16 bytes ahead of where you are trying to - // Start the decryption. - - currentIV = (await getPrevIVS3( - fixedStart - 16, - currentFile.metadata.s3ID!, - isPersonal, - user - )) as Buffer; - } - - const { s3Storage, bucket } = await this.getS3Auth(currentFile, user); - - const params: any = { - Bucket: bucket, - Key: currentFile.metadata.s3ID!, - Range: `bytes=${fixedStart}-${fixedEnd}`, - }; - - const s3ReadStream = s3Storage.getObject(params).createReadStream(); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, currentIV); - - decipher.setAutoPadding(false); - - res.writeHead(206, head); - - const allStreamsToErrorCatch = [s3ReadStream, decipher]; - - s3ReadStream.pipe(decipher); - - await awaitStreamVideo( - start, - end, - differenceStart, - decipher, - res, - req, - allStreamsToErrorCatch, - s3ReadStream - ); - - s3ReadStream.destroy(); - }; - - getThumbnail = async (user: UserInterface, id: string) => { - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const thumbnail = (await Thumbnail.findById( - new ObjectId(id) - )) as ThumbnailInterface; - - if (thumbnail.owner !== user._id.toString()) { - throw new ForbiddenError("Thumbnail Unauthorized Error"); - } - - const iv = thumbnail.IV!; - - const { s3Storage, bucket } = await this.getS3AuthThumbnail( - thumbnail, - user - ); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv); - - const params: any = { Bucket: bucket, Key: thumbnail.s3ID! }; - - const readStream = s3Storage.getObject(params).createReadStream(); - - const allStreamsToErrorCatch = [readStream, decipher]; - - const bufferData = await streamToBuffer( - readStream.pipe(decipher), - allStreamsToErrorCatch - ); - - return bufferData; - }; - - getFullThumbnail = async ( - user: UserInterface, - fileID: string, - res: Response - ) => { - const userID = user._id; - - const file = await dbUtilsFile.getFileInfo(fileID, userID.toString()); - - if (!file) throw new NotFoundError("File Thumbnail Not Found"); - - const password = user.getEncryptionKey(); - const IV = file.metadata.IV.buffer as Buffer; - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const { s3Storage, bucket } = await this.getS3Auth(file, user); - - const params: any = { Bucket: bucket, Key: file.metadata.s3ID! }; - - const readStream = s3Storage.getObject(params).createReadStream(); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - - res.set("Content-Type", "binary/octet-stream"); - res.set( - "Content-Disposition", - 'attachment; filename="' + file.filename + '"' - ); - res.set("Content-Length", file.metadata.size.toString()); - - const allStreamsToErrorCatch = [readStream, decipher]; - - await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch); - }; - - getPublicDownload = async (fileID: string, tempToken: any, res: Response) => { - const file: FileInterface = await dbUtilsFile.getPublicFile(fileID); - - if (!file || !file.metadata.link || file.metadata.link !== tempToken) { - throw new NotAuthorizedError("File Not Public"); - } - - const user = (await User.findById(file.metadata.owner)) as UserInterface; - - const password = user.getEncryptionKey(); - - if (!password) throw new ForbiddenError("Invalid Encryption Key"); - - const IV = file.metadata.IV.buffer as Buffer; - - const { s3Storage, bucket } = await this.getS3Auth(file, user); - - const params: any = { Bucket: bucket, Key: file.metadata.s3ID! }; - - const readStream = s3Storage.getObject(params).createReadStream(); - - const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); - - const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - - res.set("Content-Type", "binary/octet-stream"); - res.set( - "Content-Disposition", - 'attachment; filename="' + file.filename + '"' - ); - res.set("Content-Length", file.metadata.size.toString()); - - const allStreamsToErrorCatch = [readStream, decipher]; - - await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch); - - if (file.metadata.linkType === "one") { - await dbUtilsFile.removeOneTimePublicLink(fileID); - } - }; - - deleteFile = async (userID: string, fileID: string) => { - const file = await dbUtilsFile.getFileInfo(fileID, userID); - - if (!file) throw new NotFoundError("Delete File Not Found Error"); - - const user = (await User.findById(userID)) as UserInterface; - - const { s3Storage, bucket } = await this.getS3Auth(file, user); - - if (file.metadata.thumbnailID) { - const thumbnail = (await Thumbnail.findById( - file.metadata.thumbnailID - )) as ThumbnailInterface; - const paramsThumbnail: any = { Bucket: bucket, Key: thumbnail.s3ID! }; - await removeChunksS3(s3Storage, paramsThumbnail); - await Thumbnail.deleteOne({ _id: file.metadata.thumbnailID }); - } - - const params: any = { Bucket: bucket, Key: file.metadata.s3ID! }; - await removeChunksS3(s3Storage, params); - await File.deleteOne({ _id: file._id }); - await subtractFromStorageSize( - userID, - file.length, - file.metadata.personalFile! - ); - }; - - deleteFolder = async ( - userID: string, - folderID: string, - parentList: string[] - ) => { - const parentListString = parentList.toString(); - - await Folder.deleteMany({ - owner: userID, - parentList: { $all: parentList }, - }); - await Folder.deleteMany({ owner: userID, _id: folderID }); - - const fileList = await dbUtilsFile.getFileListByParent( - userID, - parentListString - ); - - if (!fileList) throw new NotFoundError("Delete File List Not Found"); - - const user = (await User.findById(userID)) as UserInterface; - - for (let i = 0; i < fileList.length; i++) { - const currentFile = fileList[i]; - - const { s3Storage, bucket } = await this.getS3Auth(currentFile, user); - - try { - if (currentFile.metadata.thumbnailID) { - const thumbnail = (await Thumbnail.findById( - currentFile.metadata.thumbnailID - )) as ThumbnailInterface; - const paramsThumbnail: any = { Bucket: bucket, Key: thumbnail.s3ID! }; - await removeChunksS3(s3Storage, paramsThumbnail); - await Thumbnail.deleteOne({ _id: currentFile.metadata.thumbnailID }); - } - - const params: any = { Bucket: bucket, Key: currentFile.metadata.s3ID! }; - await removeChunksS3(s3Storage, params); - await File.deleteOne({ _id: currentFile._id }); - } catch (e) { - console.log( - "Could not delete file", - currentFile.filename, - currentFile._id - ); - } - } - }; - - deleteAll = async (userID: string) => { - await Folder.deleteMany({ owner: userID }); - - const fileList = await dbUtilsFile.getFileListByOwner(userID); - - if (!fileList) - throw new NotFoundError("Delete All File List Not Found Error"); - - const user = (await User.findById(userID)) as UserInterface; - - for (let i = 0; i < fileList.length; i++) { - const currentFile = fileList[i]; - - const { s3Storage, bucket } = await this.getS3Auth(currentFile, user); - - try { - if (currentFile.metadata.thumbnailID) { - const thumbnail = (await Thumbnail.findById( - currentFile.metadata.thumbnailID - )) as ThumbnailInterface; - const paramsThumbnail: any = { Bucket: bucket, Key: thumbnail.s3ID! }; - await removeChunksS3(s3Storage, paramsThumbnail); - await Thumbnail.deleteOne({ _id: currentFile.metadata.thumbnailID }); - } - - const params: any = { Bucket: bucket, Key: currentFile.metadata.s3ID! }; - await removeChunksS3(s3Storage, params); - await File.deleteOne({ _id: currentFile._id }); - } catch (e) { - console.log( - "Could Not Remove File", - currentFile.filename, - currentFile._id - ); - } - } - }; - - trashMulti = async ( - userID: string, - items: { - type: "file" | "folder" | "quick-item"; - id: string; - file?: FileInterface; - folder?: FolderInterface; - }[] - ) => {}; - - restoreMulti = async ( - userID: string, - items: { - type: "file" | "folder" | "quick-item"; - id: string; - file?: FileInterface; - folder?: FolderInterface; - }[] - ) => {}; - - deleteMulti = async ( - userID: string, - items: { - type: "file" | "folder" | "quick-item"; - id: string; - file?: FileInterface; - folder?: FolderInterface; - }[] - ) => {}; -} - -export default S3Service; diff --git a/backend/services/ChunkService/index.ts b/backend/services/ChunkService/index.ts index e06ee50..1e8399e 100644 --- a/backend/services/ChunkService/index.ts +++ b/backend/services/ChunkService/index.ts @@ -16,8 +16,8 @@ import fixStartChunkLength from "./utils/fixStartChunkLength"; import Folder, { FolderInterface } from "../../models/folder"; import ForbiddenError from "../../utils/ForbiddenError"; import { ObjectId } from "mongodb"; -import { S3Actions } from "./S3Actions"; -import { FilesystemActions } from "./FileSystemActions"; +import { S3Actions } from "./Actions/S3Actions"; +import { FilesystemActions } from "./Actions/FileSystemActions"; import { createGenericParams } from "./utils/storageHelper"; const dbUtilsFile = new DbUtilFile(); diff --git a/backend/services/ChunkService/utils/createVideoThumbnail.ts b/backend/services/ChunkService/utils/createVideoThumbnail.ts index 90a2d6f..d448066 100644 --- a/backend/services/ChunkService/utils/createVideoThumbnail.ts +++ b/backend/services/ChunkService/utils/createVideoThumbnail.ts @@ -11,8 +11,8 @@ import { ObjectId } from "mongodb"; import File from "../../../models/file"; import ffmpeg from "fluent-ffmpeg"; import tempCreateVideoThumbnailFS from "./tempCreateVideoThumbnailFS"; -import { S3Actions } from "../S3Actions"; -import { FilesystemActions } from "../FileSystemActions"; +import { S3Actions } from "../Actions/S3Actions"; +import { FilesystemActions } from "../Actions/FileSystemActions"; import { createGenericParams } from "./storageHelper"; const storageActions = diff --git a/backend/services/ChunkService/utils/tempCreateVideoThumbnail.ts b/backend/services/ChunkService/utils/tempCreateVideoThumbnail.ts index 97410c1..68324b6 100644 --- a/backend/services/ChunkService/utils/tempCreateVideoThumbnail.ts +++ b/backend/services/ChunkService/utils/tempCreateVideoThumbnail.ts @@ -10,8 +10,8 @@ import env from "../../../enviroment/env"; import { ObjectId } from "mongodb"; import File from "../../../models/file"; import ffmpeg from "fluent-ffmpeg"; -import { S3Actions } from "../S3Actions"; -import { FilesystemActions } from "../FileSystemActions"; +import { S3Actions } from "../Actions/S3Actions"; +import { FilesystemActions } from "../Actions/FileSystemActions"; import { createGenericParams } from "./storageHelper"; const storageActions =