From 45e4b2682978c1acaadb846fe861f09b0bc9f8a7 Mon Sep 17 00:00:00 2001 From: subnub Date: Fri, 5 Jul 2024 14:07:58 -0400 Subject: [PATCH] finished BE service injection :) --- backend/controllers/file.ts | 35 +++- backend/enviroment/env.ts | 90 +++++----- backend/services/ChunkService/index.ts | 25 --- .../utils/createVideoThumbnail.ts | 154 ++++++++++++++++ .../utils/tempCreateVideoThumbnail.ts | 164 ++++++++++++++++++ 5 files changed, 398 insertions(+), 70 deletions(-) create mode 100644 backend/services/ChunkService/utils/createVideoThumbnail.ts create mode 100644 backend/services/ChunkService/utils/tempCreateVideoThumbnail.ts diff --git a/backend/controllers/file.ts b/backend/controllers/file.ts index cc9bb37..da247fa 100644 --- a/backend/controllers/file.ts +++ b/backend/controllers/file.ts @@ -13,8 +13,16 @@ import streamToBuffer from "../utils/streamToBuffer"; import env from "../enviroment/env"; import getFileSize from "../services/ChunkService/utils/getFileSize"; import File, { FileMetadateInterface } from "../models/file"; +import imageChecker from "../utils/imageChecker"; +import videoChecker from "../utils/videoChecker"; +import { S3Actions } from "../services/ChunkService/S3Actions"; +import { FilesystemActions } from "../services/ChunkService/FileSystemActions"; +import createVideoThumbnail from "../services/ChunkService/utils/createVideoThumbnail"; +import NotAuthorizedError from "../utils/NotAuthorizedError"; const fileService = new FileService(); +const storageActions = + env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); type userAccessType = { _id: string; @@ -148,6 +156,10 @@ class FileController { metadata: FileMetadateInterface ) => { try { + const user = req.user; + + if (!user) throw new NotAuthorizedError("User Not Authorized"); + const date = new Date(); let length = 0; @@ -168,7 +180,28 @@ class FileController { await currentFile.save(); - res.send(currentFile); + const imageCheck = imageChecker(currentFile.filename); + const videoCheck = videoChecker(currentFile.filename); + + if (videoCheck) { + console.log("is vidoe"); + const updatedFile = await createVideoThumbnail( + currentFile, + filename, + user + ); + + res.send(updatedFile); + } else if (currentFile.length < 15728640 && imageCheck) { + // const updatedFile = await createThumbnailAny( + // currentFile, + // filename, + // user + // ); + // return updatedFile; + } else { + res.send(currentFile); + } } catch (e: unknown) { if (!responseSent) { res.writeHead(500, { Connection: "close" }); diff --git a/backend/enviroment/env.ts b/backend/enviroment/env.ts index 01acd34..78c6bd2 100644 --- a/backend/enviroment/env.ts +++ b/backend/enviroment/env.ts @@ -1,47 +1,49 @@ export default { - key: process.env.KEY, - newKey: process.env.NEW_KEY, - passwordAccess: process.env.PASSWORD_ACCESS, - passwordRefresh: process.env.PASSWORD_REFRESH, - passwordCookie: process.env.PASSWORD_COOKIE, - createAcctBlocked: process.env.BLOCK_CREATE_ACCOUNT, - root: process.env.ROOT, - url: process.env.URL, - mongoURL: process.env.MONGODB_URL, - dbType: process.env.DB_TYPE, - fsDirectory: process.env.FS_DIRECTORY, - s3ID: process.env.S3_ID, - s3Key: process.env.S3_KEY, - s3Bucket: process.env.S3_BUCKET, - useDocumentDB: process.env.USE_DOCUMENT_DB, - documentDBBundle: process.env.DOCUMENT_DB_BUNDLE, - sendgridKey: process.env.SENDGRID_KEY, - sendgridEmail: process.env.SENDGRID_EMAIL, - remoteURL: process.env.REMOTE_URL, - disableEmailVerification: process.env.DISABLE_EMAIL_VERIFICATION, - secureCookies: process.env.SECURE_COOKIES -} + key: process.env.KEY, + newKey: process.env.NEW_KEY, + passwordAccess: process.env.PASSWORD_ACCESS, + passwordRefresh: process.env.PASSWORD_REFRESH, + passwordCookie: process.env.PASSWORD_COOKIE, + createAcctBlocked: process.env.BLOCK_CREATE_ACCOUNT, + root: process.env.ROOT, + url: process.env.URL, + mongoURL: process.env.MONGODB_URL, + dbType: process.env.DB_TYPE, + fsDirectory: process.env.FS_DIRECTORY, + s3ID: process.env.S3_ID, + s3Key: process.env.S3_KEY, + s3Bucket: process.env.S3_BUCKET, + useDocumentDB: process.env.USE_DOCUMENT_DB, + documentDBBundle: process.env.DOCUMENT_DB_BUNDLE, + sendgridKey: process.env.SENDGRID_KEY, + sendgridEmail: process.env.SENDGRID_EMAIL, + remoteURL: process.env.REMOTE_URL, + disableEmailVerification: process.env.DISABLE_EMAIL_VERIFICATION, + secureCookies: process.env.SECURE_COOKIES, + tempDirectory: process.env.TEMP_DIRECTORY, +}; module.exports = { - key: process.env.KEY, - newKey: process.env.NEW_KEY, - passwordAccess: process.env.PASSWORD_ACCESS, - passwordRefresh: process.env.PASSWORD_REFRESH, - passwordCookie: process.env.PASSWORD_COOKIE, - createAcctBlocked: process.env.BLOCK_CREATE_ACCOUNT, - root: process.env.ROOT, - url: process.env.URL, - mongoURL: process.env.MONGODB_URL, - dbType: process.env.DB_TYPE, - fsDirectory: process.env.FS_DIRECTORY, - s3ID: process.env.S3_ID, - s3Key: process.env.S3_KEY, - s3Bucket: process.env.S3_BUCKET, - useDocumentDB: process.env.USE_DOCUMENT_DB, - documentDBBundle: process.env.DOCUMENT_DB_BUNDLE, - sendgridKey: process.env.SENDGRID_KEY, - sendgridEmail: process.env.SENDGRID_EMAIL, - remoteURL: process.env.REMOTE_URL, - disableEmailVerification: process.env.DISABLE_EMAIL_VERIFICATION, - secureCookies: process.env.SECURE_COOKIES -} \ No newline at end of file + key: process.env.KEY, + newKey: process.env.NEW_KEY, + passwordAccess: process.env.PASSWORD_ACCESS, + passwordRefresh: process.env.PASSWORD_REFRESH, + passwordCookie: process.env.PASSWORD_COOKIE, + createAcctBlocked: process.env.BLOCK_CREATE_ACCOUNT, + root: process.env.ROOT, + url: process.env.URL, + mongoURL: process.env.MONGODB_URL, + dbType: process.env.DB_TYPE, + fsDirectory: process.env.FS_DIRECTORY, + s3ID: process.env.S3_ID, + s3Key: process.env.S3_KEY, + s3Bucket: process.env.S3_BUCKET, + useDocumentDB: process.env.USE_DOCUMENT_DB, + documentDBBundle: process.env.DOCUMENT_DB_BUNDLE, + sendgridKey: process.env.SENDGRID_KEY, + sendgridEmail: process.env.SENDGRID_EMAIL, + remoteURL: process.env.REMOTE_URL, + disableEmailVerification: process.env.DISABLE_EMAIL_VERIFICATION, + secureCookies: process.env.SECURE_COOKIES, + tempDirectory: process.env.TEMP_DIRECTORY, +}; diff --git a/backend/services/ChunkService/index.ts b/backend/services/ChunkService/index.ts index 080cdf9..e06ee50 100644 --- a/backend/services/ChunkService/index.ts +++ b/backend/services/ChunkService/index.ts @@ -1,37 +1,21 @@ import { Response, Request } from "express"; -import ChunkInterface from "./utils/ChunkInterface"; import { UserInterface } from "../../models/user"; import NotAuthorizedError from "../../utils/NotAuthorizedError"; import NotFoundError from "../../utils/NotFoundError"; import crypto from "crypto"; import getBusboyData from "./utils/getBusboyData"; import videoChecker from "../../utils/videoChecker"; -import fs from "fs"; import uuid from "uuid"; -import awaitUploadStreamFS from "./utils/awaitUploadStreamFS"; import File, { FileInterface } from "../../models/file"; -import getFileSize from "./utils/getFileSize"; import DbUtilFile from "../../db/utils/fileUtils/index"; import DbUtilFolder from "../../db/utils/folderUtils/index"; -import awaitStream from "./utils/awaitStream"; -import createThumbnailAny from "./utils/createThumbailAny"; -import imageChecker from "../../utils/imageChecker"; import Thumbnail, { ThumbnailInterface } from "../../models/thumbnail"; -import streamToBuffer from "../../utils/streamToBuffer"; import User from "../../models/user"; import env from "../../enviroment/env"; -import removeChunksFS from "./utils/removeChunksFS"; -import getPrevIVFS from "./utils/getPrevIVFS"; -import awaitStreamVideo from "./utils/awaitStreamVideo"; import fixStartChunkLength from "./utils/fixStartChunkLength"; import Folder, { FolderInterface } from "../../models/folder"; -import addToStoageSize from "./utils/addToStorageSize"; -import subtractFromStorageSize from "./utils/subtractFromStorageSize"; import ForbiddenError from "../../utils/ForbiddenError"; import { ObjectId } from "mongodb"; -import FolderService from "../FolderService"; -import MongoFileService from "../FileService"; -import createVideoThumbnailFS from "./utils/createVideoThumbnailFS"; import { S3Actions } from "./S3Actions"; import { FilesystemActions } from "./FileSystemActions"; import { createGenericParams } from "./utils/storageHelper"; @@ -110,7 +94,6 @@ class StorageService { }; }; - // INJECTED downloadFile = async (user: UserInterface, fileID: string, res: Response) => { const currentFile = await dbUtilsFile.getFileInfo( fileID, @@ -148,7 +131,6 @@ class StorageService { }; }; - // INJECTED getThumbnail = async (user: UserInterface, id: string) => { const password = user.getEncryptionKey(); @@ -185,7 +167,6 @@ class StorageService { }; }; - // INJECTED getFullThumbnail = async (user: UserInterface, fileID: string) => { const userID = user._id; @@ -221,7 +202,6 @@ class StorageService { }; }; - // INJECTED streamVideo = async (user: UserInterface, fileID: string, headers: any) => { const userID = user._id; const currentFile = await dbUtilsFile.getFileInfo( @@ -299,7 +279,6 @@ class StorageService { }; }; - // INJECTED getPublicDownload = async (fileID: string, tempToken: any, res: Response) => { const file: FileInterface = await dbUtilsFile.getPublicFile(fileID); @@ -340,7 +319,6 @@ class StorageService { }; }; - // INJECTED deleteMulti = async ( userID: string, items: { @@ -368,7 +346,6 @@ class StorageService { } }; - // INJECTED deleteFile = async (userID: string, fileID: string) => { const file = await dbUtilsFile.getFileInfo(fileID, userID); @@ -398,7 +375,6 @@ class StorageService { await File.deleteOne({ _id: file._id }); }; - // INJECTED deleteFolder = async (userID: string, folderID: string) => { const folder = await dbUtilsFolder.getFolderInfo(folderID, userID); @@ -455,7 +431,6 @@ class StorageService { } }; - // INJECTED deleteAll = async (userID: string) => { await Folder.deleteMany({ owner: userID }); diff --git a/backend/services/ChunkService/utils/createVideoThumbnail.ts b/backend/services/ChunkService/utils/createVideoThumbnail.ts new file mode 100644 index 0000000..90a2d6f --- /dev/null +++ b/backend/services/ChunkService/utils/createVideoThumbnail.ts @@ -0,0 +1,154 @@ +import mongoose from "../../../db/mongoose"; +import crypto from "crypto"; +import Thumbnail from "../../../models/thumbnail"; +import sharp from "sharp"; +import { FileInterface } from "../../../models/file"; +import { UserInterface } from "../../../models/user"; +import fs from "fs"; +import uuid from "uuid"; +import env from "../../../enviroment/env"; +import { ObjectId } from "mongodb"; +import File from "../../../models/file"; +import ffmpeg from "fluent-ffmpeg"; +import tempCreateVideoThumbnailFS from "./tempCreateVideoThumbnailFS"; +import { S3Actions } from "../S3Actions"; +import { FilesystemActions } from "../FileSystemActions"; +import { createGenericParams } from "./storageHelper"; + +const storageActions = + env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); + +const createVideoThumbnail = ( + file: FileInterface, + filename: string, + user: UserInterface +) => { + return new Promise((resolve, reject) => { + try { + const password = user.getEncryptionKey(); + + let CIPHER_KEY = crypto.createHash("sha256").update(password!).digest(); + + const thumbnailFilename = uuid.v4(); + + const readStreamParams = createGenericParams({ + filePath: file.metadata.filePath, + Key: file.metadata.s3ID, + }); + + const readStream = storageActions.createReadStream(readStreamParams); + + const writeStream = storageActions.createWriteStream( + readStreamParams, + readStream, + thumbnailFilename + ); + + const decipher = crypto.createDecipheriv( + "aes256", + CIPHER_KEY, + file.metadata.IV + ); + + const thumbnailIV = crypto.randomBytes(16); + + const thumbnailCipher = crypto.createCipheriv( + "aes256", + CIPHER_KEY, + thumbnailIV + ); + + const handleError = (e: Error) => { + console.log("thumbnail stream error", e); + resolve(file); + }; + + readStream.on("error", handleError); + + decipher.on("error", handleError); + + writeStream.on("error", handleError); + + thumbnailCipher.on("error", handleError); + + const decryptedReadStream = readStream.pipe(decipher); + + const handleFinish = async () => { + try { + const thumbnailModel = new Thumbnail({ + name: filename, + owner: user._id, + IV: thumbnailIV, + path: env.fsDirectory + thumbnailFilename, + }); + + await thumbnailModel.save(); + if (!file._id) { + return reject(); + } + const updateFileResponse = await File.updateOne( + { _id: new ObjectId(file._id), "metadata.owner": user._id }, + { + $set: { + "metadata.hasThumbnail": true, + "metadata.thumbnailID": thumbnailModel._id, + }, + } + ); + if (updateFileResponse.modifiedCount === 0) { + return reject(); + } + + const updatedFile = await File.findById({ + _id: new ObjectId(file._id), + "metadata.owner": user._id, + }); + + if (!updatedFile) return reject(); + + console.log("updated file", updatedFile, resolve); + + resolve(updatedFile?.toObject()); + } catch (e) { + console.log("thumbnail error", e); + resolve(file); + } + }; + + ffmpeg(decryptedReadStream, { + timeout: 60, + }) + .seek(1) + .format("image2pipe") + .outputOptions([ + "-f image2pipe", + "-vframes 1", + "-vf scale='if(gt(iw,ih),600,-1):if(gt(ih,iw),300,-1)'", + ]) + .on("start", (command) => {}) + .on("end", async () => { + await handleFinish(); + }) + .on("error", async (err, _, stderr) => { + console.log("thumbnail error attempting temp directory fix"); + if (env.tempDirectory) { + const updatedFile = await tempCreateVideoThumbnailFS( + file, + filename, + user + ); + resolve(updatedFile); + } else { + resolve(file); + } + }) + .pipe(thumbnailCipher) + .pipe(writeStream, { end: true }); + } catch (e) { + console.log("thumbnail error", e); + resolve(file); + } + }); +}; + +export default createVideoThumbnail; diff --git a/backend/services/ChunkService/utils/tempCreateVideoThumbnail.ts b/backend/services/ChunkService/utils/tempCreateVideoThumbnail.ts new file mode 100644 index 0000000..97410c1 --- /dev/null +++ b/backend/services/ChunkService/utils/tempCreateVideoThumbnail.ts @@ -0,0 +1,164 @@ +import mongoose from "../../../db/mongoose"; +import crypto from "crypto"; +import Thumbnail from "../../../models/thumbnail"; +import sharp from "sharp"; +import { FileInterface } from "../../../models/file"; +import { UserInterface } from "../../../models/user"; +import fs from "fs"; +import uuid from "uuid"; +import env from "../../../enviroment/env"; +import { ObjectId } from "mongodb"; +import File from "../../../models/file"; +import ffmpeg from "fluent-ffmpeg"; +import { S3Actions } from "../S3Actions"; +import { FilesystemActions } from "../FileSystemActions"; +import { createGenericParams } from "./storageHelper"; + +const storageActions = + env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); + +const tempCreateVideoThumbnail = ( + file: FileInterface, + filename: string, + user: UserInterface +) => { + return new Promise((resolve, reject) => { + const thumbnailFilename = uuid.v4(); + const tempDirectory = env.fsDirectory + "temp/" + thumbnailFilename; + const cleanup = () => { + fs.unlink(tempDirectory, (err) => { + if (err) console.error("CLEANUP ERROR:", err); + }); + }; + + try { + const password = user.getEncryptionKey(); + + let CIPHER_KEY = crypto.createHash("sha256").update(password!).digest(); + + const readStreamParams = createGenericParams({ + filePath: file.metadata.filePath, + Key: file.metadata.s3ID, + }); + + const readStream = storageActions.createReadStream(readStreamParams); + + const writeStream = storageActions.createWriteStream( + readStreamParams, + readStream, + thumbnailFilename + ); + + const tempWriteStream = fs.createWriteStream(tempDirectory); + const decipher = crypto.createDecipheriv( + "aes256", + CIPHER_KEY, + file.metadata.IV + ); + + const thumbnailIV = crypto.randomBytes(16); + + const thumbnailCipher = crypto.createCipheriv( + "aes256", + CIPHER_KEY, + thumbnailIV + ); + + const handleError = (e: Error) => { + console.log("thumbnail error", e); + cleanup(); + resolve(file); + }; + + const decryptedReadStream = readStream.pipe(decipher); + + readStream.on("error", handleError); + + decipher.on("error", handleError); + + writeStream.on("error", handleError); + + thumbnailCipher.on("error", handleError); + + decryptedReadStream.on("error", handleError); + + tempWriteStream.on("error", handleError); + + decryptedReadStream.pipe(tempWriteStream, { end: true }); + + const handleFinish = async () => { + try { + const thumbnailModel = new Thumbnail({ + name: filename, + owner: user._id, + IV: thumbnailIV, + path: env.fsDirectory + thumbnailFilename, + }); + + await thumbnailModel.save(); + if (!file._id) { + return reject(); + } + const updateFileResponse = await File.updateOne( + { _id: new ObjectId(file._id), "metadata.owner": user._id }, + { + $set: { + "metadata.hasThumbnail": true, + "metadata.thumbnailID": thumbnailModel._id, + }, + } + ); + if (updateFileResponse.modifiedCount === 0) { + return reject(); + } + + const updatedFile = await File.findById({ + _id: new ObjectId(file._id), + "metadata.owner": user._id, + }); + + if (!updatedFile) return reject(); + + cleanup(); + + resolve(updatedFile?.toObject()); + } catch (e) { + console.log("thumbnail error", e); + cleanup(); + resolve(file); + } + }; + + tempWriteStream.on("finish", () => { + ffmpeg(tempDirectory, { + timeout: 60, + }) + .seek(1) + .format("image2pipe") + .outputOptions([ + "-f image2pipe", + "-vframes 1", + "-vf scale='if(gt(iw,ih),600,-1):if(gt(ih,iw),300,-1)'", + ]) + .on("start", (command) => {}) + .on("end", async () => { + console.log("end"); + await handleFinish(); + }) + .on("error", (err, _, stderr) => { + console.log("error", err, stderr); + cleanup(); + resolve(file); + }) + .pipe(thumbnailCipher) + .pipe(writeStream, { end: true }); + }); + } catch (e) { + console.log("thumbnail error", e); + cleanup(); + resolve(file); + } + }); +}; + +export default tempCreateVideoThumbnail;