diff --git a/backend/controllers/file.ts b/backend/controllers/file.ts index 12fba0a..01ce340 100644 --- a/backend/controllers/file.ts +++ b/backend/controllers/file.ts @@ -10,6 +10,7 @@ import { } from "../cookies/createCookies"; import ChunkService from "../services/ChunkService"; import streamToBuffer from "../utils/streamToBuffer"; +import { read } from "fs"; const fileService = new FileService(); @@ -151,16 +152,50 @@ class FileController { }; getPublicDownload = async (req: RequestType, res: Response) => { + let responseSent = false; try { const ID = req.params.id; const tempToken = req.params.tempToken; - await this.chunkService.getPublicDownload(ID, tempToken, res); + const { readStream, decipher, file } = + await this.chunkService.getPublicDownload(ID, tempToken, res); + + readStream.on("error", (e: Error) => { + console.log("read stream error", e); + if (!responseSent) { + responseSent = true; + res.status(500).send("Server error downloading publicfile"); + } + }); + + decipher.on("error", (e: Error) => { + console.log("decipher stream error", e); + if (!responseSent) { + responseSent = true; + res.status(500).send("Server error downloading public file"); + } + }); + + res.set("Content-Type", "binary/octet-stream"); + res.set( + "Content-Disposition", + 'attachment; filename="' + file.filename + '"' + ); + res.set("Content-Length", file.metadata.size.toString()); + + readStream.pipe(decipher).pipe(res); + + // if (file.metadata.linkType === "one") { + // await dbUtilsFile.removeOneTimePublicLink(fileID); + // } } catch (e: unknown) { if (e instanceof Error) { console.log("\nGet Public Download Error File Route:", e.message); } - res.status(500).send("Server error downloading"); + if (!responseSent) { + responseSent = true; + res.status(500).send("Server error downloading public file"); + } } }; @@ -408,19 +443,70 @@ class FileController { if (!req.user) { return; } + let responseSent = false; try { const user = req.user; const fileID = req.params.id; const headers = req.headers; - await this.chunkService.streamVideo(user, fileID, headers, res, req); + const { decipher, readStream, head } = + await this.chunkService.streamVideo(user, fileID, headers); + + const cleanUp = () => { + if (readStream) readStream.destroy(); + if (decipher) decipher.end(); + }; + + const handleError = (e: Error) => { + console.log("stream video read stream error", e); + if (!responseSent) { + responseSent = true; + res.status(500).send("Server error streaming video"); + } + cleanUp(); + }; + + readStream.on("error", handleError); + + decipher.on("error", handleError); + + readStream.on("end", () => { + if (!responseSent) { + responseSent = true; + res.end(); + } + cleanUp(); + }); + + readStream.on("close", () => { + cleanUp(); + }); + + decipher.on("end", () => { + if (!responseSent) { + responseSent = true; + res.end(); + } + cleanUp(); + }); + + decipher.on("close", () => { + cleanUp(); + }); + + res.writeHead(206, head); + + readStream.pipe(decipher).pipe(res); } catch (e: unknown) { if (e instanceof Error) { console.log("\nStream Video Error File Route:", e.message); } - res.status(500).send("Server error streaming video"); + if (!responseSent) { + responseSent = true; + res.status(500).send("Server error streaming video"); + } } }; diff --git a/backend/services/ChunkService/FileSystemActions.ts b/backend/services/ChunkService/FileSystemActions.ts index c843700..e3c87d9 100644 --- a/backend/services/ChunkService/FileSystemActions.ts +++ b/backend/services/ChunkService/FileSystemActions.ts @@ -1,18 +1,26 @@ import fs from "fs"; import { UserInterface } from "../../models/user"; -import { AuthParams, IStorageActions } from "./StoreTypes"; +import { GenericParams, IStorageActions } from "./StoreTypes"; class FilesystemActions implements IStorageActions { async getAuth() { return {}; } - createReadStream(params: AuthParams): NodeJS.ReadableStream { + createReadStream(params: GenericParams): NodeJS.ReadableStream { if (!params.filePath) throw new Error("File path not configured"); const fsReadableStream = fs.createReadStream(params.filePath); return fsReadableStream; } - async removeChunks(params: AuthParams) { + createReadStreamWithRange(params: GenericParams, start: number, end: number) { + if (!params.filePath) throw new Error("File path not configured"); + const fsReadableStream = fs.createReadStream(params.filePath, { + start, + end, + }); + return fsReadableStream; + } + async removeChunks(params: GenericParams) { return new Promise((resolve, reject) => { if (!params.filePath) { reject("File path not configured"); @@ -28,6 +36,19 @@ class FilesystemActions implements IStorageActions { }); }); } + async getPrevIV(params: GenericParams, start: number) { + return new Promise((resolve, reject) => { + if (!params.filePath) throw new Error("File path not configured"); + const stream = fs.createReadStream(params.filePath, { + start, + end: start + 15, + }); + + stream.on("data", (data) => { + resolve(data); + }); + }); + } } export { FilesystemActions }; diff --git a/backend/services/ChunkService/S3Actions.ts b/backend/services/ChunkService/S3Actions.ts index 49e1d51..d32b2c9 100644 --- a/backend/services/ChunkService/S3Actions.ts +++ b/backend/services/ChunkService/S3Actions.ts @@ -1,7 +1,7 @@ import { UserInterface } from "../../models/user"; import s3 from "../../db/s3"; import env from "../../enviroment/env"; -import { AuthParams, IStorageActions } from "./StoreTypes"; +import { GenericParams, IStorageActions } from "./StoreTypes"; import internal from "stream"; class S3Actions implements IStorageActions { @@ -9,7 +9,7 @@ class S3Actions implements IStorageActions { return { s3Storage: s3, bucket: env.s3Bucket! }; } - createReadStream(params: AuthParams): internal.Readable { + createReadStream(params: GenericParams): internal.Readable { if (!params.Key) throw new Error("S3 not configured"); const { s3Storage, bucket } = this.getAuth(); const s3ReadableStream = s3Storage @@ -18,7 +18,21 @@ class S3Actions implements IStorageActions { return s3ReadableStream; } - removeChunks(params: AuthParams) { + createReadStreamWithRange( + params: GenericParams, + start: number, + end: number + ): internal.Readable { + if (!params.Key) throw new Error("S3 not configured"); + const range = `bytes=${start}-${end}`; + const { s3Storage, bucket } = this.getAuth(); + const s3ReadableStream = s3Storage + .getObject({ Key: params.Key, Bucket: bucket, Range: range }) + .createReadStream(); + return s3ReadableStream; + } + + removeChunks(params: GenericParams) { return new Promise((resolve, reject) => { if (!params.Key) { reject("S3 not configured"); @@ -34,6 +48,21 @@ class S3Actions implements IStorageActions { }); }); } + async getPrevIV(params: GenericParams, start: number) { + return new Promise((resolve, reject) => { + if (!params.Key) throw new Error("S3 not configured"); + const { s3Storage, bucket } = this.getAuth(); + const range = `bytes=${start}-${start + 15}`; + + const stream = s3Storage + .getObject({ Key: params.Key, Bucket: bucket, Range: range }) + .createReadStream(); + + stream.on("data", (data) => { + resolve(data); + }); + }); + } } export { S3Actions }; diff --git a/backend/services/ChunkService/StoreTypes.ts b/backend/services/ChunkService/StoreTypes.ts index f90abd3..1bd756d 100644 --- a/backend/services/ChunkService/StoreTypes.ts +++ b/backend/services/ChunkService/StoreTypes.ts @@ -1,6 +1,6 @@ import internal from "stream"; -export interface AuthParams { +export interface GenericParams { Key?: string; Bucket?: string; filePath?: string; @@ -10,7 +10,7 @@ export interface AuthParams { export interface IStorageActions { getAuth(): Object; createReadStream( - params: AuthParams + params: GenericParams ): NodeJS.ReadableStream | internal.Readable; - removeChunks(params: AuthParams): Promise; + removeChunks(params: GenericParams): Promise; } diff --git a/backend/services/ChunkService/index.ts b/backend/services/ChunkService/index.ts index dcb5461..ad14e88 100644 --- a/backend/services/ChunkService/index.ts +++ b/backend/services/ChunkService/index.ts @@ -46,7 +46,6 @@ class StorageService { constructor() {} uploadFile = async (user: UserInterface, busboy: any, req: Request) => { - console.log("upeload file2"); const password = user.getEncryptionKey(); if (!password) throw new ForbiddenError("Invalid Encryption Key"); @@ -60,14 +59,12 @@ class StorageService { const { file, filename: fileInfo, formData } = await getBusboyData(busboy); const filename = fileInfo.filename; - console.log("1"); const parent = formData.get("parent") || "/"; const size = formData.get("size") || ""; const personalFile = formData.get("personal-file") ? true : false; let hasThumbnail = false; let thumbnailID = ""; const isVideo = videoChecker(filename); - console.log("2", parent, typeof parent); const parentList = []; @@ -81,8 +78,6 @@ class StorageService { parentList.push("/"); } - console.log("parent list", parentList); - const systemFileName = uuid.v4(); const metadata = { @@ -127,7 +122,6 @@ class StorageService { const videoCheck = videoChecker(currentFile.filename); if (videoCheck) { - console.log("is vidoe"); const updatedFile = await createVideoThumbnailFS( currentFile, filename, @@ -255,23 +249,8 @@ class StorageService { }; }; - streamVideo = async ( - user: UserInterface, - fileID: string, - headers: any, - res: Response, - req: Request - ) => { - // To get this all working correctly with encryption and across - // All browsers took many days, tears, and some of my sanity. - // Shoutout to Tyzoid for helping me with the decryption - // And and helping me understand how the IVs work. - - // P.S I hate safari >:( - // Why do yall have to be weird with video streaming - // 90% of the issues with this are only in Safari - // Is safari going to be the next internet explorer? - + // INJECTED + streamVideo = async (user: UserInterface, fileID: string, headers: any) => { const userID = user._id; const currentFile = await dbUtilsFile.getFileInfo( fileID, @@ -306,52 +285,33 @@ class StorageService { let fixedEnd = currentFile.length; if (start === 0 && end === 1) { - // This is for Safari/iOS, Safari will request the first - // Byte before actually playing the video. Needs to be - // 16 bytes. - fixedStart = 0; fixedEnd = 15; } else { - // If you're a normal browser, or this isn't Safari's first request - // We need to make it so start is divisible by 16, since AES256 - // Has a block size of 16 bytes. - fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start); } if (+start === 0) { - // This math will not work if the start is 0 - // So if it is we just change fixed start back - // To 0. - fixedStart = 0; } - // We also need to calculate the difference between the start and the - // Fixed start position. Since there will be an offset if the original - // Request is not divisible by 16, it will not return the right part - // Of the file, you will see how we do this in the awaitStreamVideo - // code. - - const differenceStart = start - fixedStart; + const readStreamParams = createGenericParams({ + filePath: currentFile.metadata.filePath, + Key: currentFile.metadata.s3ID, + }); if (fixedStart !== 0 && start !== 0) { - // If this isn't the first request, the way AES256 works is when you try to - // Decrypt a certain part of the file that isn't the start, the IV will - // Actually be the 16 bytes ahead of where you are trying to - // Start the decryption. - - currentIV = (await getPrevIVFS( - fixedStart - 16, - currentFile.metadata.filePath! + currentIV = (await storageActions.getPrevIV( + readStreamParams, + fixedStart - 16 )) as Buffer; } - const readStream = fs.createReadStream(currentFile.metadata.filePath!, { - start: fixedStart, - end: fixedEnd, - }); + const readStream = storageActions.createReadStreamWithRange( + readStreamParams, + fixedStart, + fixedEnd + ); const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); @@ -359,26 +319,15 @@ class StorageService { decipher.setAutoPadding(false); - res.writeHead(206, head); - - const allStreamsToErrorCatch = [readStream, decipher]; - - readStream.pipe(decipher); - - await awaitStreamVideo( - start, - end, - differenceStart, + return { + readStream, decipher, - res, - req, - allStreamsToErrorCatch, - readStream - ); - - readStream.destroy(); + file: currentFile, + head, + }; }; + // INJECTED getPublicDownload = async (fileID: string, tempToken: any, res: Response) => { const file: FileInterface = await dbUtilsFile.getPublicFile(fileID); @@ -386,34 +335,37 @@ class StorageService { throw new NotAuthorizedError("File Not Public"); } + await dbUtilsFile.removeOneTimePublicLink(fileID); + const user = (await User.findById(file.metadata.owner)) as UserInterface; const password = user.getEncryptionKey(); if (!password) throw new ForbiddenError("Invalid Encryption Key"); - const IV = file.metadata.IV.buffer as Buffer; + // TODO: I believe this has to do with using conn.db instead of the mongoose driver + // We should switch to the mongoose driver + let IV = file.metadata.IV as any; + if (!(IV instanceof Buffer)) { + IV = IV.buffer; + } - const readStream = fs.createReadStream(file.metadata.filePath!); + const readStreamParams = createGenericParams({ + filePath: file.metadata.filePath, + Key: file.metadata.s3ID, + }); + + const readStream = storageActions.createReadStream(readStreamParams); const CIPHER_KEY = crypto.createHash("sha256").update(password).digest(); const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV); - res.set("Content-Type", "binary/octet-stream"); - res.set( - "Content-Disposition", - 'attachment; filename="' + file.filename + '"' - ); - res.set("Content-Length", file.metadata.size.toString()); - - const allStreamsToErrorCatch = [readStream, decipher]; - - await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch); - - if (file.metadata.linkType === "one") { - await dbUtilsFile.removeOneTimePublicLink(fileID); - } + return { + readStream, + decipher, + file: file, + }; }; // INJECTED diff --git a/backend/services/ChunkService/utils/awaitUploadStreamS3.ts b/backend/services/ChunkService/utils/awaitUploadStreamS3.ts index 30addcd..bd20412 100644 --- a/backend/services/ChunkService/utils/awaitUploadStreamS3.ts +++ b/backend/services/ChunkService/utils/awaitUploadStreamS3.ts @@ -1,11 +1,7 @@ import { ManagedUpload } from "aws-sdk/clients/s3"; import s3 from "../../../db/s3"; -const awaitUploadStreamS3 = ( - params: any, - personalFile: boolean, - s3Data: { id: string; key: string; bucket: string } -) => { +const uploadStreamS3 = (params: any) => { return new Promise((resolve, reject) => { s3.upload(params, (err: any, data: ManagedUpload.SendData) => { if (err) { @@ -18,4 +14,4 @@ const awaitUploadStreamS3 = ( }); }; -export default awaitUploadStreamS3; +export default uploadStreamS3; diff --git a/backend/services/ChunkService/utils/createVideoThumbnailFS.ts b/backend/services/ChunkService/utils/createVideoThumbnailFS.ts index 8ea52a8..1eec67b 100644 --- a/backend/services/ChunkService/utils/createVideoThumbnailFS.ts +++ b/backend/services/ChunkService/utils/createVideoThumbnailFS.ts @@ -45,43 +45,6 @@ const createVideoThumbnailFS = ( const decryptedReadStream = readStream.pipe(decipher); - // const encryptedWriteStream = thumbnailCipher - // .pipe(writeStream) - // .on("finish", async () => { - // const thumbnailModel = new Thumbnail({ - // name: filename, - // owner: user._id, - // IV: thumbnailIV, - // path: env.fsDirectory + thumbnailFilename, - // }); - - // await thumbnailModel.save(); - // if (!file._id) { - // return reject(); - // } - // const updateFileResponse = await File.updateOne( - // { _id: new ObjectId(file._id), "metadata.owner": user._id }, - // { - // $set: { - // "metadata.hasThumbnail": true, - // "metadata.thumbnailID": thumbnailModel._id, - // }, - // } - // ); - // if (updateFileResponse.modifiedCount === 0) { - // return reject(); - // } - - // const updatedFile = await File.findById({ - // _id: new ObjectId(file._id), - // "metadata.owner": user._id, - // }); - - // if (!updatedFile) return reject(); - - // resolve(updatedFile?.toObject()); - // }); - ffmpeg(decryptedReadStream, { timeout: 60, }) @@ -146,44 +109,6 @@ const createVideoThumbnailFS = ( }) .pipe(thumbnailCipher) .pipe(writeStream, { end: true }); - - // ffmpeg() - // .input(readStream.pipe(decipher)) - // .seekInput("00:00:01.00") - // .outputFormat("image2") - // .pipe(writeStream, { end: true }) - // .on("end", async () => { - // const thumbnailModel = new Thumbnail({ - // name: filename, - // owner: user._id, - // IV: thumbnailIV, - // path: env.fsDirectory + thumbnailFilename, - // }); - - // await thumbnailModel.save(); - // resolve(file); - // }) - // .on("error", (e) => { - // console.log("error", e); - // reject(e); - // }); - - // ffmpeg() - // .input(readStream.pipe(decipher)) - // .size("300x?") - // .outputOptions("-frames:v 1") - // .on("end", async () => { - // const thumbnailModel = new Thumbnail({ - // name: filename, - // owner: user._id, - // IV: thumbnailIV, - // path: env.fsDirectory + thumbnailFilename, - // }); - - // await thumbnailModel.save(); - // resolve(file); - // }) - // .output(writeStream); }); }; diff --git a/src/components/LeftSection/index.jsx b/src/components/LeftSection/index.jsx index c2fd3ec..13f5541 100644 --- a/src/components/LeftSection/index.jsx +++ b/src/components/LeftSection/index.jsx @@ -104,10 +104,9 @@ const LeftSection = (props) => { )}
-
-
    +