diff --git a/backend/controllers/file.ts b/backend/controllers/file.ts index 01ce340..cc9bb37 100644 --- a/backend/controllers/file.ts +++ b/backend/controllers/file.ts @@ -10,7 +10,9 @@ import { } from "../cookies/createCookies"; import ChunkService from "../services/ChunkService"; import streamToBuffer from "../utils/streamToBuffer"; -import { read } from "fs"; +import env from "../enviroment/env"; +import getFileSize from "../services/ChunkService/utils/getFileSize"; +import File, { FileMetadateInterface } from "../models/file"; const fileService = new FileService(); @@ -132,22 +134,84 @@ class FileController { return; } + let responseSent = false; + + const handleError = () => { + if (!responseSent) { + responseSent = true; + res.status(500).send("Server error uploading file"); + } + }; + + const handleFinish = async ( + filename: string, + metadata: FileMetadateInterface + ) => { + try { + const date = new Date(); + + let length = 0; + + if (env.dbType === "fs" && metadata.filePath) { + length = (await getFileSize(metadata.filePath)) as number; + } else { + // TODO: Fix this we should be using the encrypted file size + length = metadata.size; + } + + const currentFile = new File({ + filename, + uploadDate: date.toISOString(), + length, + metadata, + }); + + await currentFile.save(); + + res.send(currentFile); + } catch (e: unknown) { + if (!responseSent) { + res.writeHead(500, { Connection: "close" }); + res.end(); + } + } + }; + try { const user = req.user; const busboy = req.busboy; + busboy.on("error", (e: Error) => { + console.log("busboy error", e); + handleError(); + }); + req.pipe(busboy); - const file = await this.chunkService.uploadFile(user, busboy, req); + const { cipher, fileWriteStream, metadata, filename } = + await this.chunkService.uploadFile(user, busboy, req); - res.send(file); + cipher.on("error", (e: Error) => { + console.log("cipher error", e); + handleError(); + }); + + fileWriteStream.on("error", (e: Error) => { + console.log("file write stream error", e); + handleError(); + }); + + cipher.pipe(fileWriteStream).on("finish", async () => { + handleFinish(filename, metadata); + }); } catch (e: unknown) { if (e instanceof Error) { console.log("\nUploading File Error File Route:", e.message); } - - res.writeHead(500, { Connection: "close" }); - res.end(); + if (!responseSent) { + res.writeHead(500, { Connection: "close" }); + res.end(); + } } }; @@ -184,10 +248,6 @@ class FileController { res.set("Content-Length", file.metadata.size.toString()); readStream.pipe(decipher).pipe(res); - - // if (file.metadata.linkType === "one") { - // await dbUtilsFile.removeOneTimePublicLink(fileID); - // } } catch (e: unknown) { if (e instanceof Error) { console.log("\nGet Public Download Error File Route:", e.message); diff --git a/backend/models/file.ts b/backend/models/file.ts index bd67ac6..b4f2a8b 100644 --- a/backend/models/file.ts +++ b/backend/models/file.ts @@ -59,6 +59,23 @@ const fileSchema = new mongoose.Schema({ }, }); +export interface FileMetadateInterface { + owner: string; + parent: string; + parentList: string; + hasThumbnail: boolean; + isVideo: boolean; + thumbnailID?: string; + size: number; + IV: Buffer; + linkType?: "one" | "public"; + link?: string; + filePath?: string; + s3ID?: string; + personalFile?: boolean; + trashed?: boolean; +} + export interface FileInterface extends mongoose.Document { length: number; @@ -66,22 +83,7 @@ export interface FileInterface uploadDate: string; filename: string; lastErrorObject: { updatedExisting: any }; - metadata: { - owner: string; - parent: string; - parentList: string; - hasThumbnail: boolean; - isVideo: boolean; - thumbnailID?: string; - size: number; - IV: Buffer; - linkType?: "one" | "public"; - link?: string; - filePath?: string; - s3ID?: string; - personalFile?: boolean; - trashed?: boolean; - }; + metadata: FileMetadateInterface; } const File = mongoose.model("fs.files", fileSchema); diff --git a/backend/services/ChunkService/FileSystemActions.ts b/backend/services/ChunkService/FileSystemActions.ts index e3c87d9..8d311f7 100644 --- a/backend/services/ChunkService/FileSystemActions.ts +++ b/backend/services/ChunkService/FileSystemActions.ts @@ -1,6 +1,7 @@ import fs from "fs"; import { UserInterface } from "../../models/user"; import { GenericParams, IStorageActions } from "./StoreTypes"; +import env from "../../enviroment/env"; class FilesystemActions implements IStorageActions { async getAuth() { @@ -20,7 +21,7 @@ class FilesystemActions implements IStorageActions { }); return fsReadableStream; } - async removeChunks(params: GenericParams) { + removeChunks(params: GenericParams) { return new Promise((resolve, reject) => { if (!params.filePath) { reject("File path not configured"); @@ -36,7 +37,7 @@ class FilesystemActions implements IStorageActions { }); }); } - async getPrevIV(params: GenericParams, start: number) { + getPrevIV(params: GenericParams, start: number) { return new Promise((resolve, reject) => { if (!params.filePath) throw new Error("File path not configured"); const stream = fs.createReadStream(params.filePath, { @@ -49,6 +50,19 @@ class FilesystemActions implements IStorageActions { }); }); } + uploadFile = (params: GenericParams, stream: NodeJS.ReadableStream) => { + return new Promise((resolve, reject) => { + resolve(); + }); + }; + createWriteStream = ( + params: GenericParams, + stream: NodeJS.ReadableStream, + randomID: string + ) => { + const path = `${env.fsDirectory}${randomID}`; + return fs.createWriteStream(path); + }; } export { FilesystemActions }; diff --git a/backend/services/ChunkService/S3Actions.ts b/backend/services/ChunkService/S3Actions.ts index d32b2c9..b701eec 100644 --- a/backend/services/ChunkService/S3Actions.ts +++ b/backend/services/ChunkService/S3Actions.ts @@ -3,6 +3,7 @@ import s3 from "../../db/s3"; import env from "../../enviroment/env"; import { GenericParams, IStorageActions } from "./StoreTypes"; import internal from "stream"; +import { PassThrough } from "stream"; class S3Actions implements IStorageActions { getAuth() { @@ -48,7 +49,7 @@ class S3Actions implements IStorageActions { }); }); } - async getPrevIV(params: GenericParams, start: number) { + getPrevIV(params: GenericParams, start: number) { return new Promise((resolve, reject) => { if (!params.Key) throw new Error("S3 not configured"); const { s3Storage, bucket } = this.getAuth(); @@ -63,6 +64,26 @@ class S3Actions implements IStorageActions { }); }); } + createWriteStream = ( + params: GenericParams, + stream: NodeJS.ReadableStream, + randomID: string + ) => { + const pass = new PassThrough(); + + const { Key } = params; + if (!Key) throw new Error("S3 not configured"); + const { s3Storage, bucket } = this.getAuth(); + + s3Storage.upload({ Bucket: bucket, Body: stream, Key: randomID }, (err) => { + if (err) { + console.log("Amazon upload err", err); + pass.emit("error", err); + } + }); + + return pass; + }; } export { S3Actions }; diff --git a/backend/services/ChunkService/index.ts b/backend/services/ChunkService/index.ts index ad14e88..080cdf9 100644 --- a/backend/services/ChunkService/index.ts +++ b/backend/services/ChunkService/index.ts @@ -61,7 +61,6 @@ class StorageService { const filename = fileInfo.filename; const parent = formData.get("parent") || "/"; const size = formData.get("size") || ""; - const personalFile = formData.get("personal-file") ? true : false; let hasThumbnail = false; let thumbnailID = ""; const isVideo = videoChecker(filename); @@ -78,7 +77,7 @@ class StorageService { parentList.push("/"); } - const systemFileName = uuid.v4(); + const randomFilenameID = uuid.v4(); const metadata = { owner: user._id, @@ -89,53 +88,26 @@ class StorageService { isVideo, size, IV: initVect, - filePath: env.fsDirectory + systemFileName, - }; + } as any; - const fileWriteStream = fs.createWriteStream(metadata.filePath); + if (env.dbType === "fs") { + metadata.filePath = env.fsDirectory + randomFilenameID; + } else { + metadata.s3ID = randomFilenameID; + } - const totalStreamsToErrorCatch = [file, cipher, fileWriteStream]; - - await awaitUploadStreamFS( + const fileWriteStream = storageActions.createWriteStream( + metadata, file.pipe(cipher), - fileWriteStream, - req, - metadata.filePath, - totalStreamsToErrorCatch + randomFilenameID ); - const date = new Date(); - const encryptedFileSize = await getFileSize(metadata.filePath); - - const currentFile = new File({ - filename, - uploadDate: date.toISOString(), - length: encryptedFileSize, + return { + cipher, + fileWriteStream, metadata, - }); - - await currentFile.save(); - - await addToStoageSize(user, size, personalFile); - - const imageCheck = imageChecker(currentFile.filename); - const videoCheck = videoChecker(currentFile.filename); - - if (videoCheck) { - const updatedFile = await createVideoThumbnailFS( - currentFile, - filename, - user - ); - - return updatedFile; - } else if (currentFile.length < 15728640 && imageCheck) { - const updatedFile = await createThumbnailAny(currentFile, filename, user); - - return updatedFile; - } else { - return currentFile; - } + filename, + }; }; // INJECTED