started injecting uploads

backend/controllers/file.ts

@@ -10,7 +10,9 @@ import {
 } from "../cookies/createCookies";
 import ChunkService from "../services/ChunkService";
 import streamToBuffer from "../utils/streamToBuffer";
-import { read } from "fs";
+import env from "../enviroment/env";
+import getFileSize from "../services/ChunkService/utils/getFileSize";
+import File, { FileMetadateInterface } from "../models/file";
 
 const fileService = new FileService();
 
@@ -132,22 +134,84 @@ class FileController {
       return;
     }
 
+    let responseSent = false;
+
+    const handleError = () => {
+      if (!responseSent) {
+        responseSent = true;
+        res.status(500).send("Server error uploading file");
+      }
+    };
+
+    const handleFinish = async (
+      filename: string,
+      metadata: FileMetadateInterface
+    ) => {
+      try {
+        const date = new Date();
+
+        let length = 0;
+
+        if (env.dbType === "fs" && metadata.filePath) {
+          length = (await getFileSize(metadata.filePath)) as number;
+        } else {
+          // TODO: Fix this we should be using the encrypted file size
+          length = metadata.size;
+        }
+
+        const currentFile = new File({
+          filename,
+          uploadDate: date.toISOString(),
+          length,
+          metadata,
+        });
+
+        await currentFile.save();
+
+        res.send(currentFile);
+      } catch (e: unknown) {
+        if (!responseSent) {
+          res.writeHead(500, { Connection: "close" });
+          res.end();
+        }
+      }
+    };
+
     try {
       const user = req.user;
       const busboy = req.busboy;
 
+      busboy.on("error", (e: Error) => {
+        console.log("busboy error", e);
+        handleError();
+      });
+
       req.pipe(busboy);
 
-      const file = await this.chunkService.uploadFile(user, busboy, req);
+      const { cipher, fileWriteStream, metadata, filename } =
+        await this.chunkService.uploadFile(user, busboy, req);
 
-      res.send(file);
+      cipher.on("error", (e: Error) => {
+        console.log("cipher error", e);
+        handleError();
+      });
+
+      fileWriteStream.on("error", (e: Error) => {
+        console.log("file write stream error", e);
+        handleError();
+      });
+
+      cipher.pipe(fileWriteStream).on("finish", async () => {
+        handleFinish(filename, metadata);
+      });
     } catch (e: unknown) {
       if (e instanceof Error) {
         console.log("\nUploading File Error File Route:", e.message);
       }
-
-      res.writeHead(500, { Connection: "close" });
-      res.end();
+      if (!responseSent) {
+        res.writeHead(500, { Connection: "close" });
+        res.end();
+      }
     }
   };
 
@@ -184,10 +248,6 @@ class FileController {
       res.set("Content-Length", file.metadata.size.toString());
 
       readStream.pipe(decipher).pipe(res);
-
-      // if (file.metadata.linkType === "one") {
-      //   await dbUtilsFile.removeOneTimePublicLink(fileID);
-      // }
     } catch (e: unknown) {
       if (e instanceof Error) {
         console.log("\nGet Public Download Error File Route:", e.message);

backend/models/file.ts

@@ -59,6 +59,23 @@ const fileSchema = new mongoose.Schema({
   },
 });
 
+export interface FileMetadateInterface {
+  owner: string;
+  parent: string;
+  parentList: string;
+  hasThumbnail: boolean;
+  isVideo: boolean;
+  thumbnailID?: string;
+  size: number;
+  IV: Buffer;
+  linkType?: "one" | "public";
+  link?: string;
+  filePath?: string;
+  s3ID?: string;
+  personalFile?: boolean;
+  trashed?: boolean;
+}
+
 export interface FileInterface
   extends mongoose.Document<mongoose.Types.ObjectId> {
   length: number;
@@ -66,22 +83,7 @@ export interface FileInterface
   uploadDate: string;
   filename: string;
   lastErrorObject: { updatedExisting: any };
-  metadata: {
-    owner: string;
-    parent: string;
-    parentList: string;
-    hasThumbnail: boolean;
-    isVideo: boolean;
-    thumbnailID?: string;
-    size: number;
-    IV: Buffer;
-    linkType?: "one" | "public";
-    link?: string;
-    filePath?: string;
-    s3ID?: string;
-    personalFile?: boolean;
-    trashed?: boolean;
-  };
+  metadata: FileMetadateInterface;
 }
 
 const File = mongoose.model<FileInterface>("fs.files", fileSchema);

backend/services/ChunkService/FileSystemActions.ts

@@ -1,6 +1,7 @@
 import fs from "fs";
 import { UserInterface } from "../../models/user";
 import { GenericParams, IStorageActions } from "./StoreTypes";
+import env from "../../enviroment/env";
 
 class FilesystemActions implements IStorageActions {
   async getAuth() {
@@ -20,7 +21,7 @@ class FilesystemActions implements IStorageActions {
     });
     return fsReadableStream;
   }
-  async removeChunks(params: GenericParams) {
+  removeChunks(params: GenericParams) {
     return new Promise<void>((resolve, reject) => {
       if (!params.filePath) {
         reject("File path not configured");
@@ -36,7 +37,7 @@ class FilesystemActions implements IStorageActions {
       });
     });
   }
-  async getPrevIV(params: GenericParams, start: number) {
+  getPrevIV(params: GenericParams, start: number) {
     return new Promise<Buffer | string>((resolve, reject) => {
       if (!params.filePath) throw new Error("File path not configured");
       const stream = fs.createReadStream(params.filePath, {
@@ -49,6 +50,19 @@ class FilesystemActions implements IStorageActions {
       });
     });
   }
+  uploadFile = (params: GenericParams, stream: NodeJS.ReadableStream) => {
+    return new Promise<void>((resolve, reject) => {
+      resolve();
+    });
+  };
+  createWriteStream = (
+    params: GenericParams,
+    stream: NodeJS.ReadableStream,
+    randomID: string
+  ) => {
+    const path = `${env.fsDirectory}${randomID}`;
+    return fs.createWriteStream(path);
+  };
 }
 
 export { FilesystemActions };

backend/services/ChunkService/S3Actions.ts

@@ -3,6 +3,7 @@ import s3 from "../../db/s3";
 import env from "../../enviroment/env";
 import { GenericParams, IStorageActions } from "./StoreTypes";
 import internal from "stream";
+import { PassThrough } from "stream";
 
 class S3Actions implements IStorageActions {
   getAuth() {
@@ -48,7 +49,7 @@ class S3Actions implements IStorageActions {
       });
     });
   }
-  async getPrevIV(params: GenericParams, start: number) {
+  getPrevIV(params: GenericParams, start: number) {
     return new Promise<Buffer | string>((resolve, reject) => {
       if (!params.Key) throw new Error("S3 not configured");
       const { s3Storage, bucket } = this.getAuth();
@@ -63,6 +64,26 @@ class S3Actions implements IStorageActions {
       });
     });
   }
+  createWriteStream = (
+    params: GenericParams,
+    stream: NodeJS.ReadableStream,
+    randomID: string
+  ) => {
+    const pass = new PassThrough();
+
+    const { Key } = params;
+    if (!Key) throw new Error("S3 not configured");
+    const { s3Storage, bucket } = this.getAuth();
+
+    s3Storage.upload({ Bucket: bucket, Body: stream, Key: randomID }, (err) => {
+      if (err) {
+        console.log("Amazon upload err", err);
+        pass.emit("error", err);
+      }
+    });
+
+    return pass;
+  };
 }
 
 export { S3Actions };

backend/services/ChunkService/index.ts

@@ -61,7 +61,6 @@ class StorageService {
     const filename = fileInfo.filename;
     const parent = formData.get("parent") || "/";
     const size = formData.get("size") || "";
-    const personalFile = formData.get("personal-file") ? true : false;
     let hasThumbnail = false;
     let thumbnailID = "";
     const isVideo = videoChecker(filename);
@@ -78,7 +77,7 @@ class StorageService {
       parentList.push("/");
     }
 
-    const systemFileName = uuid.v4();
+    const randomFilenameID = uuid.v4();
 
     const metadata = {
       owner: user._id,
@@ -89,53 +88,26 @@ class StorageService {
       isVideo,
       size,
       IV: initVect,
-      filePath: env.fsDirectory + systemFileName,
-    };
+    } as any;
 
-    const fileWriteStream = fs.createWriteStream(metadata.filePath);
+    if (env.dbType === "fs") {
+      metadata.filePath = env.fsDirectory + randomFilenameID;
+    } else {
+      metadata.s3ID = randomFilenameID;
+    }
 
-    const totalStreamsToErrorCatch = [file, cipher, fileWriteStream];
-
-    await awaitUploadStreamFS(
+    const fileWriteStream = storageActions.createWriteStream(
+      metadata,
       file.pipe(cipher),
-      fileWriteStream,
-      req,
-      metadata.filePath,
-      totalStreamsToErrorCatch
+      randomFilenameID
     );
 
-    const date = new Date();
-    const encryptedFileSize = await getFileSize(metadata.filePath);
-
-    const currentFile = new File({
-      filename,
-      uploadDate: date.toISOString(),
-      length: encryptedFileSize,
+    return {
+      cipher,
+      fileWriteStream,
       metadata,
-    });
-
-    await currentFile.save();
-
-    await addToStoageSize(user, size, personalFile);
-
-    const imageCheck = imageChecker(currentFile.filename);
-    const videoCheck = videoChecker(currentFile.filename);
-
-    if (videoCheck) {
-      const updatedFile = await createVideoThumbnailFS(
-        currentFile,
-        filename,
-        user
-      );
-
-      return updatedFile;
-    } else if (currentFile.length < 15728640 && imageCheck) {
-      const updatedFile = await createThumbnailAny(currentFile, filename, user);
-
-      return updatedFile;
-    } else {
-      return currentFile;
-    }
+      filename,
+    };
   };
 
   // INJECTED