started folder uploads

backend/controllers/folder-controller.ts

@@ -2,6 +2,7 @@ import FolderService from "../services/folder-service/folder-service";
 import { NextFunction, Request, Response } from "express";
 import ChunkService from "../services/chunk-service/chunk-service";
 import { FolderListQueryType } from "../types/folder-types";
+import { UserInterface } from "../models/user-model";
 
 const folderService = new FolderService();
 
@@ -91,6 +92,18 @@ class FolderController {
         // handleError();
       });
 
+      req.on("end", () => {
+        console.log("end req");
+      });
+
+      req.on("close", () => {
+        console.log("close req");
+      });
+
+      req.on("error", () => {
+        console.log("error req");
+      });
+
       req.pipe(busboy);
 
       await chunkService.uploadFolder(user, busboy, req);

backend/db/mongoDB/fileDB.ts

@@ -34,7 +34,11 @@ class DbUtil {
   };
 
   getQuickList = async (userID: string, limit: number) => {
-    let query: any = { "metadata.owner": userID, "metadata.trashed": null };
+    let query: any = {
+      "metadata.owner": userID,
+      "metadata.trashed": null,
+      "metadata.processingFile": null,
+    };
 
     const fileList = await File.find(query)
       .sort({ uploadDate: -1 })
@@ -116,6 +120,27 @@ class DbUtil {
 
   // UPDATE
 
+  updateFolderUploadedFile = async (
+    fileID: string,
+    userID: string,
+    parent: string,
+    parentList: string
+  ) => {
+    const file = await File.findOneAndUpdate(
+      { _id: new ObjectId(fileID), "metadata.owner": userID },
+      {
+        $set: {
+          "metadata.parent": parent,
+          "metadata.parentList": parentList,
+        },
+        $unset: { "metadata.processingFile": null },
+      },
+      { new: true }
+    );
+
+    return file;
+  };
+
   removeOneTimePublicLink = async (
     fileID: string | mongoose.Types.ObjectId
   ) => {

backend/models/file-model.ts

@@ -61,6 +61,7 @@ const fileSchema = new mongoose.Schema({
       s3ID: String,
       personalFile: Boolean,
       trashed: Boolean,
+      processingFile: Boolean,
     },
     required: true,
   },
@@ -81,6 +82,7 @@ export interface FileMetadateInterface {
   s3ID?: string;
   personalFile?: boolean;
   trashed?: boolean | null;
+  processingFile?: boolean;
 }
 
 export interface FileInterface

backend/services/chunk-service/chunk-service.ts

@@ -141,9 +141,114 @@ class StorageService {
 
     const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect);
 
-    const fileData = await getFolderBusboyData(busboy);
+    console.log("1");
+    const { fileDataMap, parent } = await getFolderBusboyData(busboy, user);
+    console.log("2");
+    const keys = Object.keys(fileDataMap);
 
-    console.log("file data", fileData);
+    const folderPathsToCreate: Record<string, boolean> = {};
+
+    const parentList = [];
+
+    if (parent !== "/") {
+      const parentFolder = await folderDB.getFolderInfo(
+        parent,
+        user._id.toString()
+      );
+      if (!parentFolder) throw new NotFoundError("Parent Folder Not Found");
+      parentList.push(...parentFolder.parentList, parentFolder._id.toString());
+    } else {
+      parentList.push("/");
+    }
+
+    const parentName = fileDataMap[keys[0]].path.split("/")[0];
+
+    console.log("parent name", parentName);
+
+    const rootFolder = await folderDB.createFolder({
+      name: parentName,
+      parent: parentList[parentList.length - 1],
+      owner: user._id.toString(),
+      parentList: parentList,
+    });
+
+    parentList.push(rootFolder._id.toString());
+
+    for (const key of keys) {
+      const pathSplit = fileDataMap[key].path.split("/");
+      const path = pathSplit.slice(1, pathSplit.length - 1).join("/");
+      // console.log("path", path);
+
+      if (path && !folderPathsToCreate[path]) {
+        folderPathsToCreate[path] = true;
+      }
+    }
+
+    const sortedFolderPaths = Object.keys(folderPathsToCreate).sort();
+
+    const foldersCreated: Record<string, FolderInterface> = {};
+
+    for (const folderPath of sortedFolderPaths) {
+      // const parentListData = [];
+      // const tempFolderNameList = []
+      const subFolders = folderPath.split("/");
+      const parentDirectory = subFolders
+        .slice(0, subFolders.length - 1)
+        .join("/");
+
+      const tempParentList = [];
+
+      if (parentDirectory && foldersCreated[parentDirectory]) {
+        tempParentList.push(
+          ...foldersCreated[parentDirectory].parentList,
+          foldersCreated[parentDirectory]._id.toString()
+        );
+      } else {
+        tempParentList.push(...parentList);
+      }
+      // for (const subFolder of subFolders) {
+      //   // if
+      // }
+      const folderToCreate = subFolders[subFolders.length - 1];
+      //console.log("folder to create", folderToCreate);
+
+      const folder = await folderDB.createFolder({
+        name: folderToCreate,
+        parent: tempParentList[tempParentList.length - 1],
+        owner: user._id.toString(),
+        parentList: tempParentList,
+      });
+
+      foldersCreated[folderPath] = folder;
+    }
+
+    for (const key of keys) {
+      const currentFile = fileDataMap[key];
+      const parentSplit = currentFile.path.split("/");
+      const parentDirectory = parentSplit
+        .slice(1, parentSplit.length - 1)
+        .join("/");
+      if (parentDirectory && foldersCreated[parentDirectory]) {
+        await fileDB.updateFolderUploadedFile(
+          currentFile.uploadedFileId,
+          user._id.toString(),
+          foldersCreated[parentDirectory]._id.toString(),
+          [
+            ...foldersCreated[parentDirectory].parentList,
+            foldersCreated[parentDirectory]._id.toString(),
+          ].toString()
+        );
+      } else {
+        await fileDB.updateFolderUploadedFile(
+          currentFile.uploadedFileId,
+          user._id.toString(),
+          rootFolder._id.toString(),
+          [...rootFolder.parentList, rootFolder._id.toString()].toString()
+        );
+      }
+    }
+
+    console.log("folders created", fileDataMap);
   };
 
   downloadFile = async (user: UserInterface, fileID: string, res: Response) => {

backend/services/chunk-service/utils/getFolderUploadBusboyData.ts

@@ -1,62 +1,251 @@
 import { Stream } from "stream";
+import uuid from "uuid";
+import { UserInterface } from "../../../models/user-model";
+import File, {
+  FileInterface,
+  FileMetadateInterface,
+} from "../../../models/file-model";
+import env from "../../../enviroment/env";
+import { S3Actions } from "../actions/S3-actions";
+import { FilesystemActions } from "../actions/file-system-actions";
+import ForbiddenError from "../../../utils/ForbiddenError";
+import crypto from "crypto";
+import getFileSize from "./getFileSize";
+import imageChecker from "../../../utils/imageChecker";
+import videoChecker from "../../../utils/videoChecker";
+import createVideoThumbnail from "./createVideoThumbnail";
+import createThumbnail from "./createImageThumbnail";
 
-const getFolderBusboyData = (busboy: any) => {
-  type dataType = Record<string, any>;
+type FileDataType = {
+  name: string;
+  size: number;
+  type: string;
+  path: string;
+  index: number;
+  file: Stream;
+  uploadedFileId: string;
+};
 
-  return new Promise<dataType>((resolve, reject) => {
-    const formData = new Map();
+const storageActions =
+  env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
 
-    let filesProcessed = 0;
-    let filesToProcess = 0;
+const getFolderBusboyData = (busboy: any, user: UserInterface) => {
+  type dataType = Record<string, FileDataType>;
 
-    const fileDataMap: dataType = {};
+  return new Promise<{ fileDataMap: dataType; parent: string }>(
+    (resolve, reject) => {
+      const formData = new Map();
 
-    busboy.on("field", (field: any, val: any) => {
-      if (typeof val !== "string" || val !== "undefined") {
-        formData.set(field, val);
-        if (field === "file-data") {
-          const fileData = JSON.parse(val);
-          console.log("file data", fileData);
-          fileDataMap[fileData.path] = {
-            ...fileData,
-          };
-        }
-        if (field === "total-files") {
-          filesToProcess = +val;
-          console.log("total files", filesToProcess);
-        }
-      }
-    });
+      let filesProcessed = 0;
+      let filesToProcess = 0;
+      let parent = "";
 
-    busboy.on(
-      "file",
-      async (
-        data: string,
-        file: Stream,
-        fileData: {
-          filename: string;
-        }
+      const fileDataMap: dataType = {};
+
+      const uploadQueue: { file: Stream; index: string }[] = [];
+
+      let processing = false;
+
+      const handleFinish = async (
+        filename: string,
+        metadata: FileMetadateInterface
       ) => {
-        const path = fileData.filename;
-        fileDataMap[path] = {
-          ...fileDataMap[path],
-          file,
-        };
+        try {
+          const date = new Date();
 
-        filesProcessed++;
+          let length = 0;
 
-        if (filesProcessed === filesToProcess) {
-          resolve(fileDataMap);
+          if (env.dbType === "fs" && metadata.filePath) {
+            length = (await getFileSize(metadata.filePath)) as number;
+          } else {
+            // TODO: Fix this we should be using the encrypted file size
+            length = metadata.size;
+          }
+
+          const currentFile = new File({
+            filename,
+            uploadDate: date.toISOString(),
+            length,
+            metadata,
+          });
+
+          await currentFile.save();
+
+          const imageCheck = imageChecker(currentFile.filename);
+          const videoCheck = videoChecker(currentFile.filename);
+
+          if (videoCheck) {
+            const updatedFile = await createVideoThumbnail(
+              currentFile,
+              filename,
+              user
+            );
+            return updatedFile;
+          } else if (currentFile.length < 15728640 && imageCheck) {
+            const updatedFile = await createThumbnail(
+              currentFile,
+              filename,
+              user
+            );
+            return updatedFile;
+          } else {
+            return currentFile;
+          }
+        } catch (e: unknown) {
+          console.log("handle finish error", e);
         }
-        // resolve({
-        //   file,
-        //   filename,
-        //   formData,
-        // });
-        console.log("flr", fileData);
-      }
-    );
-  });
+      };
+
+      const uploadFile = (currentFile: { file: Stream; index: string }) => {
+        return new Promise<FileInterface | undefined>((resolve, reject) => {
+          const { file, index } = currentFile;
+
+          const fileData = fileDataMap[index];
+
+          const randomFilenameID = uuid.v4();
+
+          const password = user.getEncryptionKey();
+
+          if (!password) throw new ForbiddenError("Invalid Encryption Key");
+
+          const initVect = crypto.randomBytes(16);
+
+          const CIPHER_KEY = crypto
+            .createHash("sha256")
+            .update(password)
+            .digest();
+
+          const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect);
+
+          const filename = fileData.name;
+          const fileSize = fileData.size;
+
+          const metadata = {
+            owner: user._id.toString(),
+            parent: "/",
+            parentList: ["/"].toString(),
+            hasThumbnail: false,
+            thumbnailID: "",
+            isVideo: false,
+            size: fileSize,
+            IV: initVect,
+            processingFile: true,
+          } as FileMetadateInterface;
+
+          if (env.dbType === "fs") {
+            metadata.filePath = env.fsDirectory + randomFilenameID;
+          } else {
+            metadata.s3ID = randomFilenameID;
+          }
+
+          const { writeStream, emitter } = storageActions.createWriteStream(
+            metadata,
+            file.pipe(cipher),
+            randomFilenameID
+          );
+
+          cipher.pipe(writeStream);
+
+          if (emitter) {
+            emitter.on("finish", async () => {
+              const file = await handleFinish(filename, metadata);
+              resolve(file);
+            });
+          } else {
+            writeStream.on("finish", async () => {
+              const file = await handleFinish(filename, metadata);
+              resolve(file);
+            });
+          }
+        });
+      };
+
+      const processQueue = async () => {
+        if (processing) return;
+
+        processing = true;
+
+        while (uploadQueue.length > 0) {
+          const currentFile = uploadQueue.shift();
+          console.log("processing file", currentFile);
+          const file = await uploadFile(currentFile!);
+          if (!file) {
+            // TODO: Handle error
+            console.log("Error uploading file");
+            break;
+          }
+
+          fileDataMap[currentFile!.index] = {
+            ...fileDataMap[currentFile!.index],
+            uploadedFileId: file._id.toString(),
+          };
+
+          filesProcessed++;
+
+          console.log("files processed", currentFile);
+
+          if (filesProcessed === filesToProcess) {
+            resolve({ fileDataMap, parent });
+          }
+        }
+        processing = false;
+      };
+
+      busboy.on("field", (field: any, val: any) => {
+        console.log("field", field, val);
+        if (typeof val !== "string" || val !== "undefined") {
+          formData.set(field, val);
+          if (field === "file-data") {
+            const fileData = JSON.parse(val);
+            fileDataMap[fileData.index] = fileData;
+          }
+          if (field === "total-files") {
+            filesToProcess = +val;
+          }
+          if (field === "parent") {
+            parent = val;
+          }
+        }
+      });
+
+      busboy.on(
+        "file",
+        async (
+          _: string,
+          file: Stream,
+          fileData: {
+            filename: string;
+          }
+        ) => {
+          const index = fileData.filename;
+          // fileDataMap[index] = {
+          //   ...fileDataMap[index],
+          //   file,
+          // };
+
+          uploadQueue.push({ file, index });
+
+          processQueue();
+
+          // file.on("data", () => {
+          //   console.log("data");
+          // });
+
+          // console.log("file data", fileData);
+
+          // filesProcessed++;
+
+          // if (filesProcessed === filesToProcess) {
+          //   resolve(fileDataMap);
+          // }
+        }
+      );
+
+      busboy.on("error", (e: Error) => {
+        reject(e);
+      });
+    }
+  );
 };
 
 export default getFolderBusboyData;

backend/utils/createQuery.ts

@@ -20,6 +20,7 @@ export interface FileQueryInterface {
   "metadata.trashed"?: boolean | null;
   "metadata.hasThumbnail"?: boolean | null;
   "metadata.isVideo"?: boolean | null;
+  "metadata.processingFile"?: boolean | null;
 }
 
 export const createFileQuery = ({
@@ -67,6 +68,8 @@ export const createFileQuery = ({
     }
   }
 
+  query["metadata.processingFile"] = null;
+
   return query;
 };
 

src/hooks/files.ts

@@ -296,6 +296,7 @@ export const useUploader = () => {
           size: currentFile.size,
           type: currentFile.type,
           path: currentFile.webkitRelativePath,
+          index: i,
         })
       );
     }
@@ -305,7 +306,7 @@ export const useUploader = () => {
     for (let i = 0; i < files.length; i++) {
       const currentFile = files[i];
       console.log("current file", currentFile.webkitRelativePath);
-      data.append("file", currentFile, "test");
+      data.append("file", currentFile, i.toString());
     }
 
     const CancelToken = axiosNonInterceptor.CancelToken;