diff --git a/backend/controllers/folder-controller.ts b/backend/controllers/folder-controller.ts index d08c967..013b100 100644 --- a/backend/controllers/folder-controller.ts +++ b/backend/controllers/folder-controller.ts @@ -2,6 +2,7 @@ import FolderService from "../services/folder-service/folder-service"; import { NextFunction, Request, Response } from "express"; import ChunkService from "../services/chunk-service/chunk-service"; import { FolderListQueryType } from "../types/folder-types"; +import { UserInterface } from "../models/user-model"; const folderService = new FolderService(); @@ -91,6 +92,18 @@ class FolderController { // handleError(); }); + req.on("end", () => { + console.log("end req"); + }); + + req.on("close", () => { + console.log("close req"); + }); + + req.on("error", () => { + console.log("error req"); + }); + req.pipe(busboy); await chunkService.uploadFolder(user, busboy, req); diff --git a/backend/db/mongoDB/fileDB.ts b/backend/db/mongoDB/fileDB.ts index 256f617..b488b70 100644 --- a/backend/db/mongoDB/fileDB.ts +++ b/backend/db/mongoDB/fileDB.ts @@ -34,7 +34,11 @@ class DbUtil { }; getQuickList = async (userID: string, limit: number) => { - let query: any = { "metadata.owner": userID, "metadata.trashed": null }; + let query: any = { + "metadata.owner": userID, + "metadata.trashed": null, + "metadata.processingFile": null, + }; const fileList = await File.find(query) .sort({ uploadDate: -1 }) @@ -116,6 +120,27 @@ class DbUtil { // UPDATE + updateFolderUploadedFile = async ( + fileID: string, + userID: string, + parent: string, + parentList: string + ) => { + const file = await File.findOneAndUpdate( + { _id: new ObjectId(fileID), "metadata.owner": userID }, + { + $set: { + "metadata.parent": parent, + "metadata.parentList": parentList, + }, + $unset: { "metadata.processingFile": null }, + }, + { new: true } + ); + + return file; + }; + removeOneTimePublicLink = async ( fileID: string | mongoose.Types.ObjectId ) => { diff --git a/backend/models/file-model.ts b/backend/models/file-model.ts index 3e64802..f753f2a 100644 --- a/backend/models/file-model.ts +++ b/backend/models/file-model.ts @@ -61,6 +61,7 @@ const fileSchema = new mongoose.Schema({ s3ID: String, personalFile: Boolean, trashed: Boolean, + processingFile: Boolean, }, required: true, }, @@ -81,6 +82,7 @@ export interface FileMetadateInterface { s3ID?: string; personalFile?: boolean; trashed?: boolean | null; + processingFile?: boolean; } export interface FileInterface diff --git a/backend/services/chunk-service/chunk-service.ts b/backend/services/chunk-service/chunk-service.ts index 8d5a28d..e3c15b1 100644 --- a/backend/services/chunk-service/chunk-service.ts +++ b/backend/services/chunk-service/chunk-service.ts @@ -141,9 +141,114 @@ class StorageService { const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect); - const fileData = await getFolderBusboyData(busboy); + console.log("1"); + const { fileDataMap, parent } = await getFolderBusboyData(busboy, user); + console.log("2"); + const keys = Object.keys(fileDataMap); - console.log("file data", fileData); + const folderPathsToCreate: Record = {}; + + const parentList = []; + + if (parent !== "/") { + const parentFolder = await folderDB.getFolderInfo( + parent, + user._id.toString() + ); + if (!parentFolder) throw new NotFoundError("Parent Folder Not Found"); + parentList.push(...parentFolder.parentList, parentFolder._id.toString()); + } else { + parentList.push("/"); + } + + const parentName = fileDataMap[keys[0]].path.split("/")[0]; + + console.log("parent name", parentName); + + const rootFolder = await folderDB.createFolder({ + name: parentName, + parent: parentList[parentList.length - 1], + owner: user._id.toString(), + parentList: parentList, + }); + + parentList.push(rootFolder._id.toString()); + + for (const key of keys) { + const pathSplit = fileDataMap[key].path.split("/"); + const path = pathSplit.slice(1, pathSplit.length - 1).join("/"); + // console.log("path", path); + + if (path && !folderPathsToCreate[path]) { + folderPathsToCreate[path] = true; + } + } + + const sortedFolderPaths = Object.keys(folderPathsToCreate).sort(); + + const foldersCreated: Record = {}; + + for (const folderPath of sortedFolderPaths) { + // const parentListData = []; + // const tempFolderNameList = [] + const subFolders = folderPath.split("/"); + const parentDirectory = subFolders + .slice(0, subFolders.length - 1) + .join("/"); + + const tempParentList = []; + + if (parentDirectory && foldersCreated[parentDirectory]) { + tempParentList.push( + ...foldersCreated[parentDirectory].parentList, + foldersCreated[parentDirectory]._id.toString() + ); + } else { + tempParentList.push(...parentList); + } + // for (const subFolder of subFolders) { + // // if + // } + const folderToCreate = subFolders[subFolders.length - 1]; + //console.log("folder to create", folderToCreate); + + const folder = await folderDB.createFolder({ + name: folderToCreate, + parent: tempParentList[tempParentList.length - 1], + owner: user._id.toString(), + parentList: tempParentList, + }); + + foldersCreated[folderPath] = folder; + } + + for (const key of keys) { + const currentFile = fileDataMap[key]; + const parentSplit = currentFile.path.split("/"); + const parentDirectory = parentSplit + .slice(1, parentSplit.length - 1) + .join("/"); + if (parentDirectory && foldersCreated[parentDirectory]) { + await fileDB.updateFolderUploadedFile( + currentFile.uploadedFileId, + user._id.toString(), + foldersCreated[parentDirectory]._id.toString(), + [ + ...foldersCreated[parentDirectory].parentList, + foldersCreated[parentDirectory]._id.toString(), + ].toString() + ); + } else { + await fileDB.updateFolderUploadedFile( + currentFile.uploadedFileId, + user._id.toString(), + rootFolder._id.toString(), + [...rootFolder.parentList, rootFolder._id.toString()].toString() + ); + } + } + + console.log("folders created", fileDataMap); }; downloadFile = async (user: UserInterface, fileID: string, res: Response) => { diff --git a/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts b/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts index 8f7fa4d..a313f1e 100644 --- a/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts +++ b/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts @@ -1,62 +1,251 @@ import { Stream } from "stream"; +import uuid from "uuid"; +import { UserInterface } from "../../../models/user-model"; +import File, { + FileInterface, + FileMetadateInterface, +} from "../../../models/file-model"; +import env from "../../../enviroment/env"; +import { S3Actions } from "../actions/S3-actions"; +import { FilesystemActions } from "../actions/file-system-actions"; +import ForbiddenError from "../../../utils/ForbiddenError"; +import crypto from "crypto"; +import getFileSize from "./getFileSize"; +import imageChecker from "../../../utils/imageChecker"; +import videoChecker from "../../../utils/videoChecker"; +import createVideoThumbnail from "./createVideoThumbnail"; +import createThumbnail from "./createImageThumbnail"; -const getFolderBusboyData = (busboy: any) => { - type dataType = Record; +type FileDataType = { + name: string; + size: number; + type: string; + path: string; + index: number; + file: Stream; + uploadedFileId: string; +}; - return new Promise((resolve, reject) => { - const formData = new Map(); +const storageActions = + env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); - let filesProcessed = 0; - let filesToProcess = 0; +const getFolderBusboyData = (busboy: any, user: UserInterface) => { + type dataType = Record; - const fileDataMap: dataType = {}; + return new Promise<{ fileDataMap: dataType; parent: string }>( + (resolve, reject) => { + const formData = new Map(); - busboy.on("field", (field: any, val: any) => { - if (typeof val !== "string" || val !== "undefined") { - formData.set(field, val); - if (field === "file-data") { - const fileData = JSON.parse(val); - console.log("file data", fileData); - fileDataMap[fileData.path] = { - ...fileData, - }; - } - if (field === "total-files") { - filesToProcess = +val; - console.log("total files", filesToProcess); - } - } - }); + let filesProcessed = 0; + let filesToProcess = 0; + let parent = ""; - busboy.on( - "file", - async ( - data: string, - file: Stream, - fileData: { - filename: string; - } + const fileDataMap: dataType = {}; + + const uploadQueue: { file: Stream; index: string }[] = []; + + let processing = false; + + const handleFinish = async ( + filename: string, + metadata: FileMetadateInterface ) => { - const path = fileData.filename; - fileDataMap[path] = { - ...fileDataMap[path], - file, - }; + try { + const date = new Date(); - filesProcessed++; + let length = 0; - if (filesProcessed === filesToProcess) { - resolve(fileDataMap); + if (env.dbType === "fs" && metadata.filePath) { + length = (await getFileSize(metadata.filePath)) as number; + } else { + // TODO: Fix this we should be using the encrypted file size + length = metadata.size; + } + + const currentFile = new File({ + filename, + uploadDate: date.toISOString(), + length, + metadata, + }); + + await currentFile.save(); + + const imageCheck = imageChecker(currentFile.filename); + const videoCheck = videoChecker(currentFile.filename); + + if (videoCheck) { + const updatedFile = await createVideoThumbnail( + currentFile, + filename, + user + ); + return updatedFile; + } else if (currentFile.length < 15728640 && imageCheck) { + const updatedFile = await createThumbnail( + currentFile, + filename, + user + ); + return updatedFile; + } else { + return currentFile; + } + } catch (e: unknown) { + console.log("handle finish error", e); } - // resolve({ - // file, - // filename, - // formData, - // }); - console.log("flr", fileData); - } - ); - }); + }; + + const uploadFile = (currentFile: { file: Stream; index: string }) => { + return new Promise((resolve, reject) => { + const { file, index } = currentFile; + + const fileData = fileDataMap[index]; + + const randomFilenameID = uuid.v4(); + + const password = user.getEncryptionKey(); + + if (!password) throw new ForbiddenError("Invalid Encryption Key"); + + const initVect = crypto.randomBytes(16); + + const CIPHER_KEY = crypto + .createHash("sha256") + .update(password) + .digest(); + + const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect); + + const filename = fileData.name; + const fileSize = fileData.size; + + const metadata = { + owner: user._id.toString(), + parent: "/", + parentList: ["/"].toString(), + hasThumbnail: false, + thumbnailID: "", + isVideo: false, + size: fileSize, + IV: initVect, + processingFile: true, + } as FileMetadateInterface; + + if (env.dbType === "fs") { + metadata.filePath = env.fsDirectory + randomFilenameID; + } else { + metadata.s3ID = randomFilenameID; + } + + const { writeStream, emitter } = storageActions.createWriteStream( + metadata, + file.pipe(cipher), + randomFilenameID + ); + + cipher.pipe(writeStream); + + if (emitter) { + emitter.on("finish", async () => { + const file = await handleFinish(filename, metadata); + resolve(file); + }); + } else { + writeStream.on("finish", async () => { + const file = await handleFinish(filename, metadata); + resolve(file); + }); + } + }); + }; + + const processQueue = async () => { + if (processing) return; + + processing = true; + + while (uploadQueue.length > 0) { + const currentFile = uploadQueue.shift(); + console.log("processing file", currentFile); + const file = await uploadFile(currentFile!); + if (!file) { + // TODO: Handle error + console.log("Error uploading file"); + break; + } + + fileDataMap[currentFile!.index] = { + ...fileDataMap[currentFile!.index], + uploadedFileId: file._id.toString(), + }; + + filesProcessed++; + + console.log("files processed", currentFile); + + if (filesProcessed === filesToProcess) { + resolve({ fileDataMap, parent }); + } + } + processing = false; + }; + + busboy.on("field", (field: any, val: any) => { + console.log("field", field, val); + if (typeof val !== "string" || val !== "undefined") { + formData.set(field, val); + if (field === "file-data") { + const fileData = JSON.parse(val); + fileDataMap[fileData.index] = fileData; + } + if (field === "total-files") { + filesToProcess = +val; + } + if (field === "parent") { + parent = val; + } + } + }); + + busboy.on( + "file", + async ( + _: string, + file: Stream, + fileData: { + filename: string; + } + ) => { + const index = fileData.filename; + // fileDataMap[index] = { + // ...fileDataMap[index], + // file, + // }; + + uploadQueue.push({ file, index }); + + processQueue(); + + // file.on("data", () => { + // console.log("data"); + // }); + + // console.log("file data", fileData); + + // filesProcessed++; + + // if (filesProcessed === filesToProcess) { + // resolve(fileDataMap); + // } + } + ); + + busboy.on("error", (e: Error) => { + reject(e); + }); + } + ); }; export default getFolderBusboyData; diff --git a/backend/utils/createQuery.ts b/backend/utils/createQuery.ts index 3715998..4532059 100644 --- a/backend/utils/createQuery.ts +++ b/backend/utils/createQuery.ts @@ -20,6 +20,7 @@ export interface FileQueryInterface { "metadata.trashed"?: boolean | null; "metadata.hasThumbnail"?: boolean | null; "metadata.isVideo"?: boolean | null; + "metadata.processingFile"?: boolean | null; } export const createFileQuery = ({ @@ -67,6 +68,8 @@ export const createFileQuery = ({ } } + query["metadata.processingFile"] = null; + return query; }; diff --git a/src/hooks/files.ts b/src/hooks/files.ts index 0c029bb..af7dd67 100644 --- a/src/hooks/files.ts +++ b/src/hooks/files.ts @@ -296,6 +296,7 @@ export const useUploader = () => { size: currentFile.size, type: currentFile.type, path: currentFile.webkitRelativePath, + index: i, }) ); } @@ -305,7 +306,7 @@ export const useUploader = () => { for (let i = 0; i < files.length; i++) { const currentFile = files[i]; console.log("current file", currentFile.webkitRelativePath); - data.append("file", currentFile, "test"); + data.append("file", currentFile, i.toString()); } const CancelToken = axiosNonInterceptor.CancelToken;