diff --git a/backend/controllers/folder-controller.ts b/backend/controllers/folder-controller.ts index 9c599b9..dbbc999 100644 --- a/backend/controllers/folder-controller.ts +++ b/backend/controllers/folder-controller.ts @@ -85,32 +85,11 @@ class FolderController { const user = req.user; const busboy = req.busboy; - console.log("upload folder request"); - - busboy.on("error", (e: Error) => { - console.log("busboy error", e); - // handleError(); - }); - - req.on("end", () => { - console.log("end req"); - }); - - req.on("close", () => { - console.log("close req"); - }); - - req.on("error", () => { - console.log("error req"); - }); - - req.pipe(busboy); - - await chunkService.uploadFolder(user, busboy, req); + await chunkService.uploadFolder(user, busboy, req, next); res.send(); } catch (e) { - console.log("upload folder error", e); + next(e); } }; diff --git a/backend/services/chunk-service/chunk-service.ts b/backend/services/chunk-service/chunk-service.ts index 021acb4..25f0548 100644 --- a/backend/services/chunk-service/chunk-service.ts +++ b/backend/services/chunk-service/chunk-service.ts @@ -1,4 +1,4 @@ -import { Response, Request } from "express"; +import { Response, Request, NextFunction } from "express"; import { UserInterface } from "../../models/user-model"; import NotAuthorizedError from "../../utils/NotAuthorizedError"; import NotFoundError from "../../utils/NotFoundError"; @@ -130,7 +130,24 @@ class StorageService { }; }; - uploadFolder = async (user: UserInterface, busboy: any, req: Request) => { + uploadFolder = async ( + user: UserInterface, + busboy: any, + req: Request, + next: NextFunction + ) => { + req.on("error", (e: Error) => { + console.log("req error", e); + next(e); + }); + + busboy.on("error", (e: Error) => { + console.log("busboy error", e); + next(e); + }); + + req.pipe(busboy); + const { fileDataMap, parent } = await getFolderBusboyData(busboy, user); const keys = Object.keys(fileDataMap); diff --git a/backend/services/chunk-service/utils/getBusboyData.ts b/backend/services/chunk-service/utils/getBusboyData.ts index 9e50586..9e70a21 100644 --- a/backend/services/chunk-service/utils/getBusboyData.ts +++ b/backend/services/chunk-service/utils/getBusboyData.ts @@ -1,19 +1,18 @@ import { Stream } from "stream"; -const getBusboyData = (busboy: any) => { - type dataType = { - file: Stream; - filename: { - filename: string; - }; - formData: Map; +type dataType = { + file: Stream; + filename: { + filename: string; }; + formData: Map; +}; +const getBusboyData = (busboy: any) => { return new Promise((resolve, reject) => { const formData = new Map(); busboy.on("field", (field: any, val: any) => { - // ????? why does it make undefined a string? if (typeof val !== "string" || val !== "undefined") { formData.set(field, val); } @@ -21,7 +20,7 @@ const getBusboyData = (busboy: any) => { busboy.on( "file", - async ( + ( _: string, file: Stream, filename: { diff --git a/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts b/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts index a313f1e..87e3350 100644 --- a/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts +++ b/backend/services/chunk-service/utils/getFolderUploadBusboyData.ts @@ -15,6 +15,7 @@ import imageChecker from "../../../utils/imageChecker"; import videoChecker from "../../../utils/videoChecker"; import createVideoThumbnail from "./createVideoThumbnail"; import createThumbnail from "./createImageThumbnail"; +import { EventEmitter } from "events"; type FileDataType = { name: string; @@ -29,75 +30,69 @@ type FileDataType = { const storageActions = env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); -const getFolderBusboyData = (busboy: any, user: UserInterface) => { - type dataType = Record; +type dataType = Record; - return new Promise<{ fileDataMap: dataType; parent: string }>( - (resolve, reject) => { - const formData = new Map(); +const processData = (busboy: any, user: UserInterface) => { + const eventEmitter = new EventEmitter(); - let filesProcessed = 0; - let filesToProcess = 0; - let parent = ""; + try { + const formData = new Map(); - const fileDataMap: dataType = {}; + let filesProcessed = 0; + let filesToProcess = 0; + let parent = ""; - const uploadQueue: { file: Stream; index: string }[] = []; + const fileDataMap: dataType = {}; - let processing = false; + const uploadQueue: { file: Stream; index: string }[] = []; - const handleFinish = async ( - filename: string, - metadata: FileMetadateInterface - ) => { - try { - const date = new Date(); + let processing = false; - let length = 0; + const handleFinish = async ( + filename: string, + metadata: FileMetadateInterface + ) => { + const date = new Date(); - if (env.dbType === "fs" && metadata.filePath) { - length = (await getFileSize(metadata.filePath)) as number; - } else { - // TODO: Fix this we should be using the encrypted file size - length = metadata.size; - } + let length = 0; - const currentFile = new File({ - filename, - uploadDate: date.toISOString(), - length, - metadata, - }); + if (env.dbType === "fs" && metadata.filePath) { + length = (await getFileSize(metadata.filePath)) as number; + } else { + // TODO: Fix this we should be using the encrypted file size + length = metadata.size; + } - await currentFile.save(); + const currentFile = new File({ + filename, + uploadDate: date.toISOString(), + length, + metadata, + }); - const imageCheck = imageChecker(currentFile.filename); - const videoCheck = videoChecker(currentFile.filename); + await currentFile.save(); - if (videoCheck) { - const updatedFile = await createVideoThumbnail( - currentFile, - filename, - user - ); - return updatedFile; - } else if (currentFile.length < 15728640 && imageCheck) { - const updatedFile = await createThumbnail( - currentFile, - filename, - user - ); - return updatedFile; - } else { - return currentFile; - } - } catch (e: unknown) { - console.log("handle finish error", e); - } - }; + const imageCheck = imageChecker(currentFile.filename); + const videoCheck = videoChecker(currentFile.filename); - const uploadFile = (currentFile: { file: Stream; index: string }) => { - return new Promise((resolve, reject) => { + if (videoCheck) { + const updatedFile = await createVideoThumbnail( + currentFile, + filename, + user + ); + return updatedFile; + } else if (currentFile.length < 15728640 && imageCheck) { + const updatedFile = await createThumbnail(currentFile, filename, user); + return updatedFile; + } else { + return currentFile; + } + }; + + const uploadFile = (currentFile: { file: Stream; index: string }) => { + return new Promise<{ filename: string; metadata: FileMetadateInterface }>( + (resolve, reject) => { const { file, index } = currentFile; const fileData = fileDataMap[index]; @@ -144,36 +139,41 @@ const getFolderBusboyData = (busboy: any, user: UserInterface) => { randomFilenameID ); + writeStream.on("error", (e: Error) => { + console.log("write stream error", e); + reject(e); + }); + + cipher.on("error", (e: Error) => { + console.log("cipher error", e); + reject(e); + }); + cipher.pipe(writeStream); if (emitter) { - emitter.on("finish", async () => { - const file = await handleFinish(filename, metadata); - resolve(file); + emitter.on("finish", () => { + resolve({ filename, metadata }); }); } else { - writeStream.on("finish", async () => { - const file = await handleFinish(filename, metadata); - resolve(file); + writeStream.on("finish", () => { + resolve({ filename, metadata }); }); } - }); - }; + } + ); + }; - const processQueue = async () => { - if (processing) return; + const processQueue = async () => { + if (processing) return; - processing = true; + processing = true; + try { while (uploadQueue.length > 0) { const currentFile = uploadQueue.shift(); - console.log("processing file", currentFile); - const file = await uploadFile(currentFile!); - if (!file) { - // TODO: Handle error - console.log("Error uploading file"); - break; - } + const { filename, metadata } = await uploadFile(currentFile!); + const file = await handleFinish(filename, metadata); fileDataMap[currentFile!.index] = { ...fileDataMap[currentFile!.index], @@ -182,66 +182,71 @@ const getFolderBusboyData = (busboy: any, user: UserInterface) => { filesProcessed++; - console.log("files processed", currentFile); - if (filesProcessed === filesToProcess) { - resolve({ fileDataMap, parent }); + eventEmitter.emit("finish", { fileDataMap, parent }); } } - processing = false; - }; + } catch (e) { + console.log("error", e); + eventEmitter.emit("error", e); + } - busboy.on("field", (field: any, val: any) => { - console.log("field", field, val); - if (typeof val !== "string" || val !== "undefined") { - formData.set(field, val); - if (field === "file-data") { - const fileData = JSON.parse(val); - fileDataMap[fileData.index] = fileData; - } - if (field === "total-files") { - filesToProcess = +val; - } - if (field === "parent") { - parent = val; - } + processing = false; + }; + + busboy.on("field", (field: any, val: any) => { + if (typeof val !== "string" || val !== "undefined") { + formData.set(field, val); + if (field === "file-data") { + const fileData = JSON.parse(val); + fileDataMap[fileData.index] = fileData; } + if (field === "total-files") { + filesToProcess = +val; + } + if (field === "parent") { + parent = val; + } + } + }); + + busboy.on( + "file", + async ( + _: string, + file: Stream, + fileData: { + filename: string; + } + ) => { + const index = fileData.filename; + + uploadQueue.push({ file, index }); + + processQueue(); + } + ); + + busboy.on("error", (e: Error) => { + console.log("busboy error", e); + eventEmitter.emit("error", e); + }); + } catch (e) { + console.log("get folder busboy data error", e); + eventEmitter.emit("error", e); + } + + return eventEmitter; +}; + +const getFolderBusboyData = (busboy: any, user: UserInterface) => { + return new Promise<{ fileDataMap: dataType; parent: string }>( + (resolve, reject) => { + const fileEventEmitter = processData(busboy, user); + fileEventEmitter.on("finish", (data) => { + resolve(data); }); - - busboy.on( - "file", - async ( - _: string, - file: Stream, - fileData: { - filename: string; - } - ) => { - const index = fileData.filename; - // fileDataMap[index] = { - // ...fileDataMap[index], - // file, - // }; - - uploadQueue.push({ file, index }); - - processQueue(); - - // file.on("data", () => { - // console.log("data"); - // }); - - // console.log("file data", fileData); - - // filesProcessed++; - - // if (filesProcessed === filesToProcess) { - // resolve(fileDataMap); - // } - } - ); - - busboy.on("error", (e: Error) => { + fileEventEmitter.on("error", (e) => { reject(e); }); } diff --git a/src/components/UploadItem/UploadItem.tsx b/src/components/UploadItem/UploadItem.tsx index 648ede5..3e57792 100644 --- a/src/components/UploadItem/UploadItem.tsx +++ b/src/components/UploadItem/UploadItem.tsx @@ -1,6 +1,5 @@ import { connect } from "react-redux"; -import React, { memo, useEffect, useRef } from "react"; -import { useFilesClient, useQuickFilesClient } from "../../hooks/files"; +import React, { memo } from "react"; import { getCancelToken } from "../../utils/cancelTokenManager"; import CloseIcon from "../../icons/CloseIcon"; import CheckCircleIcon from "../../icons/CheckCircleIcon"; @@ -30,6 +29,20 @@ const UploadItem: React.FC = (props) => { } }); + const ProgressBar = memo(() => { + if (completed) { + return
; + } else if (type === "file") { + return ( + + ); + } else if (canceled) { + return
; + } else { + return ; + } + }); + return (
@@ -44,12 +57,7 @@ const UploadItem: React.FC = (props) => {
- {(type === "file" || completed) && ( - - )} - {type === "folder" && !completed && ( - - )} +
diff --git a/src/styles/components/_Spinner.scss b/src/styles/components/_Spinner.scss index 0174109..4f29a3a 100644 --- a/src/styles/components/_Spinner.scss +++ b/src/styles/components/_Spinner.scss @@ -74,6 +74,26 @@ border: none; } +.custom-progress-failed { + width: 100%; + height: 5px; + appearance: none; + border-radius: 3px; + border: none; + background-color: red; + margin-top: 15px; +} + +.custom-progress-success { + width: 100%; + height: 5px; + appearance: none; + border-radius: 3px; + border: none; + background-color: green; + margin-top: 15px; +} + .custom-progress.indeterminate { position: relative; overflow: hidden;