diff --git a/backend/controllers/file-controller.ts b/backend/controllers/file-controller.ts index affe947..5a2f527 100644 --- a/backend/controllers/file-controller.ts +++ b/backend/controllers/file-controller.ts @@ -141,7 +141,8 @@ class FileController { const handleError = () => { if (!responseSent) { responseSent = true; - res.status(500).send("Server error uploading file"); + res.writeHead(500, { Connection: "close" }); + res.end(); } }; @@ -215,7 +216,7 @@ class FileController { req.pipe(busboy); - const { cipher, fileWriteStream, metadata, filename } = + const { cipher, fileWriteStream, emitter, metadata, filename } = await this.chunkService.uploadFile(user, busboy, req); cipher.on("error", (e: Error) => { @@ -228,9 +229,17 @@ class FileController { handleError(); }); - cipher.pipe(fileWriteStream).on("finish", async () => { - handleFinish(filename, metadata); - }); + if (emitter) { + emitter.on("finish", async () => { + await handleFinish(filename, metadata); + }); + } else { + fileWriteStream.on("finish", async () => { + await handleFinish(filename, metadata); + }); + } + + cipher.pipe(fileWriteStream); } catch (e: unknown) { if (!responseSent) { res.writeHead(500, { Connection: "close" }); @@ -616,18 +625,10 @@ class FileController { readStream.on("error", (e: Error) => { console.log("read stream error", e); - if (!responseSent) { - responseSent = true; - res.status(500).send("Server error downloading file"); - } }); decipher.on("error", (e: Error) => { - console.log("decipher stream error", e); - if (!responseSent) { - responseSent = true; - res.status(500).send("Server error downloading file"); - } + console.log("decipher stream errozr", e); }); res.set("Content-Type", "binary/octet-stream"); @@ -642,10 +643,6 @@ class FileController { if (e instanceof Error) { console.log("\nDownload File Error File Route:", e.message); } - if (!responseSent) { - responseSent = true; - res.status(500).send("Server error downloading file"); - } } }; diff --git a/backend/services/chunk-service/actions/S3-actions.ts b/backend/services/chunk-service/actions/S3-actions.ts index 736d0d9..e959725 100644 --- a/backend/services/chunk-service/actions/S3-actions.ts +++ b/backend/services/chunk-service/actions/S3-actions.ts @@ -1,8 +1,9 @@ import s3 from "../../../db/connections/s3"; import env from "../../../enviroment/env"; import { GenericParams, IStorageActions } from "../store-types"; -import internal from "stream"; +import internal, { EventEmitter } from "stream"; import { PassThrough } from "stream"; +import stream from "stream"; class S3Actions implements IStorageActions { getAuth() { @@ -65,23 +66,25 @@ class S3Actions implements IStorageActions { } createWriteStream = ( params: GenericParams, - stream: NodeJS.ReadableStream, + readStream: NodeJS.ReadableStream, randomID: string ) => { - const pass = new PassThrough(); + const passThrough = new stream.PassThrough(); + const emitter = new EventEmitter(); - const { Key } = params; - if (!Key) throw new Error("S3 not configured"); const { s3Storage, bucket } = this.getAuth(); - s3Storage.upload({ Bucket: bucket, Body: stream, Key: randomID }, (err) => { - if (err) { - console.log("S3 upload err", err); - pass.emit("error", err); + s3Storage.upload( + { Bucket: bucket, Body: passThrough, Key: randomID }, + (err) => { + if (err) { + console.log("S3 upload err", err); + } + emitter.emit("finish"); } - }); + ); - return pass; + return { writeStream: passThrough, emitter }; }; } diff --git a/backend/services/chunk-service/actions/file-system-actions.ts b/backend/services/chunk-service/actions/file-system-actions.ts index de4c3fd..f184c34 100644 --- a/backend/services/chunk-service/actions/file-system-actions.ts +++ b/backend/services/chunk-service/actions/file-system-actions.ts @@ -61,7 +61,10 @@ class FilesystemActions implements IStorageActions { randomID: string ) => { const path = `${env.fsDirectory}${randomID}`; - return fs.createWriteStream(path); + return { + writeStream: fs.createWriteStream(path), + emitter: null, + }; }; } diff --git a/backend/services/chunk-service/chunk-service.ts b/backend/services/chunk-service/chunk-service.ts index 3728412..f2fa668 100644 --- a/backend/services/chunk-service/chunk-service.ts +++ b/backend/services/chunk-service/chunk-service.ts @@ -19,6 +19,7 @@ import { createGenericParams } from "./utils/storageHelper"; import { Readable } from "stream"; import ThumbnailDB from "../../db/mongoDB/thumbnailDB"; import UserDB from "../../db/mongoDB/userDB"; +import fixEndChunkLength from "./utils/fixEndChunkLength"; const fileDB = new FileDB(); const folderDB = new FolderDB(); @@ -111,7 +112,7 @@ class StorageService { metadata.s3ID = randomFilenameID; } - const fileWriteStream = storageActions.createWriteStream( + const { writeStream, emitter } = storageActions.createWriteStream( metadata, file.pipe(cipher), randomFilenameID @@ -119,7 +120,8 @@ class StorageService { return { cipher, - fileWriteStream, + fileWriteStream: writeStream, + emitter, metadata, filename, }; @@ -245,7 +247,10 @@ class StorageService { let currentIV = IV; let fixedStart = 0; - let fixedEnd = currentFile.length; + let fixedEnd = + currentFile.length % 16 === 0 + ? currentFile.length + : fixEndChunkLength(currentFile.length); if (start === 0 && end === 1) { fixedStart = 0; diff --git a/backend/services/chunk-service/utils/createImageThumbnail.ts b/backend/services/chunk-service/utils/createImageThumbnail.ts index 1aa3013..3c0885d 100644 --- a/backend/services/chunk-service/utils/createImageThumbnail.ts +++ b/backend/services/chunk-service/utils/createImageThumbnail.ts @@ -4,7 +4,7 @@ import Thumbnail from "../../../models/thumbnail-model"; import sharp from "sharp"; import { FileInterface } from "../../../models/file-model"; import { UserInterface } from "../../../models/user-model"; -import fs from "fs"; +import fs, { write } from "fs"; import uuid from "uuid"; import env from "../../../enviroment/env"; import { ObjectId } from "mongodb"; @@ -18,12 +18,20 @@ const conn = mongoose.connection; const storageActions = env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); +const waitTimer = () => { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve(null); + }, 5000); + }); +}; + const createThumbnail = ( file: FileInterface, filename: string, user: UserInterface ) => { - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { try { const password = user.getEncryptionKey(); @@ -33,6 +41,12 @@ const createThumbnail = ( const thumbnailIV = crypto.randomBytes(16); + console.log("file metadata", file.metadata); + + // if (env.dbType === "s3") { + // await waitTimer(); + // } + const params = createGenericParams({ filePath: file.metadata.filePath, Key: file.metadata.s3ID, @@ -40,7 +54,7 @@ const createThumbnail = ( const readStream = storageActions.createReadStream(params); - const writeStream = storageActions.createWriteStream( + const { writeStream, emitter } = storageActions.createWriteStream( params, readStream, thumbnailFilename @@ -61,6 +75,7 @@ const createThumbnail = ( owner: user._id, IV: thumbnailIV, path: env.fsDirectory + thumbnailFilename, + s3ID: thumbnailFilename, }); await thumbnailModel.save(); @@ -119,9 +134,15 @@ const createThumbnail = ( .pipe(thumbnailCipher) .pipe(writeStream); - writeStream.on("finish", async () => { - await handleFinish(); - }); + if (emitter) { + emitter.on("finish", async () => { + await handleFinish(); + }); + } else { + writeStream.on("finish", async () => { + await handleFinish(); + }); + } } catch (e) { console.log("File service upload thumbnail error", e); resolve(file); diff --git a/backend/services/chunk-service/utils/createVideoThumbnail.ts b/backend/services/chunk-service/utils/createVideoThumbnail.ts index 5fba7b3..0100cbb 100644 --- a/backend/services/chunk-service/utils/createVideoThumbnail.ts +++ b/backend/services/chunk-service/utils/createVideoThumbnail.ts @@ -38,7 +38,7 @@ const createVideoThumbnail = ( const readStream = storageActions.createReadStream(readStreamParams); - const writeStream = storageActions.createWriteStream( + const { writeStream } = storageActions.createWriteStream( readStreamParams, readStream, thumbnailFilename @@ -80,6 +80,7 @@ const createVideoThumbnail = ( owner: user._id, IV: thumbnailIV, path: env.fsDirectory + thumbnailFilename, + s3ID: thumbnailFilename, }); await thumbnailModel.save(); @@ -106,8 +107,6 @@ const createVideoThumbnail = ( if (!updatedFile) return reject(); - console.log("updated file", updatedFile, resolve); - resolve(updatedFile?.toObject()); } catch (e) { console.log("thumbnail error", e); @@ -129,7 +128,7 @@ const createVideoThumbnail = ( .on("end", async () => { await handleFinish(); }) - .on("error", async (err, _, stderr) => { + .on("error", async (err, sdf, stderr) => { console.log("thumbnail error attempting temp directory fix"); if (env.tempDirectory) { const updatedFile = await tempCreateVideoThumbnailFS( @@ -141,6 +140,7 @@ const createVideoThumbnail = ( } else { resolve(file); } + // resolve(file); }) .pipe(thumbnailCipher) .pipe(writeStream, { end: true }); diff --git a/backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts b/backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts index 993e7ad..5c47da6 100644 --- a/backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts +++ b/backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts @@ -10,6 +10,12 @@ import env from "../../../enviroment/env"; import { ObjectId } from "mongodb"; import File from "../../../models/file-model"; import ffmpeg from "fluent-ffmpeg"; +import { S3Actions } from "../actions/S3-actions"; +import { FilesystemActions } from "../actions/file-system-actions"; +import { createGenericParams } from "./storageHelper"; + +const storageActions = + env.dbType === "s3" ? new S3Actions() : new FilesystemActions(); const tempCreateVideoThumbnailFS = ( file: FileInterface, @@ -23,13 +29,20 @@ const tempCreateVideoThumbnailFS = ( const thumbnailFilename = uuid.v4(); - const readStream = fs.createReadStream(file.metadata.filePath!); - console.log("env", env); - const writeStream = fs.createWriteStream( - env.fsDirectory + thumbnailFilename + const readStreamParams = createGenericParams({ + filePath: file.metadata.filePath, + Key: file.metadata.s3ID, + }); + + const readStream = storageActions.createReadStream(readStreamParams); + + const { writeStream } = storageActions.createWriteStream( + readStreamParams, + readStream, + thumbnailFilename ); - const tempDirectory = env.fsDirectory + "temp/" + thumbnailFilename; + const tempDirectory = env.tempDirectory + thumbnailFilename; const tempWriteStream = fs.createWriteStream(tempDirectory); const decipher = crypto.createDecipheriv( "aes256", @@ -49,105 +62,107 @@ const tempCreateVideoThumbnailFS = ( decryptedReadStream.pipe(tempWriteStream, { end: true }); - ffmpeg(tempDirectory, { - timeout: 60, - }) - .seek(1) - .format("image2pipe") - .outputOptions([ - "-f image2pipe", - "-vframes 1", - "-vf scale='if(gt(iw,ih),600,-1):if(gt(ih,iw),300,-1)'", - ]) - .on("start", (command) => { - /** - * log - */ + tempWriteStream.on("finish", () => { + ffmpeg(tempDirectory, { + timeout: 60, }) - .on("end", async () => { - console.log("end"); - const thumbnailModel = new Thumbnail({ - name: filename, - owner: user._id, - IV: thumbnailIV, - path: env.fsDirectory + thumbnailFilename, - }); + .seek(1) + .format("image2pipe") + .outputOptions([ + "-f image2pipe", + "-vframes 1", + "-vf scale='if(gt(iw,ih),600,-1):if(gt(ih,iw),300,-1)'", + ]) + .on("start", (command) => { + /** + * log + */ + }) + .on("end", async () => { + const thumbnailModel = new Thumbnail({ + name: filename, + owner: user._id, + IV: thumbnailIV, + path: env.fsDirectory + thumbnailFilename, + s3ID: thumbnailFilename, + }); - await thumbnailModel.save(); - if (!file._id) { - return reject(); - } - const updateFileResponse = await File.updateOne( - { _id: new ObjectId(file._id), "metadata.owner": user._id }, - { - $set: { - "metadata.hasThumbnail": true, - "metadata.thumbnailID": thumbnailModel._id, - }, + await thumbnailModel.save(); + if (!file._id) { + return reject(); + } + const updateFileResponse = await File.updateOne( + { _id: new ObjectId(file._id), "metadata.owner": user._id }, + { + $set: { + "metadata.hasThumbnail": true, + "metadata.thumbnailID": thumbnailModel._id, + }, + } + ); + if (updateFileResponse.modifiedCount === 0) { + return reject(); } - ); - if (updateFileResponse.modifiedCount === 0) { - return reject(); - } - const updatedFile = await File.findById({ - _id: new ObjectId(file._id), - "metadata.owner": user._id, - }); + const updatedFile = await File.findById({ + _id: new ObjectId(file._id), + "metadata.owner": user._id, + }); - if (!updatedFile) return reject(); + if (!updatedFile) return reject(); - fs.unlink(tempDirectory, (err) => { - resolve(updatedFile?.toObject()); - }); - }) - .on("error", (err, _, stderr) => { - console.log("error", err, stderr); - resolve(file); - /** - * log - */ - }) - .pipe(thumbnailCipher) - .pipe(writeStream, { end: true }); + fs.unlink(tempDirectory, (err) => { + resolve(updatedFile?.toObject()); + }); + }) + .on("error", (err, _, stderr) => { + console.log("error", err, stderr); + resolve(file); + /** + * log + */ + }) + .pipe(thumbnailCipher) + .pipe(writeStream, { end: true }); - // ffmpeg() - // .input(readStream.pipe(decipher)) - // .seekInput("00:00:01.00") - // .outputFormat("image2") - // .pipe(writeStream, { end: true }) - // .on("end", async () => { - // const thumbnailModel = new Thumbnail({ - // name: filename, - // owner: user._id, - // IV: thumbnailIV, - // path: env.fsDirectory + thumbnailFilename, - // }); + // ffmpeg() + // .input(readStream.pipe(decipher)) + // .seekInput("00:00:01.00") + // .outputFormat("image2") + // .pipe(writeStream, { end: true }) + // .on("end", async () => { + // const thumbnailModel = new Thumbnail({ + // name: filename, + // owner: user._id, + // IV: thumbnailIV, + // path: env.fsDirectory + thumbnailFilename, + // }); - // await thumbnailModel.save(); - // resolve(file); - // }) - // .on("error", (e) => { - // console.log("error", e); - // reject(e); - // }); + // await thumbnailModel.save(); + // resolve(file); + // }) + // .on("error", (e) => { + // console.log("error", e); + // reject(e); + // }); - // ffmpeg() - // .input(readStream.pipe(decipher)) - // .size("300x?") - // .outputOptions("-frames:v 1") - // .on("end", async () => { - // const thumbnailModel = new Thumbnail({ - // name: filename, - // owner: user._id, - // IV: thumbnailIV, - // path: env.fsDirectory + thumbnailFilename, - // }); + // ffmpeg() + // .input(readStream.pipe(decipher)) + // .size("300x?") + // .outputOptions("-frames:v 1") + // .on("end", async () => { + // const thumbnailModel = new Thumbnail({ + // name: filename, + // owner: user._id, + // IV: thumbnailIV, + // path: env.fsDirectory + thumbnailFilename, + // }); - // await thumbnailModel.save(); - // resolve(file); - // }) - // .output(writeStream); + // await thumbnailModel.save(); + // resolve(file); + // }) + // .output(writeStream); + }); }); }; diff --git a/backend/utils/createQuery.ts b/backend/utils/createQuery.ts index ffc0b74..edbec1b 100644 --- a/backend/utils/createQuery.ts +++ b/backend/utils/createQuery.ts @@ -35,7 +35,7 @@ export const createFileQuery = ({ if (search && search !== "") { query["filename"] = new RegExp(search, "i"); - } else { + } else if (!mediaMode) { query["metadata.parent"] = parent; } @@ -58,6 +58,7 @@ export const createFileQuery = ({ if (mediaMode) { query["metadata.hasThumbnail"] = true; } + return query; }; diff --git a/serverUtils/migrateMyDrive4.js b/serverUtils/migrateMyDrive4.js new file mode 100644 index 0000000..64ac0d5 --- /dev/null +++ b/serverUtils/migrateMyDrive4.js @@ -0,0 +1,47 @@ +const getEnvVariables = require("../dist/enviroment/get-env-variables"); +getEnvVariables(); +const mongoose = require("./mongoServerUtil"); +const conn = mongoose.connection; +const File = require("../dist/models/file-model"); + +const waitForDatabase = () => { + return new Promise((resolve, reject) => { + if (conn.readyState !== 1) { + conn.once("open", () => { + resolve(); + }); + } else { + resolve(); + } + }); +}; + +const updateDocs = async () => { + console.log("\nWaiting for database..."); + await waitForDatabase(); + console.log("Connected to database\n"); + + console.log("Getting file list..."); + const files = await File.find({}); + console.log("Found", files.length, "files"); + + for (let i = 0; i < files.length; i++) { + const currentFile = files[i]; + + await File.updateOne( + { _id: currentFile._id }, + { + $set: { + "metadata.owner": currentFile.metadata.owner.toString(), + "metadata.thumbnailID": currentFile.metadata.thumbnailID.toString(), + }, + } + ); + } + + console.log("Done"); + + process.exit(); +}; + +updateDocs(); diff --git a/serverUtils/mongoServerUtil.js b/serverUtils/mongoServerUtil.js index 5a8c31a..49c0f1e 100644 --- a/serverUtils/mongoServerUtil.js +++ b/serverUtils/mongoServerUtil.js @@ -1,11 +1,5 @@ const mongoose = require("mongoose"); -mongoose.connect(process.env.MONGODB_URL, { - useNewUrlParser: true, - useCreateIndex: true, - useUnifiedTopology: true, - socketTimeoutMS: 30000000, - keepAlive: true, -}) +mongoose.connect(process.env.MONGODB_URL, {}); -module.exports = mongoose; \ No newline at end of file +module.exports = mongoose; diff --git a/src/components/Folders/Folders.tsx b/src/components/Folders/Folders.tsx index 8700bb2..522284f 100644 --- a/src/components/Folders/Folders.tsx +++ b/src/components/Folders/Folders.tsx @@ -67,7 +67,7 @@ const Folders = memo(() => { return (
-
+

{ const drawerOpen = useAppSelector((state) => state.leftSection.drawOpen); @@ -19,25 +21,43 @@ const Header = () => { }, [toggleDrawer] ); + + const closeDrawerClick = useCallback( + (e: any) => { + e.stopPropagation(); + dispatch(closeDrawer()); + }, + [closeDrawer] + ); + return (