From 4821c5a40883a1015d0e85594dda1d12ed1ebb78 Mon Sep 17 00:00:00 2001 From: subnub Date: Fri, 17 Apr 2020 00:13:12 -0400 Subject: [PATCH] Added path option for storing files on file system, started to add s3 support --- backend/controllers/file.ts | 2 +- backend/controllers/fileSystem.ts | 2 +- backend/db/s3.ts | 6 +++++ backend/enviroment/env.js | 3 ++- .../ChunkService/FileSystemService.ts | 23 +++++++++++++++- backend/services/ChunkService/MongoService.ts | 25 ++++++++++++++++++ .../ChunkService/utils/awaitUploadStreamFS.ts | 9 ++++--- backend/services/FileService/indexnew.ts | 26 ------------------- .../FileService/utils/createThumbnailFS.ts | 6 ++--- package.json | 1 + 10 files changed, 66 insertions(+), 37 deletions(-) create mode 100644 backend/db/s3.ts diff --git a/backend/controllers/file.ts b/backend/controllers/file.ts index cee7a0d..ccc179e 100644 --- a/backend/controllers/file.ts +++ b/backend/controllers/file.ts @@ -578,7 +578,7 @@ class FileController { const userID = req.user._id; const fileID = req.body.id; - await fileService.deleteFile(userID, fileID); + await mongoService.deleteFile(userID, fileID); res.send() diff --git a/backend/controllers/fileSystem.ts b/backend/controllers/fileSystem.ts index 7ecb22f..cd546fd 100644 --- a/backend/controllers/fileSystem.ts +++ b/backend/controllers/fileSystem.ts @@ -576,7 +576,7 @@ class FileSystemController { const userID = req.user._id; const fileID = req.body.id; - await fileService.deleteFile(userID, fileID); + await fileSystemService.deleteFile(userID, fileID); res.send() diff --git a/backend/db/s3.ts b/backend/db/s3.ts new file mode 100644 index 0000000..73dd6b3 --- /dev/null +++ b/backend/db/s3.ts @@ -0,0 +1,6 @@ +import AWS from "aws-sdk"; + +AWS.config.update({ + accessKeyId: "", + secretAccessKey: "" +}); \ No newline at end of file diff --git a/backend/enviroment/env.js b/backend/enviroment/env.js index 309fd68..c72bc5d 100644 --- a/backend/enviroment/env.js +++ b/backend/enviroment/env.js @@ -6,5 +6,6 @@ module.exports = { root: process.env.ROOT, url: process.env.URL, mongoURL: process.env.MONGODB_URL, - dbType: process.env.DB_TYPE + dbType: process.env.DB_TYPE, + fsDirectory: process.env.FS_DIRECTORY } \ No newline at end of file diff --git a/backend/services/ChunkService/FileSystemService.ts b/backend/services/ChunkService/FileSystemService.ts index 98b9e91..bee4730 100644 --- a/backend/services/ChunkService/FileSystemService.ts +++ b/backend/services/ChunkService/FileSystemService.ts @@ -18,6 +18,8 @@ import imageChecker from "../../utils/imageChecker"; import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail"; import streamToBuffer from "../../utils/streamToBuffer"; import User from "../../models/user"; +import env from "../../enviroment/env"; +import { removeChunksFS } from "./utils/awaitUploadStreamFS"; const dbUtilsFile = new DbUtilFile(); @@ -62,7 +64,7 @@ class FileSystemService { isVideo, size, IV: initVect, - filePath: `/Users/kylehoell/Documents/fstestdata/${systemFileName}` + filePath: env.fsDirectory + systemFileName } const fileWriteStream = fs.createWriteStream(metadata.filePath); @@ -255,6 +257,25 @@ class FileSystemService { await dbUtilsFile.removeOneTimePublicLink(fileID); } } + + deleteFile = async(userID: string, fileID: string) => { + + const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID); + + if (!file) throw new NotFoundError("Delete File Not Found Error"); + + if (file.metadata.thumbnailID) { + + const thumbnail = await Thumbnail.findById(file.metadata.thumbnailID) as ThumbnailInterface; + const thumbnailPath = thumbnail.path!; + await removeChunksFS(thumbnailPath); + + await Thumbnail.deleteOne({_id: file.metadata.thumbnailID}); + } + + await removeChunksFS(file.metadata.filePath!); + await File.deleteOne({_id: file._id}); + } } export default FileSystemService; \ No newline at end of file diff --git a/backend/services/ChunkService/MongoService.ts b/backend/services/ChunkService/MongoService.ts index 2fe6b6c..16a39f3 100644 --- a/backend/services/ChunkService/MongoService.ts +++ b/backend/services/ChunkService/MongoService.ts @@ -250,6 +250,31 @@ class MongoService implements ChunkInterface { await awaitStream(readStream.pipe(decipher), res); } + deleteFile = async(userID: string, fileID: string) => { + + let bucket = new mongoose.mongo.GridFSBucket(conn.db, { + chunkSizeBytes: 1024 * 255, + }); + + const file = await dbUtilsFile.getFileInfo(fileID, userID); + + if (!file) throw new NotFoundError("Delete File Not Found Error"); + + if (file.metadata.thumbnailID) { + + await Thumbnail.deleteOne({_id: file.metadata.thumbnailID}); + } + + if (file.metadata.isVideo && file.metadata.transcoded) { + try { + await bucket.delete(new ObjectID(file.metadata.transcodedID)); + } catch (e) { + console.log("Could Not Find Transcoded Video"); + } + } + + await bucket.delete(new ObjectID(fileID)); + } } export default MongoService; \ No newline at end of file diff --git a/backend/services/ChunkService/utils/awaitUploadStreamFS.ts b/backend/services/ChunkService/utils/awaitUploadStreamFS.ts index cdec25f..64ec29b 100644 --- a/backend/services/ChunkService/utils/awaitUploadStreamFS.ts +++ b/backend/services/ChunkService/utils/awaitUploadStreamFS.ts @@ -1,7 +1,7 @@ import {Request} from "express" import fs from "fs"; -const removeChunks = (path: string) => { +export const removeChunksFS = (path: string) => { return new Promise((resolve, reject) => { @@ -12,6 +12,7 @@ const removeChunks = (path: string) => { } console.log("File Removed"); + resolve(); }) }) } @@ -22,7 +23,7 @@ const awaitUploadStream = (inputSteam: any, outputStream: any, req: Request, inputSteam.on("error", (e: Error) => { - removeChunks(path); + removeChunksFS(path); reject({ message: "Await Stream Input Error", @@ -33,7 +34,7 @@ const awaitUploadStream = (inputSteam: any, outputStream: any, req: Request, outputStream.on("error", (e: Error) => { - removeChunks(path); + removeChunksFS(path); reject({ message: "Await Stream Output Error", @@ -46,7 +47,7 @@ const awaitUploadStream = (inputSteam: any, outputStream: any, req: Request, console.log("Upload Request Cancelling..."); - removeChunks(path); + removeChunksFS(path); }) inputSteam.pipe(outputStream).on("finish", (data: T) => { diff --git a/backend/services/FileService/indexnew.ts b/backend/services/FileService/indexnew.ts index 90338ed..7c925ea 100644 --- a/backend/services/FileService/indexnew.ts +++ b/backend/services/FileService/indexnew.ts @@ -231,32 +231,6 @@ class MongoFileService { return file; } - - deleteFile = async(userID: string, fileID: string) => { - - let bucket = new mongoose.mongo.GridFSBucket(conn.db, { - chunkSizeBytes: 1024 * 255, - }); - - const file = await dbUtilsFile.getFileInfo(fileID, userID); - - if (!file) throw new NotFoundError("Delete File Not Found Error"); - - if (file.metadata.thumbnailID) { - - await Thumbnail.deleteOne({_id: file.metadata.thumbnailID}); - } - - if (file.metadata.isVideo && file.metadata.transcoded) { - try { - await bucket.delete(ObjectID(file.metadata.transcodedID)); - } catch (e) { - console.log("Could Not Find Transcoded Video"); - } - } - - await bucket.delete(ObjectID(fileID)); - } } export default MongoFileService; \ No newline at end of file diff --git a/backend/services/FileService/utils/createThumbnailFS.ts b/backend/services/FileService/utils/createThumbnailFS.ts index f874980..a9eca0e 100644 --- a/backend/services/FileService/utils/createThumbnailFS.ts +++ b/backend/services/FileService/utils/createThumbnailFS.ts @@ -1,7 +1,6 @@ const mongoose = require("../../../db/mongoose") const conn = mongoose.connection; const crypto= require("crypto"); -const env = require("../../../enviroment/env"); import Thumbnail from "../../../models/thumbnail"; const ObjectID = require('mongodb').ObjectID const sharp = require("sharp"); @@ -10,6 +9,7 @@ import {FileInterface} from "../../../models/file"; import {UserInterface} from "../../../models/user"; import fs from "fs"; import uuid from "uuid"; +import env from "../../../enviroment/env"; const createThumbnailFS = (file: FileInterface, filename: string, user: UserInterface) => { @@ -22,7 +22,7 @@ const createThumbnailFS = (file: FileInterface, filename: string, user: UserInte const thumbnailFilename = uuid.v4(); const readStream = fs.createReadStream(file.metadata.filePath!); - const writeStream = fs.createWriteStream(`/Users/kylehoell/Documents/fstestdata/${thumbnailFilename}`); + const writeStream = fs.createWriteStream(env.fsDirectory + thumbnailFilename); const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, file.metadata.IV); readStream.on("error", (e: Error) => { @@ -58,7 +58,7 @@ const createThumbnailFS = (file: FileInterface, filename: string, user: UserInte writeStream.on("finish", async() => { console.log("Thumbnail written"); - const thumbnailModel = new Thumbnail({name: filename, owner: user._id, IV: thumbnailIV, path: `/Users/kylehoell/Documents/fstestdata/${thumbnailFilename}`}); + const thumbnailModel = new Thumbnail({name: filename, owner: user._id, IV: thumbnailIV, path: env.fsDirectory + thumbnailFilename}); await thumbnailModel.save(); diff --git a/package.json b/package.json index 7d55069..3712115 100755 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "@types/mongoose": "^5.7.10", "@types/node": "^13.11.1", "@types/uuid": "^7.0.2", + "aws-sdk": "^2.657.0", "axios": "^0.19.2", "babel-polyfill": "^6.26.0", "bcrypt": "^3.0.8",