diff --git a/.DS_Store b/.DS_Store index d99ca88..1bee935 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/backend/controllers/fileS3.ts b/backend/controllers/fileS3.ts new file mode 100644 index 0000000..2bad71a --- /dev/null +++ b/backend/controllers/fileS3.ts @@ -0,0 +1,593 @@ +import { Request, Response } from "express"; + +//const FileService = require("../services/FileService") +import FileService from "../services/FileService/indexnew"; + +const fileService = new FileService() + +import S3FileService from "../services/ChunkService/S3Service"; +const s3FileService = new S3FileService(); + +import {UserInterface} from "../models/user"; + +interface RequestType extends Request { + user?: UserInterface, + auth?: any, + busboy?: any, +} + +class FileSystemController { + + // fileService: ; + + constructor() { + + } + + getThumbnail = async(req: RequestType, res: Response) => { + + + if (!req.user) { + + return; + } + + try { + + + const user = req.user; + const id = req.params.id; + + const decryptedThumbnail = await s3FileService.getThumbnail(user, id); + + res.send(decryptedThumbnail); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send(); + } + + } + + async getFullThumbnail(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const user = req.user; + const fileID = req.params.id; + + await s3FileService.getFullThumbnail(user, fileID, res); + + } catch (e) { + const code = e.code || 500; + console.log(e.message, e.exception) + return res.status(code).send(); + } + } + + async uploadFile(req: RequestType, res: Response) { + + if (!req.user) { + + return + } + + try { + + const user = req.user; + const busboy = req.busboy; + + req.pipe(busboy); + + const file = await s3FileService.uploadFile(user, busboy, req); + + res.send(file); + + console.log("file uploaded"); + + } catch (e) { + const code = e.code || 500; + console.log(e.message, e.exception) + return res.status(code).send(); + } + } + + async getPublicDownload(req: RequestType, res: Response) { + + try { + + const ID = req.params.id; + const tempToken = req.params.tempToken; + + await s3FileService.getPublicDownload(ID, tempToken, res); + + } catch (e) { + + const code = e.code || 500; + const message = e.message || e; + + console.log(message, e); + res.status(code).send(); + } + } + + async removeLink(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const id = req.params.id; + const userID = req.user._id; + + await fileService.removeLink(userID, id) + + res.send(); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send(); + } + + } + + async makePublic(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const fileID = req.params.id; + const user = req.user; + + const token = await fileService.makePublic(user, fileID); + + res.send(token); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send(); + } + } + + async getPublicInfo(req: RequestType, res: Response) { + + try { + + const id = req.params.id; + const tempToken = req.params.tempToken; + + const file = await fileService.getPublicInfo(id, tempToken); + + res.send(file); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } + + async makeOneTimePublic(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const id = req.params.id; + const userID = req.user._id; + + const token = await fileService.makeOneTimePublic(userID, id); + + res.send(token); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + + } + + async getFileInfo(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const fileID = req.params.id; + const userID = req.user._id; + + const file = await fileService.getFileInfo(userID, fileID); + + res.send(file); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } + + async getQuickList(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const userID = req.user._id; + + const quickList = await fileService.getQuickList(userID); + + res.send(quickList); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } + + async getList(req: RequestType, res: Response) { + + if (!req.user) { + return + } + + try { + + const query = req.query; + const userID = req.user._id; + + const fileList = await fileService.getList(userID, query); + + res.send(fileList); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } + + async getDownloadToken(req: RequestType, res: Response) { + + if (!req.user) { + return + } + + try { + + const user = req.user; + + const tempToken = await fileService.getDownloadToken(user); + + res.send({tempToken}); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } + + async getDownloadTokenVideo(req: RequestType, res: Response) { + + if (!req.user) { + return + } + + try { + + const user = req.user; + const cookie = req.headers.uuid as string; + + const tempToken = await fileService.getDownloadTokenVideo(user, cookie); + + res.send({tempToken}); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } + + async removeTempToken(req: RequestType, res: Response) { + + if (!req.user) { + return + } + + try { + + const user = req.user + const tempToken = req.params.tempToken; + + await fileService.removeTempToken(user, tempToken); + + res.send(); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } + + // async transcodeVideo(req: RequestType, res: Response) { + + // if (!req.user) { + + // return; + // } + + // try { + + // console.log("transcode request", req.body.file._id) + + // const user = req.user; + // const body = req.body; + + // await fileService.transcodeVideo(user, body); + + // res.send("Finished"); + + // } catch (e) { + + // const code = e.code || 500; + // console.log(e.message, e.exception) + // return res.status(code).send(); + // } + // } + + // async removeTranscodeVideo(req: RequestType, res: Response) { + + // if (!req.user) { + + // return; + // } + + // try { + + // const fileID = req.body.id; + // const userID = req.user._id; + + // await fileService.removeTranscodeVideo(userID, fileID); + + // res.send(); + + // } catch (e) { + + // const code = e.code || 500; + + // console.log(e); + // res.status(code).send() + // } + // } + + // async streamTranscodedVideo(req: RequestType, res: Response) { + + // if (!req.auth || !req.user) { + // return; + // } + + // try { + + // console.log("stream request transcoded", req.params.id) + + // const fileID = req.params.id; + // const userID = req.user._id; + // const headers = req.headers; + + // await fileService.streamTranscodedVideo(userID, fileID, headers, res); + + // } catch (e) { + + // const code = e.code || 500; + // const message = e.message || e; + + // console.log(message, e); + // res.status(code).send(); + // } + // } + + async streamVideo(req: RequestType, res: Response) { + + if (!req.auth || !req.user) { + return; + } + + try { + + const user = req.user; + const fileID = req.params.id; + const headers = req.headers; + + console.log("stream request", req.params.id) + + await s3FileService.streamVideo(user, fileID, headers, res); + + } catch (e) { + + const code = e.code || 500; + const message = e.message || e; + + console.log(message, e); + res.status(code).send(); + } + + } + + async downloadFile(req: RequestType, res: Response) { + + if (!req.auth || !req.user) { + return; + } + + try { + + console.log("download request") + + const user = req.user; + const fileID = req.params.id; + + //await fileService.downloadFile(user, fileID, res); + await s3FileService.downloadFile(user, fileID, res); + + } catch (e) { + + const code = e.code || 500; + const message = e.message || e; + + console.log(message, e); + res.status(code).send(); + } + } + + async getSuggestedList(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const userID = req.user._id; + let searchQuery = req.query.search || ""; + + const {fileList, folderList} = await fileService.getSuggestedList(userID, searchQuery); + + return res.send({folderList, fileList}) + + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } + + async renameFile(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const fileID = req.body.id; + const title = req.body.title + const userID = req.user._id; + + await fileService.renameFile(userID, fileID, title) + + res.send(); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + + } + + async moveFile(req: RequestType, res: Response) { + + console.log("move request"); + + if (!req.user) { + return; + } + + try { + + const fileID = req.body.id; + const userID = req.user._id; + const parentID = req.body.parent; + + console.log(fileID, userID, parentID); + + await fileService.moveFile(userID, fileID, parentID); + + res.send(); + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + + } + + async deleteFile(req: RequestType, res: Response) { + + if (!req.user) { + return; + } + + try { + + const userID = req.user._id; + const fileID = req.body.id; + + await s3FileService.deleteFile(userID, fileID); + + res.send() + + } catch (e) { + + const code = e.code || 500; + + console.log(e); + res.status(code).send() + } + } +} + +export default FileSystemController; diff --git a/backend/controllers/fileSystem.ts b/backend/controllers/fileSystem.ts index 0ed4914..cd546fd 100644 --- a/backend/controllers/fileSystem.ts +++ b/backend/controllers/fileSystem.ts @@ -8,9 +8,6 @@ const fileService = new FileService() import FileSystemService from "../services/ChunkService/FileSystemService"; const fileSystemService = new FileSystemService(); -import S3FileService from "../services/ChunkService/S3Service"; -const s3FileService = new S3FileService(); - import {UserInterface} from "../models/user"; interface RequestType extends Request { @@ -89,7 +86,7 @@ class FileSystemController { req.pipe(busboy); - const file = await s3FileService.uploadFile(user, busboy, req); + const file = await fileSystemService.uploadFile(user, busboy, req); res.send(file); @@ -448,7 +445,7 @@ class FileSystemController { console.log("stream request", req.params.id) - await s3FileService.streamVideo(user, fileID, headers, res); + await fileSystemService.streamVideo(user, fileID, headers, res); } catch (e) { @@ -475,7 +472,7 @@ class FileSystemController { const fileID = req.params.id; //await fileService.downloadFile(user, fileID, res); - await s3FileService.downloadFile(user, fileID, res); + await fileSystemService.downloadFile(user, fileID, res); } catch (e) { diff --git a/backend/express-routers/file.ts b/backend/express-routers/file.ts index bbf8242..a7a1cb9 100644 --- a/backend/express-routers/file.ts +++ b/backend/express-routers/file.ts @@ -8,14 +8,17 @@ const tempAuthVideo = require("../middleware/tempAuthVideo") import FileController from "../controllers/file"; import FsFileController from "../controllers/fileSystem"; +import S3FileController from "../controllers/fileS3"; import env from "../enviroment/env"; -let fileController: FileController | FsFileController; +let fileController: FileController | FsFileController | S3FileController; if (env.dbType === "mongo") { fileController = new FileController(); -} else { +} else if (env.dbType === "fs") { fileController = new FsFileController(); +} else { + fileController = new S3FileController(); } //const fileController = new FileController() diff --git a/backend/models/thumbnail.ts b/backend/models/thumbnail.ts index e31f30a..8526060 100644 --- a/backend/models/thumbnail.ts +++ b/backend/models/thumbnail.ts @@ -17,10 +17,11 @@ const thumbnailSchema = new mongoose.Schema({ path: { type: String }, + IV: { type: Buffer, - } - + }, + s3ID: String }, { timestamps: true }) @@ -32,6 +33,7 @@ export interface ThumbnailInterface extends Document { data?: any, path?: string, IV?: Buffer, + s3ID? : string, } const Thumbnail = mongoose.model("Thumbnail", thumbnailSchema); diff --git a/backend/services/ChunkService/S3Service.ts b/backend/services/ChunkService/S3Service.ts index cd58688..2b8c479 100644 --- a/backend/services/ChunkService/S3Service.ts +++ b/backend/services/ChunkService/S3Service.ts @@ -15,17 +15,17 @@ import uuid from "uuid"; import awaitUploadStreamS3 from "./utils/awaitUploadStreamS3"; import getFileSize from "./utils/getFileSize"; import awaitStream from "./utils/awaitStream"; -import createThumbnailFS from "../FileService/utils/createThumbnailFS"; +import createThumbnailS3 from "../FileService/utils/createThumbnailS3"; import imageChecker from "../../utils/imageChecker"; import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail"; import streamToBuffer from "../../utils/streamToBuffer"; -import { removeChunksFS } from "./utils/awaitUploadStreamFS"; +import removeChunksS3 from "../FileService/utils/removeChunksS3"; import DbUtilFile from "../../db/utils/fileUtils/index"; const dbUtilsFile = new DbUtilFile(); -class S3Service { +class S3Service implements ChunkInterface { constructor() { @@ -86,20 +86,19 @@ class S3Service { await currentFile.save(); - console.log("Sending file...") - - return currentFile; + const imageCheck = imageChecker(currentFile.filename); - // if (finishedFile.length < 15728640 && imageCheck) { + if (currentFile.length < 15728640 && imageCheck) { - // // const updatedFile = await createThumbnail(finishedFile, filename, user); + console.log("Creating thumbnail...") + const updatedFile = await createThumbnailS3(currentFile, filename, user); - // // return updatedFile; + return updatedFile; - // } else { + } else { - // return finishedFile; - // } + return currentFile; + } } downloadFile = async(user: UserInterface, fileID: string, res: Response) => { @@ -171,6 +170,119 @@ class S3Service { await awaitStream(s3ReadStream.pipe(decipher), res); } + + getThumbnail = async(user: UserInterface, id: string) => { + + const password = user.getEncryptionKey(); + + if (!password) throw new NotAuthorizedError("Invalid Encryption Key") + + const thumbnail = await Thumbnail.findById(id) as ThumbnailInterface; + + if (thumbnail.owner !== user._id.toString()) { + + throw new NotAuthorizedError('Thumbnail Unauthorized Error'); + } + + const iv = thumbnail.IV!; + + const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() + + const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv); + + const params: any = {Bucket: env.s3Bucket, Key: thumbnail.s3ID!}; + + const readStream = s3.getObject(params).createReadStream(); + + const bufferData = await streamToBuffer(readStream.pipe(decipher)); + + return bufferData; + } + + getFullThumbnail = async(user: UserInterface, fileID: string, res: Response) => { + + const userID = user._id; + + const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID); + + if (!file) throw new NotFoundError("File Thumbnail Not Found"); + + const password = user.getEncryptionKey(); + const IV = file.metadata.IV.buffer + + if (!password) throw new NotAuthorizedError("Invalid Encryption Key") + + const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!}; + + const readStream = s3.getObject(params).createReadStream(); + + const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() + + const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV); + + res.set('Content-Type', 'binary/octet-stream'); + res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"'); + res.set('Content-Length', file.metadata.size.toString()); + + console.log("Sending Full Thumbnail...") + await awaitStream(readStream.pipe(decipher), res); + console.log("Full thumbnail sent"); + } + + getPublicDownload = async(fileID: string, tempToken: any, res: Response) => { + + const file: FileInterface = await dbUtilsFile.getPublicFile(fileID); + + if (!file || !file.metadata.link || file.metadata.link !== tempToken) { + throw new NotAuthorizedError("File Not Public"); + } + + const user = await User.findById(file.metadata.owner) as UserInterface; + + const password = user.getEncryptionKey(); + + if (!password) throw new NotAuthorizedError("Invalid Encryption Key"); + + const IV = file.metadata.IV.buffer + + const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!}; + + const readStream = s3.getObject(params).createReadStream(); + + const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() + + const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV); + + res.set('Content-Type', 'binary/octet-stream'); + res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"'); + res.set('Content-Length', file.metadata.size.toString()); + + await awaitStream(readStream.pipe(decipher), res); + + if (file.metadata.linkType === "one") { + console.log("removing public link"); + await dbUtilsFile.removeOneTimePublicLink(fileID); + } + } + + deleteFile = async(userID: string, fileID: string) => { + + const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID); + + if (!file) throw new NotFoundError("Delete File Not Found Error"); + + if (file.metadata.thumbnailID) { + + const thumbnail = await Thumbnail.findById(file.metadata.thumbnailID) as ThumbnailInterface; + const paramsThumbnail: any = {Bucket: env.s3Bucket, Key: thumbnail.s3ID!}; + await removeChunksS3(paramsThumbnail); + await Thumbnail.deleteOne({_id: file.metadata.thumbnailID}); + } + + const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!}; + await removeChunksS3(params); + await File.deleteOne({_id: file._id}); + } } export default S3Service; \ No newline at end of file diff --git a/backend/services/FileService/utils/createThumbnailS3.ts b/backend/services/FileService/utils/createThumbnailS3.ts new file mode 100644 index 0000000..ff09aaa --- /dev/null +++ b/backend/services/FileService/utils/createThumbnailS3.ts @@ -0,0 +1,169 @@ +const mongoose = require("../../../db/mongoose") +const conn = mongoose.connection; +const crypto= require("crypto"); +import Thumbnail from "../../../models/thumbnail"; +const ObjectID = require('mongodb').ObjectID +const sharp = require("sharp"); +const concat = require("concat-stream") +import {FileInterface} from "../../../models/file"; +import {UserInterface} from "../../../models/user"; +import fs from "fs"; +import uuid from "uuid"; +import env from "../../../enviroment/env"; +import s3 from "../../../db/s3"; +import awaitUploadStreamS3 from "../../ChunkService/utils/awaitUploadStreamS3"; + +const createThumbnailS3 = (file: FileInterface, filename: string, user: UserInterface) => { + + return new Promise((resolve, reject) => { + + const password = user.getEncryptionKey(); + + let CIPHER_KEY = crypto.createHash('sha256').update(password).digest() + + const thumbnailFilename = uuid.v4(); + + const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!}; + + const readStream = s3.getObject(params).createReadStream(); + + //const readStream = fs.createReadStream(file.metadata.filePath!); + //const writeStream = fs.createWriteStream(env.fsDirectory + thumbnailFilename); + + + const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, file.metadata.IV); + + + + + readStream.on("error", (e: Error) => { + console.log("File service upload thumbnail error", e); + resolve(file); + }) + + // writeStream.on("error", (e: Error) => { + // console.log("File service upload write thumbnail error", e); + // resolve(file); + // }) + + decipher.on("error", (e: Error) => { + console.log("File service upload thumbnail decipher error", e); + resolve(file) + }) + + try { + + + const thumbnailIV = crypto.randomBytes(16); + + const thumbnailCipher = crypto.createCipheriv("aes256", CIPHER_KEY, thumbnailIV); + + const imageResize = sharp().resize(300).on("error", (e: Error) => { + + console.log("resize error", e); + resolve(file); + + }) + + const paramsWrite: any = { + Bucket: env.s3Bucket, + Body : readStream.pipe(decipher).pipe(imageResize).pipe(thumbnailCipher), + Key : thumbnailFilename + }; + + //readStream.pipe(decipher).pipe(imageResize).pipe(thumbnailCipher).pipe(writeStream); + //await awaitUploadStreamS3(paramsWrite); + + s3.upload(paramsWrite, async(err: any, data: any) => { + + if (err) { + console.log("Amazon Upload Error", err); + reject("Amazon upload error"); + } + + console.log("Thumbnail written"); + + const thumbnailModel = new Thumbnail({name: thumbnailFilename, owner: user._id, IV: thumbnailIV, s3ID: thumbnailFilename}); + + await thumbnailModel.save(); + + let updatedFile = await conn.db.collection("fs.files") + .findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}}) + + updatedFile = updatedFile.value; + + updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}} + + resolve(updatedFile); + }) + + // writeStream.on("finish", async() => { + // console.log("Thumbnail written"); + + // const thumbnailModel = new Thumbnail({name: filename, owner: user._id, IV: thumbnailIV, path: env.fsDirectory + thumbnailFilename}); + + // await thumbnailModel.save(); + + // let updatedFile = await conn.db.collection("fs.files") + // .findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}}) + + // updatedFile = updatedFile.value; + + // updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}} + + // resolve(updatedFile); + // }) + + } catch (e) { + + console.log("Thumbnail error", e); + resolve(file); + } + + // try { + + // const concatStream = concat(async(bufferData: Buffer) => { + + // const thumbnailIV = crypto.randomBytes(16); + + // const thumbnailCipher = crypto.createCipheriv("aes256", CIPHER_KEY, thumbnailIV); + + // bufferData = Buffer.concat([thumbnailIV, thumbnailCipher.update(bufferData), thumbnailCipher.final()]); + + // const thumbnailModel = new Thumbnail({name: filename, owner: user._id, data: bufferData}); + + // await thumbnailModel.save(); + + // let updatedFile = await conn.db.collection("fs.files") + // .findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}}) + + // updatedFile = updatedFile.value; + + // updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}} + + // resolve(updatedFile); + + // }).on("error", (e: Error) => { + // console.log("File service upload concat stream error", e); + // resolve(file); + // }) + + // const imageResize = sharp().resize(300).on("error", (e: Error) => { + + // console.log("resize error", e); + // resolve(file); + // }) + + // readStream.pipe(decipher).pipe(imageResize).pipe(concatStream); + + + // } catch (e) { + + // console.log(e); + // resolve(file); + // } + + }) +} + +export default createThumbnailS3; \ No newline at end of file diff --git a/backend/services/FileService/utils/removeChunksS3.ts b/backend/services/FileService/utils/removeChunksS3.ts new file mode 100644 index 0000000..f0180ee --- /dev/null +++ b/backend/services/FileService/utils/removeChunksS3.ts @@ -0,0 +1,21 @@ +import s3 from "../../../db/s3"; + +const removeChunksS3 = (parmas: any) => { + + return new Promise((resolve, reject) => { + + s3.deleteObject(parmas, (err, data) => { + + if (err) { + console.log("Could not remove S3 file"); + reject("Could Not Remove S3 File"); + } + + console.log("Delete S3 file"); + resolve(); + }) + + }) +} + +export default removeChunksS3; \ No newline at end of file