S3 for file chunks is now fully operational, and can used by changing the config file. MongoDB, FileSystem, and S3 are now working fully.

This commit is contained in:
subnub
2020-04-17 16:43:45 -04:00
parent 4a03523c96
commit 9b83b06556
8 changed files with 919 additions and 22 deletions
Vendored
BIN
View File
Binary file not shown.
+593
View File
@@ -0,0 +1,593 @@
import { Request, Response } from "express";
//const FileService = require("../services/FileService")
import FileService from "../services/FileService/indexnew";
const fileService = new FileService()
import S3FileService from "../services/ChunkService/S3Service";
const s3FileService = new S3FileService();
import {UserInterface} from "../models/user";
interface RequestType extends Request {
user?: UserInterface,
auth?: any,
busboy?: any,
}
class FileSystemController {
// fileService: ;
constructor() {
}
getThumbnail = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
}
try {
const user = req.user;
const id = req.params.id;
const decryptedThumbnail = await s3FileService.getThumbnail(user, id);
res.send(decryptedThumbnail);
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send();
}
}
async getFullThumbnail(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const user = req.user;
const fileID = req.params.id;
await s3FileService.getFullThumbnail(user, fileID, res);
} catch (e) {
const code = e.code || 500;
console.log(e.message, e.exception)
return res.status(code).send();
}
}
async uploadFile(req: RequestType, res: Response) {
if (!req.user) {
return
}
try {
const user = req.user;
const busboy = req.busboy;
req.pipe(busboy);
const file = await s3FileService.uploadFile(user, busboy, req);
res.send(file);
console.log("file uploaded");
} catch (e) {
const code = e.code || 500;
console.log(e.message, e.exception)
return res.status(code).send();
}
}
async getPublicDownload(req: RequestType, res: Response) {
try {
const ID = req.params.id;
const tempToken = req.params.tempToken;
await s3FileService.getPublicDownload(ID, tempToken, res);
} catch (e) {
const code = e.code || 500;
const message = e.message || e;
console.log(message, e);
res.status(code).send();
}
}
async removeLink(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const id = req.params.id;
const userID = req.user._id;
await fileService.removeLink(userID, id)
res.send();
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send();
}
}
async makePublic(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const fileID = req.params.id;
const user = req.user;
const token = await fileService.makePublic(user, fileID);
res.send(token);
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send();
}
}
async getPublicInfo(req: RequestType, res: Response) {
try {
const id = req.params.id;
const tempToken = req.params.tempToken;
const file = await fileService.getPublicInfo(id, tempToken);
res.send(file);
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async makeOneTimePublic(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const id = req.params.id;
const userID = req.user._id;
const token = await fileService.makeOneTimePublic(userID, id);
res.send(token);
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async getFileInfo(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const fileID = req.params.id;
const userID = req.user._id;
const file = await fileService.getFileInfo(userID, fileID);
res.send(file);
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async getQuickList(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const userID = req.user._id;
const quickList = await fileService.getQuickList(userID);
res.send(quickList);
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async getList(req: RequestType, res: Response) {
if (!req.user) {
return
}
try {
const query = req.query;
const userID = req.user._id;
const fileList = await fileService.getList(userID, query);
res.send(fileList);
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async getDownloadToken(req: RequestType, res: Response) {
if (!req.user) {
return
}
try {
const user = req.user;
const tempToken = await fileService.getDownloadToken(user);
res.send({tempToken});
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async getDownloadTokenVideo(req: RequestType, res: Response) {
if (!req.user) {
return
}
try {
const user = req.user;
const cookie = req.headers.uuid as string;
const tempToken = await fileService.getDownloadTokenVideo(user, cookie);
res.send({tempToken});
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async removeTempToken(req: RequestType, res: Response) {
if (!req.user) {
return
}
try {
const user = req.user
const tempToken = req.params.tempToken;
await fileService.removeTempToken(user, tempToken);
res.send();
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
// async transcodeVideo(req: RequestType, res: Response) {
// if (!req.user) {
// return;
// }
// try {
// console.log("transcode request", req.body.file._id)
// const user = req.user;
// const body = req.body;
// await fileService.transcodeVideo(user, body);
// res.send("Finished");
// } catch (e) {
// const code = e.code || 500;
// console.log(e.message, e.exception)
// return res.status(code).send();
// }
// }
// async removeTranscodeVideo(req: RequestType, res: Response) {
// if (!req.user) {
// return;
// }
// try {
// const fileID = req.body.id;
// const userID = req.user._id;
// await fileService.removeTranscodeVideo(userID, fileID);
// res.send();
// } catch (e) {
// const code = e.code || 500;
// console.log(e);
// res.status(code).send()
// }
// }
// async streamTranscodedVideo(req: RequestType, res: Response) {
// if (!req.auth || !req.user) {
// return;
// }
// try {
// console.log("stream request transcoded", req.params.id)
// const fileID = req.params.id;
// const userID = req.user._id;
// const headers = req.headers;
// await fileService.streamTranscodedVideo(userID, fileID, headers, res);
// } catch (e) {
// const code = e.code || 500;
// const message = e.message || e;
// console.log(message, e);
// res.status(code).send();
// }
// }
async streamVideo(req: RequestType, res: Response) {
if (!req.auth || !req.user) {
return;
}
try {
const user = req.user;
const fileID = req.params.id;
const headers = req.headers;
console.log("stream request", req.params.id)
await s3FileService.streamVideo(user, fileID, headers, res);
} catch (e) {
const code = e.code || 500;
const message = e.message || e;
console.log(message, e);
res.status(code).send();
}
}
async downloadFile(req: RequestType, res: Response) {
if (!req.auth || !req.user) {
return;
}
try {
console.log("download request")
const user = req.user;
const fileID = req.params.id;
//await fileService.downloadFile(user, fileID, res);
await s3FileService.downloadFile(user, fileID, res);
} catch (e) {
const code = e.code || 500;
const message = e.message || e;
console.log(message, e);
res.status(code).send();
}
}
async getSuggestedList(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const userID = req.user._id;
let searchQuery = req.query.search || "";
const {fileList, folderList} = await fileService.getSuggestedList(userID, searchQuery);
return res.send({folderList, fileList})
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async renameFile(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const fileID = req.body.id;
const title = req.body.title
const userID = req.user._id;
await fileService.renameFile(userID, fileID, title)
res.send();
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async moveFile(req: RequestType, res: Response) {
console.log("move request");
if (!req.user) {
return;
}
try {
const fileID = req.body.id;
const userID = req.user._id;
const parentID = req.body.parent;
console.log(fileID, userID, parentID);
await fileService.moveFile(userID, fileID, parentID);
res.send();
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async deleteFile(req: RequestType, res: Response) {
if (!req.user) {
return;
}
try {
const userID = req.user._id;
const fileID = req.body.id;
await s3FileService.deleteFile(userID, fileID);
res.send()
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
}
export default FileSystemController;
+3 -6
View File
@@ -8,9 +8,6 @@ const fileService = new FileService()
import FileSystemService from "../services/ChunkService/FileSystemService";
const fileSystemService = new FileSystemService();
import S3FileService from "../services/ChunkService/S3Service";
const s3FileService = new S3FileService();
import {UserInterface} from "../models/user";
interface RequestType extends Request {
@@ -89,7 +86,7 @@ class FileSystemController {
req.pipe(busboy);
const file = await s3FileService.uploadFile(user, busboy, req);
const file = await fileSystemService.uploadFile(user, busboy, req);
res.send(file);
@@ -448,7 +445,7 @@ class FileSystemController {
console.log("stream request", req.params.id)
await s3FileService.streamVideo(user, fileID, headers, res);
await fileSystemService.streamVideo(user, fileID, headers, res);
} catch (e) {
@@ -475,7 +472,7 @@ class FileSystemController {
const fileID = req.params.id;
//await fileService.downloadFile(user, fileID, res);
await s3FileService.downloadFile(user, fileID, res);
await fileSystemService.downloadFile(user, fileID, res);
} catch (e) {
+5 -2
View File
@@ -8,14 +8,17 @@ const tempAuthVideo = require("../middleware/tempAuthVideo")
import FileController from "../controllers/file";
import FsFileController from "../controllers/fileSystem";
import S3FileController from "../controllers/fileS3";
import env from "../enviroment/env";
let fileController: FileController | FsFileController;
let fileController: FileController | FsFileController | S3FileController;
if (env.dbType === "mongo") {
fileController = new FileController();
} else {
} else if (env.dbType === "fs") {
fileController = new FsFileController();
} else {
fileController = new S3FileController();
}
//const fileController = new FileController()
+4 -2
View File
@@ -17,10 +17,11 @@ const thumbnailSchema = new mongoose.Schema({
path: {
type: String
},
IV: {
type: Buffer,
}
},
s3ID: String
}, {
timestamps: true
})
@@ -32,6 +33,7 @@ export interface ThumbnailInterface extends Document {
data?: any,
path?: string,
IV?: Buffer,
s3ID? : string,
}
const Thumbnail = mongoose.model<ThumbnailInterface>("Thumbnail", thumbnailSchema);
+124 -12
View File
@@ -15,17 +15,17 @@ import uuid from "uuid";
import awaitUploadStreamS3 from "./utils/awaitUploadStreamS3";
import getFileSize from "./utils/getFileSize";
import awaitStream from "./utils/awaitStream";
import createThumbnailFS from "../FileService/utils/createThumbnailFS";
import createThumbnailS3 from "../FileService/utils/createThumbnailS3";
import imageChecker from "../../utils/imageChecker";
import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail";
import streamToBuffer from "../../utils/streamToBuffer";
import { removeChunksFS } from "./utils/awaitUploadStreamFS";
import removeChunksS3 from "../FileService/utils/removeChunksS3";
import DbUtilFile from "../../db/utils/fileUtils/index";
const dbUtilsFile = new DbUtilFile();
class S3Service {
class S3Service implements ChunkInterface {
constructor() {
@@ -86,20 +86,19 @@ class S3Service {
await currentFile.save();
console.log("Sending file...")
return currentFile;
const imageCheck = imageChecker(currentFile.filename);
// if (finishedFile.length < 15728640 && imageCheck) {
if (currentFile.length < 15728640 && imageCheck) {
// // const updatedFile = await createThumbnail(finishedFile, filename, user);
console.log("Creating thumbnail...")
const updatedFile = await createThumbnailS3(currentFile, filename, user);
// // return updatedFile;
return updatedFile;
// } else {
} else {
// return finishedFile;
// }
return currentFile;
}
}
downloadFile = async(user: UserInterface, fileID: string, res: Response) => {
@@ -171,6 +170,119 @@ class S3Service {
await awaitStream(s3ReadStream.pipe(decipher), res);
}
getThumbnail = async(user: UserInterface, id: string) => {
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const thumbnail = await Thumbnail.findById(id) as ThumbnailInterface;
if (thumbnail.owner !== user._id.toString()) {
throw new NotAuthorizedError('Thumbnail Unauthorized Error');
}
const iv = thumbnail.IV!;
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv);
const params: any = {Bucket: env.s3Bucket, Key: thumbnail.s3ID!};
const readStream = s3.getObject(params).createReadStream();
const bufferData = await streamToBuffer(readStream.pipe(decipher));
return bufferData;
}
getFullThumbnail = async(user: UserInterface, fileID: string, res: Response) => {
const userID = user._id;
const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("File Thumbnail Not Found");
const password = user.getEncryptionKey();
const IV = file.metadata.IV.buffer
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!};
const readStream = s3.getObject(params).createReadStream();
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size.toString());
console.log("Sending Full Thumbnail...")
await awaitStream(readStream.pipe(decipher), res);
console.log("Full thumbnail sent");
}
getPublicDownload = async(fileID: string, tempToken: any, res: Response) => {
const file: FileInterface = await dbUtilsFile.getPublicFile(fileID);
if (!file || !file.metadata.link || file.metadata.link !== tempToken) {
throw new NotAuthorizedError("File Not Public");
}
const user = await User.findById(file.metadata.owner) as UserInterface;
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key");
const IV = file.metadata.IV.buffer
const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!};
const readStream = s3.getObject(params).createReadStream();
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size.toString());
await awaitStream(readStream.pipe(decipher), res);
if (file.metadata.linkType === "one") {
console.log("removing public link");
await dbUtilsFile.removeOneTimePublicLink(fileID);
}
}
deleteFile = async(userID: string, fileID: string) => {
const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("Delete File Not Found Error");
if (file.metadata.thumbnailID) {
const thumbnail = await Thumbnail.findById(file.metadata.thumbnailID) as ThumbnailInterface;
const paramsThumbnail: any = {Bucket: env.s3Bucket, Key: thumbnail.s3ID!};
await removeChunksS3(paramsThumbnail);
await Thumbnail.deleteOne({_id: file.metadata.thumbnailID});
}
const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!};
await removeChunksS3(params);
await File.deleteOne({_id: file._id});
}
}
export default S3Service;
@@ -0,0 +1,169 @@
const mongoose = require("../../../db/mongoose")
const conn = mongoose.connection;
const crypto= require("crypto");
import Thumbnail from "../../../models/thumbnail";
const ObjectID = require('mongodb').ObjectID
const sharp = require("sharp");
const concat = require("concat-stream")
import {FileInterface} from "../../../models/file";
import {UserInterface} from "../../../models/user";
import fs from "fs";
import uuid from "uuid";
import env from "../../../enviroment/env";
import s3 from "../../../db/s3";
import awaitUploadStreamS3 from "../../ChunkService/utils/awaitUploadStreamS3";
const createThumbnailS3 = (file: FileInterface, filename: string, user: UserInterface) => {
return new Promise<FileInterface>((resolve, reject) => {
const password = user.getEncryptionKey();
let CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const thumbnailFilename = uuid.v4();
const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!};
const readStream = s3.getObject(params).createReadStream();
//const readStream = fs.createReadStream(file.metadata.filePath!);
//const writeStream = fs.createWriteStream(env.fsDirectory + thumbnailFilename);
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, file.metadata.IV);
readStream.on("error", (e: Error) => {
console.log("File service upload thumbnail error", e);
resolve(file);
})
// writeStream.on("error", (e: Error) => {
// console.log("File service upload write thumbnail error", e);
// resolve(file);
// })
decipher.on("error", (e: Error) => {
console.log("File service upload thumbnail decipher error", e);
resolve(file)
})
try {
const thumbnailIV = crypto.randomBytes(16);
const thumbnailCipher = crypto.createCipheriv("aes256", CIPHER_KEY, thumbnailIV);
const imageResize = sharp().resize(300).on("error", (e: Error) => {
console.log("resize error", e);
resolve(file);
})
const paramsWrite: any = {
Bucket: env.s3Bucket,
Body : readStream.pipe(decipher).pipe(imageResize).pipe(thumbnailCipher),
Key : thumbnailFilename
};
//readStream.pipe(decipher).pipe(imageResize).pipe(thumbnailCipher).pipe(writeStream);
//await awaitUploadStreamS3(paramsWrite);
s3.upload(paramsWrite, async(err: any, data: any) => {
if (err) {
console.log("Amazon Upload Error", err);
reject("Amazon upload error");
}
console.log("Thumbnail written");
const thumbnailModel = new Thumbnail({name: thumbnailFilename, owner: user._id, IV: thumbnailIV, s3ID: thumbnailFilename});
await thumbnailModel.save();
let updatedFile = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}})
updatedFile = updatedFile.value;
updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}}
resolve(updatedFile);
})
// writeStream.on("finish", async() => {
// console.log("Thumbnail written");
// const thumbnailModel = new Thumbnail({name: filename, owner: user._id, IV: thumbnailIV, path: env.fsDirectory + thumbnailFilename});
// await thumbnailModel.save();
// let updatedFile = await conn.db.collection("fs.files")
// .findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}})
// updatedFile = updatedFile.value;
// updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}}
// resolve(updatedFile);
// })
} catch (e) {
console.log("Thumbnail error", e);
resolve(file);
}
// try {
// const concatStream = concat(async(bufferData: Buffer) => {
// const thumbnailIV = crypto.randomBytes(16);
// const thumbnailCipher = crypto.createCipheriv("aes256", CIPHER_KEY, thumbnailIV);
// bufferData = Buffer.concat([thumbnailIV, thumbnailCipher.update(bufferData), thumbnailCipher.final()]);
// const thumbnailModel = new Thumbnail({name: filename, owner: user._id, data: bufferData});
// await thumbnailModel.save();
// let updatedFile = await conn.db.collection("fs.files")
// .findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}})
// updatedFile = updatedFile.value;
// updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}}
// resolve(updatedFile);
// }).on("error", (e: Error) => {
// console.log("File service upload concat stream error", e);
// resolve(file);
// })
// const imageResize = sharp().resize(300).on("error", (e: Error) => {
// console.log("resize error", e);
// resolve(file);
// })
// readStream.pipe(decipher).pipe(imageResize).pipe(concatStream);
// } catch (e) {
// console.log(e);
// resolve(file);
// }
})
}
export default createThumbnailS3;
@@ -0,0 +1,21 @@
import s3 from "../../../db/s3";
const removeChunksS3 = (parmas: any) => {
return new Promise((resolve, reject) => {
s3.deleteObject(parmas, (err, data) => {
if (err) {
console.log("Could not remove S3 file");
reject("Could Not Remove S3 File");
}
console.log("Delete S3 file");
resolve();
})
})
}
export default removeChunksS3;