From 4a03523c96eb89c7ee3e6f7088abd1237e846d73 Mon Sep 17 00:00:00 2001 From: subnub Date: Fri, 17 Apr 2020 01:20:54 -0400 Subject: [PATCH] Started adding the Amazon S3 Chunk Service, so far Uploading, Downloading, and Streaming video works --- backend/controllers/fileSystem.ts | 9 +- backend/db/s3.ts | 11 +- backend/enviroment/env.js | 5 +- backend/models/file.ts | 4 +- .../ChunkService/FileSystemService.ts | 2 +- backend/services/ChunkService/S3Service.ts | 176 ++++++++++++++++++ .../ChunkService/utils/awaitUploadStreamS3.ts | 21 +++ backend/utils/streamToBuffer.ts | 2 +- 8 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 backend/services/ChunkService/S3Service.ts create mode 100644 backend/services/ChunkService/utils/awaitUploadStreamS3.ts diff --git a/backend/controllers/fileSystem.ts b/backend/controllers/fileSystem.ts index cd546fd..0ed4914 100644 --- a/backend/controllers/fileSystem.ts +++ b/backend/controllers/fileSystem.ts @@ -8,6 +8,9 @@ const fileService = new FileService() import FileSystemService from "../services/ChunkService/FileSystemService"; const fileSystemService = new FileSystemService(); +import S3FileService from "../services/ChunkService/S3Service"; +const s3FileService = new S3FileService(); + import {UserInterface} from "../models/user"; interface RequestType extends Request { @@ -86,7 +89,7 @@ class FileSystemController { req.pipe(busboy); - const file = await fileSystemService.uploadFile(user, busboy, req); + const file = await s3FileService.uploadFile(user, busboy, req); res.send(file); @@ -445,7 +448,7 @@ class FileSystemController { console.log("stream request", req.params.id) - await fileSystemService.streamVideo(user, fileID, headers, res); + await s3FileService.streamVideo(user, fileID, headers, res); } catch (e) { @@ -472,7 +475,7 @@ class FileSystemController { const fileID = req.params.id; //await fileService.downloadFile(user, fileID, res); - await fileSystemService.downloadFile(user, fileID, res); + await s3FileService.downloadFile(user, fileID, res); } catch (e) { diff --git a/backend/db/s3.ts b/backend/db/s3.ts index 73dd6b3..17b945c 100644 --- a/backend/db/s3.ts +++ b/backend/db/s3.ts @@ -1,6 +1,11 @@ import AWS from "aws-sdk"; +import env from "../enviroment/env"; AWS.config.update({ - accessKeyId: "", - secretAccessKey: "" -}); \ No newline at end of file + accessKeyId: env.s3ID, + secretAccessKey: env.s3Key +}); + +const s3 = new AWS.S3(); + +export default s3; \ No newline at end of file diff --git a/backend/enviroment/env.js b/backend/enviroment/env.js index c72bc5d..3fc3d0e 100644 --- a/backend/enviroment/env.js +++ b/backend/enviroment/env.js @@ -7,5 +7,8 @@ module.exports = { url: process.env.URL, mongoURL: process.env.MONGODB_URL, dbType: process.env.DB_TYPE, - fsDirectory: process.env.FS_DIRECTORY + fsDirectory: process.env.FS_DIRECTORY, + s3ID: process.env.S3_ID, + s3Key: process.env.S3_KEY, + s3Bucket: process.env.S3_BUCKET } \ No newline at end of file diff --git a/backend/models/file.ts b/backend/models/file.ts index 44c18df..dc1ac0f 100644 --- a/backend/models/file.ts +++ b/backend/models/file.ts @@ -51,7 +51,8 @@ const fileSchema = new mongoose.Schema({ }, linkType: String, link: String, - filePath: String + filePath: String, + s3ID: String, }, required: true @@ -76,6 +77,7 @@ export interface FileInterface extends Document { linkType?: 'one' | 'public', link?: string, filePath?: string, + s3ID?: string, } } diff --git a/backend/services/ChunkService/FileSystemService.ts b/backend/services/ChunkService/FileSystemService.ts index bee4730..c7a570d 100644 --- a/backend/services/ChunkService/FileSystemService.ts +++ b/backend/services/ChunkService/FileSystemService.ts @@ -24,7 +24,7 @@ import { removeChunksFS } from "./utils/awaitUploadStreamFS"; const dbUtilsFile = new DbUtilFile(); // implements ChunkInterface -class FileSystemService { +class FileSystemService implements ChunkInterface { constructor() { diff --git a/backend/services/ChunkService/S3Service.ts b/backend/services/ChunkService/S3Service.ts new file mode 100644 index 0000000..cd58688 --- /dev/null +++ b/backend/services/ChunkService/S3Service.ts @@ -0,0 +1,176 @@ +import File, { FileInterface } from "../../models/file"; +import User, {UserInterface} from "../../models/user"; +import s3 from "../../db/s3"; +import env from "../../enviroment/env"; + +import { Response, Request } from "express"; +import ChunkInterface from "./utils/ChunkInterface"; +import NotAuthorizedError from "../../utils/NotAuthorizedError"; +import NotFoundError from "../../utils/NotFoundError"; +import crypto from "crypto"; +import getBusboyData from "./utils/getBusboyData"; +import videoChecker from "../../utils/videoChecker"; +import fs from "fs"; +import uuid from "uuid"; +import awaitUploadStreamS3 from "./utils/awaitUploadStreamS3"; +import getFileSize from "./utils/getFileSize"; +import awaitStream from "./utils/awaitStream"; +import createThumbnailFS from "../FileService/utils/createThumbnailFS"; +import imageChecker from "../../utils/imageChecker"; +import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail"; +import streamToBuffer from "../../utils/streamToBuffer"; +import { removeChunksFS } from "./utils/awaitUploadStreamFS"; + +import DbUtilFile from "../../db/utils/fileUtils/index"; +const dbUtilsFile = new DbUtilFile(); + + +class S3Service { + + constructor() { + + } + + uploadFile = async(user: UserInterface, busboy: any, req: Request) => { + + const password = user.getEncryptionKey(); + + if (!password) throw new NotAuthorizedError("Invalid Encryption Key") + + const initVect = crypto.randomBytes(16); + + const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() + + const cipher = crypto.createCipheriv('aes256', CIPHER_KEY, initVect); + + const {file, filename, formData} = await getBusboyData(busboy); + + const parent = formData.get("parent") || "/" + const parentList = formData.get("parentList") || "/"; + const size = formData.get("size") || "" + let hasThumbnail = false; + let thumbnailID = "" + const isVideo = videoChecker(filename) + + const randomS3ID = uuid.v4(); + + const metadata = { + owner: user._id, + parent, + parentList, + hasThumbnail, + thumbnailID, + isVideo, + size, + IV: initVect, + s3ID: randomS3ID + } + + const params = { + Bucket: env.s3Bucket, + Body : file.pipe(cipher), + Key : randomS3ID + }; + + await awaitUploadStreamS3(params); + + const date = new Date(); + const encryptedFileSize = size; //await getFileSize(metadata.filePath); + + const currentFile = new File({ + filename, + uploadDate: date.toISOString(), + length: encryptedFileSize, + metadata + }); + + await currentFile.save(); + + console.log("Sending file...") + + return currentFile; + + // if (finishedFile.length < 15728640 && imageCheck) { + + // // const updatedFile = await createThumbnail(finishedFile, filename, user); + + // // return updatedFile; + + // } else { + + // return finishedFile; + // } + } + + downloadFile = async(user: UserInterface, fileID: string, res: Response) => { + + const currentFile: FileInterface = await dbUtilsFile.getFileInfo(fileID, user._id); + + if (!currentFile) throw new NotFoundError("Download File Not Found"); + + const password = user.getEncryptionKey(); + + if (!password) throw new NotAuthorizedError("Invalid Encryption Key") + + const IV = currentFile.metadata.IV.buffer + + const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() + + const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV); + + res.set('Content-Type', 'binary/octet-stream'); + res.set('Content-Disposition', 'attachment; filename="' + currentFile.filename + '"'); + res.set('Content-Length', currentFile.metadata.size.toString()); + + const params: any = {Bucket: env.s3Bucket, Key: currentFile.metadata.s3ID!}; + + const s3ReadStream = s3.getObject(params).createReadStream(); + + await awaitStream(s3ReadStream.pipe(decipher), res); + + } + + streamVideo = async(user: UserInterface, fileID: string, headers: any, res: Response) => { + + const userID = user._id; + const currentFile: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID); + + if (!currentFile) throw new NotFoundError("Video File Not Found"); + + const password = user.getEncryptionKey(); + + if (!password) throw new NotAuthorizedError("Invalid Encryption Key") + + const fileSize = currentFile.metadata.size; + + const range = headers.range + const parts = range.replace(/bytes=/, "").split("-") + let start = parseInt(parts[0], 10) + let end = parts[1] + ? parseInt(parts[1], 10) + : fileSize-1 + const chunksize = (end-start)+1 + const IV = currentFile.metadata.IV.buffer + + let head = { + 'Content-Range': 'bytes ' + start + '-' + end + '/' + fileSize, + 'Accept-Ranges': 'bytes', + 'Content-Length': chunksize, + 'Content-Type': 'video/mp4'} + + const params: any = {Bucket: env.s3Bucket, Key: currentFile.metadata.s3ID!, Range: `bytes=${start}-${end}`}; + + const s3ReadStream = s3.getObject(params).createReadStream(); + + const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() + + const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV); + + res.writeHead(206, head); + + await awaitStream(s3ReadStream.pipe(decipher), res); + + } +} + +export default S3Service; \ No newline at end of file diff --git a/backend/services/ChunkService/utils/awaitUploadStreamS3.ts b/backend/services/ChunkService/utils/awaitUploadStreamS3.ts new file mode 100644 index 0000000..4b8f9b0 --- /dev/null +++ b/backend/services/ChunkService/utils/awaitUploadStreamS3.ts @@ -0,0 +1,21 @@ +import s3 from "../../../db/s3"; + +const awaitUploadStreamS3 = (params: any) => { + + return new Promise((resolve, reject) => { + + s3.upload(params, (err: any, data: any) => { + + if (err) { + console.log("Amazon Upload Error", err); + reject("Amazon upload error"); + } + + console.log("Amazon File Uploaded", data); + resolve(); + }) + + }) +} + +export default awaitUploadStreamS3; \ No newline at end of file diff --git a/backend/utils/streamToBuffer.ts b/backend/utils/streamToBuffer.ts index b959c24..ec50ee6 100644 --- a/backend/utils/streamToBuffer.ts +++ b/backend/utils/streamToBuffer.ts @@ -1,6 +1,6 @@ const streamToBuffer = (stream: any) => { const chunks: any[] = [] - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { stream.on('data', (chunk: any) => chunks.push(chunk)) stream.on('error', reject) stream.on('end', () => resolve(Buffer.concat(chunks)))