From 1fb0b350ca181030898906a0a1c125ff40606ad4 Mon Sep 17 00:00:00 2001 From: subnub Date: Thu, 30 Apr 2020 16:43:19 -0400 Subject: [PATCH] Finally seems like video streaming works correctly with encryption, now just need to improve folder deletion and cleanup the backend --- backend/controllers/fileSystem.ts | 8 +-- backend/middleware/auth.ts | 2 +- backend/middleware/tempAuth.ts | 2 +- backend/middleware/tempAuthVideo.ts | 2 +- backend/models/user.ts | 6 +- .../ChunkService/FileSystemService.ts | 42 ++++++++--- backend/services/ChunkService/MongoService.ts | 32 +++++++-- backend/services/ChunkService/S3Service.ts | 31 ++++++-- .../ChunkService/utils/awaitStreamVideo.ts | 70 +++++++++++++++++++ .../ChunkService/utils/fixEndChunkLength.ts | 6 ++ .../ChunkService/utils/fixStartChunkLength.ts | 6 ++ .../ChunkService/utils/getPrevIVFS.ts | 19 +++++ .../ChunkService/utils/getPrevIVMongo.ts | 23 ++++++ .../ChunkService/utils/getPrevIVS3.ts | 19 +++++ .../ChunkService/utils/removeTempToken.ts | 13 ++++ src/components/Homepage/HomePage.js | 2 +- src/components/MainSection/index.js | 10 +++ src/styles/components/_Header.scss | 14 +++- src/styles/components/_HomePage.scss | 20 +++++- 19 files changed, 297 insertions(+), 30 deletions(-) create mode 100644 backend/services/ChunkService/utils/awaitStreamVideo.ts create mode 100644 backend/services/ChunkService/utils/fixEndChunkLength.ts create mode 100644 backend/services/ChunkService/utils/fixStartChunkLength.ts create mode 100644 backend/services/ChunkService/utils/getPrevIVFS.ts create mode 100644 backend/services/ChunkService/utils/getPrevIVMongo.ts create mode 100644 backend/services/ChunkService/utils/getPrevIVS3.ts create mode 100644 backend/services/ChunkService/utils/removeTempToken.ts diff --git a/backend/controllers/fileSystem.ts b/backend/controllers/fileSystem.ts index 4c9ec2c..da6e0ca 100644 --- a/backend/controllers/fileSystem.ts +++ b/backend/controllers/fileSystem.ts @@ -14,6 +14,7 @@ interface RequestType extends Request { user?: UserInterface, auth?: any, busboy: any, + encryptedTempToken?: any, } class FileSystemController { @@ -314,6 +315,7 @@ class FileSystemController { try { + console.log("Incoming Video Stream Request"); const user = req.user; const cookie = req.headers.uuid as string; @@ -442,8 +444,6 @@ class FileSystemController { const user = req.user; const fileID = req.params.id; const headers = req.headers; - - console.log("stream request", req.params.id) await fileSystemService.streamVideo(user, fileID, headers, res); @@ -465,13 +465,9 @@ class FileSystemController { } try { - - console.log("download request") - const user = req.user; const fileID = req.params.id; - //await fileService.downloadFile(user, fileID, res); await fileSystemService.downloadFile(user, fileID, res); } catch (e) { diff --git a/backend/middleware/auth.ts b/backend/middleware/auth.ts index a7a2c3e..c0971ae 100644 --- a/backend/middleware/auth.ts +++ b/backend/middleware/auth.ts @@ -22,7 +22,7 @@ const auth = async(req: RequestType, res: Response, next: NextFunction) => { const token = req.header("Authorization")!.replace("Bearer ", ""); - const decoded = await jwt.verify(token, env.password!) as jwtType; + const decoded = jwt.verify(token, env.password!) as jwtType; const iv = decoded.iv; diff --git a/backend/middleware/tempAuth.ts b/backend/middleware/tempAuth.ts index 6e173d3..663aa58 100644 --- a/backend/middleware/tempAuth.ts +++ b/backend/middleware/tempAuth.ts @@ -23,7 +23,7 @@ const tempAuth = async(req: RequestType, res: Response, next: NextFunction) => { const token = req.params.tempToken; - const decoded = await jwt.verify(token, env.password!) as jwtType; + const decoded = jwt.verify(token, env.password!) as jwtType; const iv = decoded.iv; diff --git a/backend/middleware/tempAuthVideo.ts b/backend/middleware/tempAuthVideo.ts index 07feb4d..964e47c 100644 --- a/backend/middleware/tempAuthVideo.ts +++ b/backend/middleware/tempAuthVideo.ts @@ -24,7 +24,7 @@ const tempAuthVideo = async(req: RequestType, res: Response, next: NextFunction) const token = req.params.tempToken; - const decoded = await jwt.verify(token, env.password!) as jwtType; + const decoded = jwt.verify(token, env.password!) as jwtType; const iv = decoded.iv; diff --git a/backend/models/user.ts b/backend/models/user.ts index c83b15c..a6a3d94 100644 --- a/backend/models/user.ts +++ b/backend/models/user.ts @@ -267,8 +267,8 @@ userSchema.methods.generateTempAuthToken = async function() { const iv = crypto.randomBytes(16); - const user = this; - const token = jwt.sign({_id:user._id.toString(), iv}, env.password, {expiresIn:2}); + const user = this as UserInterface; + const token = jwt.sign({_id:user._id.toString(), iv}, env.password, {expiresIn: "3000ms"}); const publicKey = user.publicKey; const encryptionKey = user.getEncryptionKey(); @@ -276,6 +276,8 @@ userSchema.methods.generateTempAuthToken = async function() { user.tempTokens = user.tempTokens.concat({token: encryptedToken}); + //console.log("generate temp auth token", user); + await user.save(); return token; } diff --git a/backend/services/ChunkService/FileSystemService.ts b/backend/services/ChunkService/FileSystemService.ts index d4d408b..3cb156f 100644 --- a/backend/services/ChunkService/FileSystemService.ts +++ b/backend/services/ChunkService/FileSystemService.ts @@ -20,6 +20,10 @@ import streamToBuffer from "../../utils/streamToBuffer"; import User from "../../models/user"; import env from "../../enviroment/env"; import { removeChunksFS } from "./utils/awaitUploadStreamFS"; +import removeTempToken from "./utils/removeTempToken"; +import getPrevIVFS from "./utils/getPrevIVFS"; +import awaitStreamVideo from "./utils/awaitStreamVideo"; +import fixStartChunkLength from "./utils/fixStartChunkLength"; const dbUtilsFile = new DbUtilFile(); @@ -209,29 +213,51 @@ class FileSystemService implements ChunkInterface { let end = parts[1] ? parseInt(parts[1], 10) : fileSize-1 - const chunksize = (end-start)+1 const IV = currentFile.metadata.IV.buffer - + const chunksize = (end-start)+1 + let head = { 'Content-Range': 'bytes ' + start + '-' + end + '/' + fileSize, 'Accept-Ranges': 'bytes', 'Content-Length': chunksize, 'Content-Type': 'video/mp4'} - - const readStream = fs.createReadStream(currentFile.metadata.filePath!, - {start: start, - end: end}); + let currentIV = IV; + + let fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start); + + if (+start === 0) { + + fixedStart = 0; + } + + const fixedEnd = currentFile.length; //end % 16 === 0 ? end + 15: (fixEndChunkLength(end) - 1) + 16; + + const differenceStart = start - fixedStart; + + if (fixedStart !== 0 && start !== 0) { + + currentIV = await getPrevIVFS(fixedStart - 16, currentFile.metadata.filePath!); + } + + const readStream = fs.createReadStream(currentFile.metadata.filePath!, { + start: fixedStart, + end: fixedEnd, + }); const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() - const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV); + const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, currentIV); + + decipher.setAutoPadding(false); res.writeHead(206, head); const allStreamsToErrorCatch = [readStream, decipher]; - await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch); + readStream.pipe(decipher); + + await awaitStreamVideo(start, end, differenceStart, decipher, res, allStreamsToErrorCatch); } getPublicDownload = async(fileID: string, tempToken: any, res: Response) => { diff --git a/backend/services/ChunkService/MongoService.ts b/backend/services/ChunkService/MongoService.ts index 449afc4..3fd9633 100644 --- a/backend/services/ChunkService/MongoService.ts +++ b/backend/services/ChunkService/MongoService.ts @@ -22,6 +22,9 @@ import removeChunks from "../FileService/utils/removeChunks"; import getBusboyData from "./utils/getBusboyData"; import ChunkInterface from "./utils/ChunkInterface"; +import fixStartChunkLength from "./utils/fixStartChunkLength"; +import getPrevIVMongo from "./utils/getPrevIVMongo"; +import awaitStreamVideo from "./utils/awaitStreamVideo"; const conn = mongoose.connection; @@ -243,24 +246,45 @@ class MongoService implements ChunkInterface { 'Content-Length': chunksize, 'Content-Type': 'video/mp4'} + let currentIV = IV; + + let fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start); + + if (+start === 0) { + + fixedStart = 0; + } + + const fixedEnd = currentFile.length; //end % 16 === 0 ? end + 15: (fixEndChunkLength(end) - 1) + 16; + + const differenceStart = start - fixedStart; + + if (fixedStart !== 0 && start !== 0) { + + currentIV = await getPrevIVMongo(fixedStart - 16, fileID); + } + const bucket = new mongoose.mongo.GridFSBucket(conn.db, { chunkSizeBytes: 1024 }); const readStream = bucket.openDownloadStream(new ObjectID(fileID), { - start: start, - end: end + start: fixedStart, + end: fixedEnd, }); const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() - const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV); + const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, currentIV); + decipher.setAutoPadding(false); res.writeHead(206, head); const allStreamsToErrorCatch = [readStream, decipher]; - await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch); + readStream.pipe(decipher); + + await awaitStreamVideo(start, end, differenceStart, decipher, res, allStreamsToErrorCatch); } deleteFile = async(userID: string, fileID: string) => { diff --git a/backend/services/ChunkService/S3Service.ts b/backend/services/ChunkService/S3Service.ts index f3f5606..c82010c 100644 --- a/backend/services/ChunkService/S3Service.ts +++ b/backend/services/ChunkService/S3Service.ts @@ -20,6 +20,10 @@ import imageChecker from "../../utils/imageChecker"; import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail"; import streamToBuffer from "../../utils/streamToBuffer"; import removeChunksS3 from "../FileService/utils/removeChunksS3"; +import fixStartChunkLength from "./utils/fixStartChunkLength"; +import fixEndChunkLength from "./utils/fixEndChunkLength"; +import getPrevIVS3 from "./utils/getPrevIVS3"; +import awaitStreamVideo from "./utils/awaitStreamVideo"; import DbUtilFile from "../../db/utils/fileUtils/index"; const dbUtilsFile = new DbUtilFile(); @@ -159,20 +163,39 @@ class S3Service implements ChunkInterface { 'Content-Length': chunksize, 'Content-Type': 'video/mp4'} - const params: any = {Bucket: env.s3Bucket, Key: currentFile.metadata.s3ID!, Range: `bytes=${start}-${end}`}; + let currentIV = IV; + + let fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start); + + if (+start === 0) { + + fixedStart = 0; + } + + const fixedEnd = fileSize % 16 === 0 ? fileSize : fixEndChunkLength(fileSize); //end % 16 === 0 ? end + 15: (fixEndChunkLength(end) - 1) + 16; + + const differenceStart = start - fixedStart; + + if (fixedStart !== 0 && start !== 0) { + + currentIV = await getPrevIVS3(fixedStart - 16, currentFile.metadata.s3ID!); + } + + const params: any = {Bucket: env.s3Bucket, Key: currentFile.metadata.s3ID!, Range: `bytes=${fixedStart}-${fixedEnd}`}; const s3ReadStream = s3.getObject(params).createReadStream(); const CIPHER_KEY = crypto.createHash('sha256').update(password).digest() - const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV); + const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, currentIV); res.writeHead(206, head); const allStreamsToErrorCatch = [s3ReadStream, decipher]; - await awaitStream(s3ReadStream.pipe(decipher), res, allStreamsToErrorCatch); - + s3ReadStream.pipe(decipher); + + await awaitStreamVideo(start, end, differenceStart, decipher, res, allStreamsToErrorCatch); } getThumbnail = async(user: UserInterface, id: string) => { diff --git a/backend/services/ChunkService/utils/awaitStreamVideo.ts b/backend/services/ChunkService/utils/awaitStreamVideo.ts new file mode 100644 index 0000000..ad31154 --- /dev/null +++ b/backend/services/ChunkService/utils/awaitStreamVideo.ts @@ -0,0 +1,70 @@ +import {Response} from "express" + +const awaitStreamVideo = (start: number, end:number, differenceStart: number, + decipher: any, res: Response, streamsToErrorCatch: any[]) => { + + return new Promise((resolve, reject) => { + + let firstBytesRemoved = false; + let sizeCounter = 0; + + decipher.on("data", (data: Buffer | string) => { + + if (+start === 0 && +end === 1) { + + const dataCoverted = data.toString("hex"); + + let neededData = dataCoverted.substring(0, 4); + + const dataBack = Buffer.from(neededData, "hex"); + + res.write(dataBack); + res.flush(); + return; + } + + if (!firstBytesRemoved) { + + const dataCoverted = data.toString("hex"); + + let neededData = dataCoverted.substring(differenceStart * 2); + + const dataBack = Buffer.from(neededData, "hex"); + + firstBytesRemoved = true; + + sizeCounter += dataBack.length; + + res.write(dataBack); + res.flush(); + return; + } + + + res.write(data); + res.flush(); + sizeCounter += data.length; + }) + + decipher.on("end", () => { + res.end(); + resolve(); + }) + + streamsToErrorCatch.forEach((currentStream) => { + + currentStream.on("error", (e: Error) => { + + reject({ + message: "Await Video Stream Input Error", + code: 500, + error: e + }) + + }) + }) + + }) +} + +export default awaitStreamVideo; \ No newline at end of file diff --git a/backend/services/ChunkService/utils/fixEndChunkLength.ts b/backend/services/ChunkService/utils/fixEndChunkLength.ts new file mode 100644 index 0000000..df87d0d --- /dev/null +++ b/backend/services/ChunkService/utils/fixEndChunkLength.ts @@ -0,0 +1,6 @@ +const fixEndChunkLength = (length: number) => { + + return Math.floor((length-1) / 16) * 16 + 16; +} + +export default fixEndChunkLength; \ No newline at end of file diff --git a/backend/services/ChunkService/utils/fixStartChunkLength.ts b/backend/services/ChunkService/utils/fixStartChunkLength.ts new file mode 100644 index 0000000..ca34370 --- /dev/null +++ b/backend/services/ChunkService/utils/fixStartChunkLength.ts @@ -0,0 +1,6 @@ +const fixStartChunkLength = (length: number) => { + + return Math.floor((length-1) / 16) * 16 - 16; +} + +export default fixStartChunkLength; \ No newline at end of file diff --git a/backend/services/ChunkService/utils/getPrevIVFS.ts b/backend/services/ChunkService/utils/getPrevIVFS.ts new file mode 100644 index 0000000..fbea067 --- /dev/null +++ b/backend/services/ChunkService/utils/getPrevIVFS.ts @@ -0,0 +1,19 @@ +import fs from "fs"; + +const getPrevIV = (start: number, path: string) => { + + return new Promise((resolve, reject) => { + + const stream = fs.createReadStream(path, { + start, + end: start + 15 + }) + + stream.on("data", (data) => { + + resolve(data); + }) + }) +} + +export default getPrevIV; \ No newline at end of file diff --git a/backend/services/ChunkService/utils/getPrevIVMongo.ts b/backend/services/ChunkService/utils/getPrevIVMongo.ts new file mode 100644 index 0000000..0a47177 --- /dev/null +++ b/backend/services/ChunkService/utils/getPrevIVMongo.ts @@ -0,0 +1,23 @@ +import mongoose from "../../../db/mongoose"; +import { ObjectID } from "mongodb" +const conn = mongoose.connection; + +const getPrevIV = (start: number, fileID: string) => { + + return new Promise((resolve, reject) => { + + const bucket = new mongoose.mongo.GridFSBucket(conn.db); + + const stream = bucket.openDownloadStream(new ObjectID(fileID), { + start: start, + end: start + 16 + }); + + stream.on("data", (data) => { + + resolve(data); + }) + }) +} + +export default getPrevIV; \ No newline at end of file diff --git a/backend/services/ChunkService/utils/getPrevIVS3.ts b/backend/services/ChunkService/utils/getPrevIVS3.ts new file mode 100644 index 0000000..3caa34b --- /dev/null +++ b/backend/services/ChunkService/utils/getPrevIVS3.ts @@ -0,0 +1,19 @@ +import s3 from "../../../db/s3"; +import env from "../../../enviroment/env"; + +const getPrevIV = (start: number, key: string) => { + + return new Promise((resolve, reject) => { + + const params: any = {Bucket: env.s3Bucket, Key: key, Range: `bytes=${start}-${start + 15}`}; + + const stream = s3.getObject(params).createReadStream(); + + stream.on("data", (data) => { + + resolve(data); + }) + }) +} + +export default getPrevIV; \ No newline at end of file diff --git a/backend/services/ChunkService/utils/removeTempToken.ts b/backend/services/ChunkService/utils/removeTempToken.ts new file mode 100644 index 0000000..3d30d01 --- /dev/null +++ b/backend/services/ChunkService/utils/removeTempToken.ts @@ -0,0 +1,13 @@ +import { UserInterface } from "../../../models/user"; + +const removeTempToken = async(user: UserInterface, tempToken: any,) => { + + user.tempTokens = user.tempTokens.filter((filterToken) => { + + return filterToken.token !== tempToken; + }) + + await user.save(); +} + +export default removeTempToken; \ No newline at end of file diff --git a/src/components/Homepage/HomePage.js b/src/components/Homepage/HomePage.js index de3dfbb..747e7c2 100644 --- a/src/components/Homepage/HomePage.js +++ b/src/components/Homepage/HomePage.js @@ -8,7 +8,7 @@ import PhotoViewer from "../PhotoViewer" const HomePage = (props) => ( -
+
diff --git a/src/components/MainSection/index.js b/src/components/MainSection/index.js index 386219e..e64743f 100644 --- a/src/components/MainSection/index.js +++ b/src/components/MainSection/index.js @@ -74,8 +74,18 @@ class MainSectionContainer extends React.Component { document.body.appendChild(link); link.href = finalUrl; link.setAttribute('type', 'hidden'); + link.setAttribute("download", true); link.click(); + // const form = document.createElement("form"); + // form.action = finalUrl; + // form.method = "GET"; + // form.setAttribute("type", "hidden"); + + // document.body.append(form); + + // form.submit(); + }).catch((err) => { console.log(err) }) diff --git a/src/styles/components/_Header.scss b/src/styles/components/_Header.scss index 7576da9..04b5dc2 100644 --- a/src/styles/components/_Header.scss +++ b/src/styles/components/_Header.scss @@ -6,9 +6,21 @@ border-bottom: 1px solid #dadce0; justify-content: space-between; /* display: none; */ - position: absolute; + position: fixed; /* top: 0; */ width: 100%; + + @media (max-width: $desktop-breakpoint) { + text-align: center; + display: flex; + flex-direction: row; + border-bottom: 1px solid #dadce0; + justify-content: space-between; + position: initial; + width: 100%; + height: 80px; + padding: 0; + } } .header__title { diff --git a/src/styles/components/_HomePage.scss b/src/styles/components/_HomePage.scss index 549d692..50bf44c 100644 --- a/src/styles/components/_HomePage.scss +++ b/src/styles/components/_HomePage.scss @@ -23,7 +23,11 @@ overflow-x: hidden; @media (max-width: $desktop-breakpoint) { - width: 100vw; + // margin-top: 77px; + display: flex; + flex-direction: column; + overflow-x: hidden; + width: unset; } } @@ -36,12 +40,26 @@ overflow: hidden; } +.outter-wrapper { + height: 100vh; + width: 100vw; +} + .main-page { background: white; height: 100vh; width: 100vw; display: inline-flex; padding-top: 71px; + + @media (max-width: $desktop-breakpoint) { + background: #fff; + /* height: 100vh; */ + height: calc(100% - 80px); + width: 100vw; + display: inline-flex; + padding: 0; + } }