oxmc/myDrive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #13 from subnub/typescript-fs-s3

Browse Source 
Typescript fs s3
This commit is contained in:
subnub
2020-05-25 21:02:28 -04:00
committed by GitHub
parent 306e519eaa 698a41732d
commit 59e28a7d8f
153 changed files with 4963 additions and 2174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.DS_Store
Vendored
BIN
View File
Binary file not shown.
.dockerignore
+1
View File
@@ -0,0 +1 @@
node_modules
.gitignore
+3
View File
@@ -16,3 +16,6 @@ certificate.key
package-lock.json
._*
.DS_Store
dist/
rds-combined-ca-bundle.pem
docker-variables.env
Dockerfile
+16
View File
@@ -0,0 +1,16 @@
FROM node:13
WORKDIR /usr/app
COPY package*.json ./
RUN npm install
COPY . .
RUN npm run build:docker
EXPOSE 8080
EXPOSE 3000
CMD [ "npm", "run", "start"]
backend/controllers/file.js → backend/controllers/file.ts
+68 -112
View File
@@ -1,15 +1,33 @@
const FileService = require("../services/FileService")
import { Request, Response } from "express";
import FileService from "../services/FileService";
import MongoService from "../services/ChunkService/MongoService";
import FileSystemService from "../services/ChunkService/FileSystemService";
import S3Service from "../services/ChunkService/S3Service";
import {UserInterface} from "../models/user";
import uuid from "uuid";
import tempStorage from "../tempStorage/tempStorage";
const fileService = new FileService()
interface RequestType extends Request {
user?: UserInterface,
auth?: boolean,
busboy: any,
}
type ChunkServiceType = MongoService | FileSystemService | S3Service;
class FileController {
constructor() {
chunkService: ChunkServiceType;
constructor(chunkService: ChunkServiceType) {
this.chunkService = chunkService;
}
getThumbnail = async(req: RequestType, res: Response) => {
async getThumbnail(req, res) {
if (!req.user) {
@@ -17,12 +35,14 @@ class FileController {
}
try {
const user = req.user;
const id = req.params.id;
const decryptedThumbnail = await fileService.getThumbnail(user, id);
const decryptedThumbnail = await this.chunkService.getThumbnail(user, id);
res.send(decryptedThumbnail);
} catch (e) {
@@ -35,7 +55,7 @@ class FileController {
}
async getFullThumbnail(req, res) {
getFullThumbnail = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -46,7 +66,7 @@ class FileController {
const user = req.user;
const fileID = req.params.id;
await fileService.getFullThumbnail(user, fileID, res);
await this.chunkService.getFullThumbnail(user, fileID, res);
} catch (e) {
const code = e.code || 500;
@@ -55,7 +75,7 @@ class FileController {
}
}
async uploadFile(req, res) {
uploadFile = async(req: RequestType, res: Response) => {
if (!req.user) {
@@ -69,7 +89,7 @@ class FileController {
req.pipe(busboy);
const file = await fileService.upload(user, busboy, req);
const file = await this.chunkService.uploadFile(user, busboy, req);
res.send(file);
@@ -82,14 +102,14 @@ class FileController {
}
}
async getPublicDownload(req, res) {
getPublicDownload = async(req: RequestType, res: Response) => {
try {
const ID = req.params.id;
const tempToken = req.params.tempToken;
await fileService.publicDownload(ID, tempToken, res);
await this.chunkService.getPublicDownload(ID, tempToken, res);
} catch (e) {
@@ -101,7 +121,7 @@ class FileController {
}
}
async removeLink(req, res) {
removeLink = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -126,7 +146,7 @@ class FileController {
}
async makePublic(req, res) {
makePublic = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -150,7 +170,7 @@ class FileController {
}
}
async getPublicInfo(req, res) {
getPublicInfo = async(req: RequestType, res: Response) => {
try {
@@ -170,7 +190,7 @@ class FileController {
}
}
async makeOneTimePublic(req, res) {
makeOneTimePublic = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -195,7 +215,7 @@ class FileController {
}
async getFileInfo(req, res) {
getFileInfo = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -219,7 +239,7 @@ class FileController {
}
}
async getQuickList(req, res) {
getQuickList = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -242,7 +262,7 @@ class FileController {
}
}
async getList(req, res) {
getList = async(req: RequestType, res: Response) => {
if (!req.user) {
return
@@ -266,7 +286,7 @@ class FileController {
}
}
async getDownloadToken(req, res) {
getDownloadToken = async(req: RequestType, res: Response) => {
if (!req.user) {
return
@@ -289,7 +309,7 @@ class FileController {
}
}
async getDownloadTokenVideo(req, res) {
getDownloadTokenVideo = async(req: RequestType, res: Response) => {
if (!req.user) {
return
@@ -298,7 +318,7 @@ class FileController {
try {
const user = req.user;
const cookie = req.headers.uuid
const cookie = req.headers.uuid as string;
const tempToken = await fileService.getDownloadTokenVideo(user, cookie);
@@ -313,7 +333,7 @@ class FileController {
}
}
async removeTempToken(req, res) {
removeTempToken = async(req: RequestType, res: Response) => {
if (!req.user) {
return
@@ -323,8 +343,9 @@ class FileController {
const user = req.user
const tempToken = req.params.tempToken;
await fileService.removeTempToken(user, tempToken);
const currentUUID = req.params.uuid;
await fileService.removeTempToken(user, tempToken, currentUUID);
res.send();
@@ -337,84 +358,7 @@ class FileController {
}
}
async transcodeVideo(req, res) {
if (!req.user) {
return;
}
try {
console.log("transcode request", req.body.file._id)
const user = req.user;
const body = req.body;
await fileService.transcodeVideo(user, body);
res.send("Finished");
} catch (e) {
const code = e.code || 500;
console.log(e.message, e.exception)
return res.status(code).send();
}
}
async removeTranscodeVideo(req, res) {
if (!req.user) {
return;
}
try {
const fileID = req.body.id;
const userID = req.user._id;
await fileService.removeTranscodeVideo(userID, fileID);
res.send();
} catch (e) {
const code = e.code || 500;
console.log(e);
res.status(code).send()
}
}
async streamTranscodedVideo(req, res) {
if (!req.auth || !req.user) {
return;
}
try {
console.log("stream request transcoded", req.params.id)
const fileID = req.params.id;
const userID = req.user._id;
const headers = req.headers;
await fileService.streamTranscodedVideo(userID, fileID, headers, res);
} catch (e) {
const code = e.code || 500;
const message = e.message || e;
console.log(message, e);
res.status(code).send();
}
}
async streamVideo(req, res) {
streamVideo = async(req: RequestType, res: Response) => {
if (!req.auth || !req.user) {
return;
@@ -425,11 +369,23 @@ class FileController {
const user = req.user;
const fileID = req.params.id;
const headers = req.headers;
//tempStorage[req.params.uuid] = uuid.v4();
console.log("stream request", req.params.id)
console.log("stream request 2", tempStorage);
// req.on("close", () => {
// console.log("req closed stream");
// })
// req.on("abort", () => {
// console.log("Aborted");
// })
await fileService.streamVideo(user, fileID, headers, res);
await this.chunkService.streamVideo(user, fileID, headers, res, req);
//console.log("stream finished");
} catch (e) {
const code = e.code || 500;
@@ -441,7 +397,7 @@ class FileController {
}
async downloadFile(req, res) {
downloadFile = async(req: RequestType, res: Response) => {
if (!req.auth || !req.user) {
return;
@@ -454,7 +410,7 @@ class FileController {
const user = req.user;
const fileID = req.params.id;
await fileService.downloadFile(user, fileID, res);
await this.chunkService.downloadFile(user, fileID, res);
} catch (e) {
@@ -466,7 +422,7 @@ class FileController {
}
}
async getSuggestedList(req, res) {
getSuggestedList = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -491,7 +447,7 @@ class FileController {
}
}
async renameFile(req, res) {
renameFile = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -517,7 +473,7 @@ class FileController {
}
async moveFile(req, res) {
moveFile = async(req: RequestType, res: Response) => {
console.log("move request");
@@ -547,7 +503,7 @@ class FileController {
}
async deleteFile(req, res) {
deleteFile = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -558,7 +514,7 @@ class FileController {
const userID = req.user._id;
const fileID = req.body.id;
await fileService.deleteFile(userID, fileID);
await this.chunkService.deleteFile(userID, fileID);
res.send()
@@ -572,4 +528,4 @@ class FileController {
}
}
module.exports = FileController;
export default FileController;
backend/controllers/folder.js → backend/controllers/folder.ts
+29 -14
View File
@@ -1,13 +1,28 @@
const FolderService = require("../services/FolderService");
import FolderService from "../services/FolderService";
import { Request, Response } from "express";
import {UserInterface} from "../models/user";
import MongoService from "../services/ChunkService/MongoService";
import FileSystemService from "../services/ChunkService/FileSystemService";
import S3Service from "../services/ChunkService/S3Service";
const folderService = new FolderService();
interface RequestType extends Request {
user?: UserInterface,
}
type ChunkServiceType = MongoService | FileSystemService | S3Service;
class FolderController {
constructor() {
chunkService: ChunkServiceType;
constructor(chunkService: ChunkServiceType) {
this.chunkService = chunkService;
}
async uploadFolder(req, res) {
uploadFolder = async(req: RequestType, res: Response) => {
if (!req.user) {
@@ -31,7 +46,7 @@ class FolderController {
}
}
async deleteFolder(req, res) {
deleteFolder = async(req: RequestType, res: Response) => {
if (!req.user) {
@@ -44,7 +59,7 @@ class FolderController {
const folderID = req.body.id;
const parentList = req.body.parentList
await folderService.deleteFolder(userID, folderID, parentList);
await this.chunkService.deleteFolder(userID, folderID, parentList);
res.send();
@@ -57,7 +72,7 @@ class FolderController {
}
}
async deleteAll(req, res) {
deleteAll = async(req: RequestType, res: Response) => {
if (!req.user) {
@@ -68,7 +83,7 @@ class FolderController {
const userID = req.user._id;
await folderService.deleteAll(userID);
await this.chunkService.deleteAll(userID);
res.send();
@@ -81,7 +96,7 @@ class FolderController {
}
}
async getInfo(req, res) {
getInfo = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -105,7 +120,7 @@ class FolderController {
}
}
async getSubfolderList(req, res) {
getSubfolderList = async(req: RequestType, res: Response) => {
if (!req.user) {
@@ -115,7 +130,7 @@ class FolderController {
try {
const userID = req.user._id;
const folderID = req.query.id;
const folderID = req.query.id as string;
const {folderIDList, folderNameList} = await folderService.getFolderSublist(userID, folderID);
@@ -130,7 +145,7 @@ class FolderController {
}
}
async getFolderList(req, res) {
getFolderList = async(req: RequestType, res: Response) => {
if (!req.user) {
@@ -155,7 +170,7 @@ class FolderController {
}
}
async moveFolder(req, res) {
moveFolder = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -181,7 +196,7 @@ class FolderController {
}
}
async renameFolder(req, res) {
renameFolder = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
@@ -208,4 +223,4 @@ class FolderController {
}
}
module.exports = FolderController;
export default FolderController;
backend/controllers/storage.js
-31
View File
@@ -1,31 +0,0 @@
const env = require("../enviroment/env")
const disk = require('diskusage');
class StorageController {
constructor() {
}
async getStorageInfo(req, res) {
if (!req.user) {
return;
}
try {
const info = await disk.check(env.root);
res.send(info)
} catch (e) {
console.log(e);
res.status(500).send(e);
}
}
}
module.exports = StorageController;
backend/controllers/storage.ts
+41
View File
@@ -0,0 +1,41 @@
import disk from "diskusage";
import env from "../enviroment/env";
import { Request, Response } from "express";
import { UserInterface } from "../models/user";
interface RequestType extends Request {
user?: UserInterface,
}
class StorageController {
constructor() {
}
getStorageInfo = async(req: RequestType, res: Response) => {
if (!req.user) {
return;
}
try {
if (!env.root || env.root.length === 0) {
return;
}
const info = await disk.check(env.root!);
res.send(info)
} catch (e) {
console.log(e);
res.status(500).send(e);
}
}
}
export default StorageController;
backend/controllers/user.js → backend/controllers/user.ts
+28 -20
View File
@@ -1,21 +1,29 @@
const env = require("../enviroment/env");
const UserService = require("../services/UserService")
import env from "../enviroment/env";
import UserService from "../services/UserService";
import { Request, Response } from "express";
import { UserInterface } from "../models/user";
const UserProvider = new UserService();
interface RequestType extends Request {
user?: UserInterface,
encryptedToken?: string
}
class UserController {
constructor() {
}
async getUser(req, res) {
getUser = async(req: RequestType, res: Response) => {
try {
const user = req.user;
user.tokens = undefined;
user.tempTokens = undefined;
user.password = undefined;
const user = req.user!;
user.tokens = [];
user.tempTokens = [];
user.password = '';
user.privateKey = undefined;
user.publicKey = undefined;
@@ -27,7 +35,7 @@ class UserController {
}
}
async login(req, res) {
login = async(req: RequestType, res: Response) => {
try {
@@ -35,9 +43,9 @@ class UserController {
const {user, token} = await UserProvider.login(body);
user.tokens = undefined;
user.tempTokens = undefined;
user.password = undefined;
user.tokens = [];
user.tempTokens = [];
user.password = '';
user.privateKey = undefined;
user.publicKey = undefined;
@@ -52,7 +60,7 @@ class UserController {
}
}
async logout(req, res) {
logout = async(req: RequestType, res: Response) => {
if (!req.user) {
@@ -74,13 +82,13 @@ class UserController {
}
}
async logoutAll(req, res) {
logoutAll = async(req: RequestType, res: Response) => {
try {
const user = req.user;
await UserProvider.logoutAll(user);
await UserProvider.logoutAll(user!);
res.send();
@@ -90,7 +98,7 @@ class UserController {
}
}
async createUser(req, res) {
createUser = async(req: RequestType, res: Response) => {
if (env.createAcctBlocked) {
@@ -101,9 +109,9 @@ class UserController {
const {user, token} = await UserProvider.create(req.body);
user.tokens = undefined;
user.tempTokens = undefined;
user.password = undefined;
user.tokens = [];
user.tempTokens = []
user.password = '';
user.privateKey = undefined;
user.publicKey = undefined;
@@ -118,7 +126,7 @@ class UserController {
}
}
async changePassword(req, res) {
changePassword = async(req: RequestType, res: Response) => {
if (!req.user) {
@@ -145,4 +153,4 @@ class UserController {
}
}
module.exports = UserController;
export default UserController;
backend/db/mongoose.js
-10
View File
@@ -1,10 +0,0 @@
const mongoose = require("mongoose");
const env = require("../enviroment/env");
mongoose.connect(env.mongoURL, {
useNewUrlParser: true,
useCreateIndex: true,
useUnifiedTopology: true,
})
module.exports = mongoose;
backend/db/mongoose.ts
+43
View File
@@ -0,0 +1,43 @@
import mongoose from "mongoose";
import env from "../enviroment/env";
import fs from "fs";
const DBUrl = env.mongoURL as string;
if (env.useDocumentDB === "true") {
console.log("Using DocumentDB");
if (env.documentDBBundle === "true") {
const fileBuffer = fs.readFileSync('./rds-combined-ca-bundle.pem');
const mongooseCertificateConnect = mongoose as any;
mongooseCertificateConnect.connect(DBUrl, {
useCreateIndex: true,
useUnifiedTopology: true,
sslValidate: true,
sslCA: fileBuffer
})
} else {
mongoose.connect(DBUrl, {
useNewUrlParser: true,
useCreateIndex: true,
useUnifiedTopology: true,
sslValidate: true,
})
}
} else {
mongoose.connect(DBUrl, {
useNewUrlParser: true,
useCreateIndex: true,
useUnifiedTopology: true,
})
}
export default mongoose;
backend/db/mongooseServerUtils.ts
+12
View File
@@ -0,0 +1,12 @@
import mongoose from "mongoose";
import env from "../enviroment/env";
mongoose.connect(env.mongoURL!, {
useNewUrlParser: true,
useCreateIndex: true,
useUnifiedTopology: true,
socketTimeoutMS: 30000000,
keepAlive: true,
})
export default mongoose;
backend/db/s3.ts
+11
View File
@@ -0,0 +1,11 @@
import AWS from "aws-sdk";
import env from "../enviroment/env";
AWS.config.update({
accessKeyId: env.s3ID,
secretAccessKey: env.s3Key
});
const s3 = new AWS.S3();
export default s3;
backend/db/utils/fileUtils/index.js → backend/db/utils/fileUtils/index.ts
+47 -52
View File
@@ -1,98 +1,104 @@
const mongoose = require("../../mongoose")
import mongoose from "../../mongoose";
import {ObjectID} from "mongodb";
import {FileInterface} from "../../../models/file";
import {UserInterface} from "../../../models/user";
import { QueryInterface } from "../../../utils/createQuery";
const conn = mongoose.connection;
const ObjectID = require('mongodb').ObjectID
const DbUtil = function() {
class DbUtil {
this.getPublicFile = async(fileID) => {
constructor() {
}
getPublicFile = async(fileID: string) => {
let file = await conn.db.collection("fs.files")
.findOne({"_id": ObjectID(fileID)});
.findOne({"_id": new ObjectID(fileID)}) as FileInterface;
return file;
}
this.removeOneTimePublicLink = async(fileID) => {
removeOneTimePublicLink = async(fileID: string) => {
const file = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": ObjectID(fileID)}, {
"$unset": {"metadata.linkType": "", "metadata.link": ""}});
.findOneAndUpdate({"_id": new ObjectID(fileID)}, {
"$unset": {"metadata.linkType": "", "metadata.link": ""}}) as FileInterface;
return file;
}
this.removeLink = async(fileID, userID) => {
removeLink = async(fileID: string, userID: string) => {
const file = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": ObjectID(fileID),
.findOneAndUpdate({"_id": new ObjectID(fileID),
"metadata.owner": userID},
{"$unset": {"metadata.linkType": "", "metadata.link": ""}})
{"$unset": {"metadata.linkType": "", "metadata.link": ""}}) as FileInterface;
return file;
}
this.makePublic = async(fileID, userID, token) => {
makePublic = async(fileID: string, userID: string, token: string) => {
const file = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": ObjectID(fileID),
.findOneAndUpdate({"_id": new ObjectID(fileID),
"metadata.owner": userID},
{"$set": {"metadata.linkType": "public", "metadata.link": token}})
{"$set": {"metadata.linkType": "public", "metadata.link": token}}) as FileInterface
return file;
}
this.getPublicInfo = async(fileID, tempToken) => {
getPublicInfo = async(fileID: string, tempToken: string) => {
const file = await conn.db.collection("fs.files")
.findOne({"_id": ObjectID(fileID), "metadata.link": tempToken})
.findOne({"_id": new ObjectID(fileID), "metadata.link": tempToken}) as FileInterface;
return file;
}
this.makeOneTimePublic = async(fileID, userID, token) => {
makeOneTimePublic = async(fileID: string, userID: string, token: string) => {
const file = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": ObjectID(fileID),
.findOneAndUpdate({"_id": new ObjectID(fileID),
"metadata.owner": userID},
{"$set": {"metadata.linkType": "one", "metadata.link": token}})
{"$set": {"metadata.linkType": "one", "metadata.link": token}}) as FileInterface;
return file;
}
this.getFileInfo = async(fileID, userID) => {
getFileInfo = async(fileID: string, userID: string) => {
const file = await conn.db.collection("fs.files")
.findOne({"metadata.owner": userID, "_id": ObjectID(fileID)});
.findOne({"metadata.owner": userID, "_id": new ObjectID(fileID)}) as FileInterface;
return file;
}
this.getQuickList = async(userID) => {
getQuickList = async(userID: string) => {
const fileList = await conn.db.collection("fs.files")
.find({"metadata.owner": userID})
.sort({uploadDate: -1})
.limit(10)
.toArray();
.toArray() as FileInterface[];
return fileList;
}
this.getList = async(queryObj, sortBy, limit) => {
getList = async(queryObj: QueryInterface, sortBy: string, limit: number) => {
const fileList = await conn.db.collection("fs.files")
.find(queryObj)
.sort(sortBy)
.limit(limit)
.toArray();
.toArray() as FileInterface[];
return fileList;
}
this.removeTempToken = async(user, tempToken) => {
removeTempToken = async(user: UserInterface, tempToken: string) => {
user.tempTokens = user.tempTokens.filter((filterToken) => {
return filterToken.token !== tempToken;
@@ -101,66 +107,55 @@ const DbUtil = function() {
return user;
}
this.removeTranscodeVideo = async(fileID, userID) => {
const file = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": ObjectID(fileID), "metadata.owner": userID},
{"$unset": {"metadata.transcoded": "", "metadata.transcodedIV": "",
"metadata.transcoded_size": "", "metadata.transcodedID": ""}})
return file;
}
this.getFileSearchList = async(userID, searchQuery) => {
getFileSearchList = async(userID: string, searchQuery: RegExp) => {
const fileList = await conn.db.collection("fs.files")
.find({"metadata.owner": userID, "filename": searchQuery})
.limit(10)
.toArray();
.toArray() as FileInterface[];
return fileList;
}
this.renameFile = async(fileID, userID, title) => {
renameFile = async(fileID: string, userID: string, title: string) => {
const file = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": ObjectID(fileID),
"metadata.owner": userID}, {"$set": {"filename": title}})
.findOneAndUpdate({"_id": new ObjectID(fileID),
"metadata.owner": userID}, {"$set": {"filename": title}}) as FileInterface;
return file;
}
this.moveFile = async(fileID, userID, parent, parentList) => {
moveFile = async(fileID: string, userID: string, parent: string, parentList: string) => {
const file = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": ObjectID(fileID),
.findOneAndUpdate({"_id": new ObjectID(fileID),
"metadata.owner": userID}, {"$set": {"metadata.parent": parent, "metadata.parentList": parentList}})
return file;
}
this.getFileListByParent = async(userID, parentListString) => {
getFileListByParent = async(userID: string, parentListString: string) => {
const fileList = await conn.db.collection("fs.files")
.find({"metadata.owner": userID,
"metadata.parentList": {$regex : `.*${parentListString}.*`}}).toArray();
"metadata.parentList": {$regex : `.*${parentListString}.*`}}).toArray() as FileInterface[];
return fileList;
}
this.getFileListByOwner = async(userID) => {
getFileListByOwner = async(userID: string) => {
const fileList = await conn.db.collection("fs.files")
.find({"metadata.owner": userID}).toArray();
.find({"metadata.owner": userID}).toArray() as FileInterface[];
return fileList;
}
this.removeChunksByID = async(fileID) => {
removeChunksByID = async(fileID: string) => {
await conn.db.collection("fs.chunks").deleteMany({files_id: fileID});
}
}
module.exports = DbUtil;
export default DbUtil;
backend/db/utils/folderUtils/index.js
-61
View File
@@ -1,61 +0,0 @@
const Folder = require("../../../models/folder");
const ObjectID = require('mongodb').ObjectID
const DbUtil = function() {
this.getFolderSearchList = async(userID, searchQuery) => {
const folderList = await Folder.find({"owner": userID, "name": searchQuery}).limit(10);
return folderList;
}
this.getFolderInfo = async(folderID, userID) => {
const folder = await Folder.findOne({"owner": userID, "_id": ObjectID(folderID)});
return folder;
}
this.getFolderListByParent = async(userID, parent, sortBy) => {
const folderList = await Folder.find({"owner": userID, "parent": parent})
.sort(sortBy);
return folderList;
}
this.getFolderListBySearch = async(userID, searchQuery, sortBy) => {
const folderList = await Folder.find({"name": searchQuery,"owner": userID})
.sort(sortBy);
return folderList;
}
this.moveFolder = async(folderID, userID, parent, parentList) => {
const folder = await Folder.findOneAndUpdate({"_id": ObjectID(folderID),
"owner": userID}, {"$set": {"parent": parent, "parentList": parentList}});
return folder;
}
this.renameFolder = async(folderID, userID, title) => {
const folder = await Folder.findOneAndUpdate({"_id": ObjectID(folderID),
"owner": userID}, {"$set": {"name": title}});
return folder;
}
this.findAllFoldersByParent = async(parentID, userID) => {
const folderList = await Folder.find({"parentList": parentID, "owner": userID});
return folderList;
}
}
module.exports = DbUtil;
backend/db/utils/folderUtils/index.ts
+64
View File
@@ -0,0 +1,64 @@
import Folder, { FolderInterface } from "../../../models/folder";
import {ObjectID} from "mongodb";
class DbUtil {
constructor() {
}
getFolderSearchList = async(userID: string, searchQuery: RegExp) => {
const folderList = await Folder.find({"owner": userID, "name": searchQuery}).limit(10) as FolderInterface[];
return folderList;
}
getFolderInfo = async(folderID: string, userID: string) => {
const folder = await Folder.findOne({"owner": userID, "_id": new ObjectID(folderID)}) as FolderInterface;
return folder;
}
getFolderListByParent = async(userID: string, parent: string, sortBy: string) => {
const folderList = await Folder.find({"owner": userID, "parent": parent})
.sort(sortBy) as FolderInterface[];
return folderList;
}
getFolderListBySearch = async(userID: string, searchQuery: string, sortBy: string) => {
const folderList = await Folder.find({"name": searchQuery,"owner": userID})
.sort(sortBy) as FolderInterface[];
return folderList;
}
moveFolder = async(folderID: string, userID: string, parent: string, parentList: string[]) => {
const folder = await Folder.findOneAndUpdate({"_id": new ObjectID(folderID),
"owner": userID}, {"$set": {"parent": parent, "parentList": parentList}}) as FolderInterface;
return folder;
}
renameFolder = async(folderID: string, userID: string, title: string) => {
const folder = await Folder.findOneAndUpdate({"_id": new ObjectID(folderID),
"owner": userID}, {"$set": {"name": title}}) as FolderInterface;
return folder;
}
findAllFoldersByParent = async(parentID: string, userID: string) => {
const folderList = await Folder.find({"parentList": parentID, "owner": userID}) as FolderInterface[];
return folderList;
}
}
export default DbUtil;
backend/enviroment/env.js
-9
View File
@@ -1,9 +0,0 @@
module.exports = {
key: process.env.KEY,
newKey: process.env.NEW_KEY,
password: process.env.PASSWORD,
createAcctBlocked: process.env.BLOCK_CREATE_ACCOUNT,
root: process.env.ROOT,
url: process.env.URL,
mongoURL: process.env.MONGODB_URL
}
backend/enviroment/env.ts
+16
View File
@@ -0,0 +1,16 @@
export default {
key: process.env.KEY,
newKey: process.env.NEW_KEY,
password: process.env.PASSWORD,
createAcctBlocked: process.env.BLOCK_CREATE_ACCOUNT,
root: process.env.ROOT,
url: process.env.URL,
mongoURL: process.env.MONGODB_URL,
dbType: process.env.DB_TYPE,
fsDirectory: process.env.FS_DIRECTORY,
s3ID: process.env.S3_ID,
s3Key: process.env.S3_KEY,
s3Bucket: process.env.S3_BUCKET,
useDocumentDB: process.env.USE_DOCUMENT_DB,
documentDBBundle: process.env.DOCUMENT_DB_BUNDLE
}
backend/enviroment/getEnvVariables.ts
+23
View File
@@ -0,0 +1,23 @@
import path from "path";
const getEnvVariables = () => {
const configPath = path.join(__dirname, "..", "..", "config");
const processType = process.env.NODE_ENV;
if (processType === 'production' || processType === undefined) {
require('dotenv').config({ path: configPath + "/prod.env"})
} else if (processType === 'development') {
require('dotenv').config({ path: configPath + "/dev.env"})
} else if (processType === 'test') {
require('dotenv').config({ path: configPath + "/test.env"})
}
}
export default getEnvVariables;
backend/express-routers/file.js → backend/express-routers/file.ts
+30 -15
View File
@@ -1,16 +1,35 @@
const express = require("express");
const router = new express.Router;
const auth = require("../middleware/auth");
const tempAuth = require("../middleware/tempAuth")
const tempAuthVideo = require("../middleware/tempAuthVideo")
const FileController = require("../controllers/file");
import {Router} from "express";
import auth from "../middleware/auth";
import tempAuth from "../middleware/tempAuth";
import tempAuthVideo from "../middleware/tempAuthVideo";
import FileController from "../controllers/file";
import env from "../enviroment/env";
import MongoService from "../services/ChunkService/MongoService";
import FileSystemService from "../services/ChunkService/FileSystemService";
import S3Service from "../services/ChunkService/S3Service";
const fileController = new FileController()
let fileController: FileController;
if (env.dbType === "mongo") {
const mongoService = new MongoService();
fileController = new FileController(mongoService);
} else if (env.dbType === "fs") {
const fileSystemService = new FileSystemService();
fileController = new FileController(fileSystemService);
} else {
const s3Service = new S3Service();
fileController = new FileController(s3Service);
}
const router = Router();
router.post("/file-service/upload", auth, fileController.uploadFile);
router.post("/file-service/transcode-video", auth, fileController.transcodeVideo);
router.get("/file-service/thumbnail/:id", auth, fileController.getThumbnail);
router.get("/file-service/full-thumbnail/:id", auth, fileController.getFullThumbnail);
@@ -29,8 +48,6 @@ router.get("/file-service/download/get-token", auth, fileController.getDownloadT
router.get("/file-service/download/get-token-video", auth, fileController.getDownloadTokenVideo);
router.get("/file-service/stream-video-transcoded/:id/:tempToken/:uuid", tempAuthVideo, fileController.streamTranscodedVideo);
router.get("/file-service/stream-video/:id/:tempToken/:uuid", tempAuthVideo, fileController.streamVideo);
router.get("/file-service/download/:id/:tempToken", tempAuth, fileController.downloadFile);
@@ -47,10 +64,8 @@ router.patch("/file-service/move", auth, fileController.moveFile);
router.delete("/file-service/remove-link/:id", auth, fileController.removeLink);
router.delete("/file-service/remove/token-video/:tempToken", auth, fileController.removeTempToken);
router.delete("/file-service/transcode-video/remove", auth, fileController.removeTranscodeVideo);
router.delete("/file-service/remove/token-video/:tempToken/:uuid", auth, fileController.removeTempToken);
router.delete("/file-service/remove", auth, fileController.deleteFile);
module.exports = router;
export default router;
backend/express-routers/folder.js
-23
View File
@@ -1,23 +0,0 @@
const express = require("express");
const router = new express.Router;
const auth = require("../middleware/auth");
const FolderController = require("../controllers/folder");
const folderController = new FolderController();
router.post("/folder-service/upload", auth, folderController.uploadFolder);
router.delete("/folder-service/remove", auth, folderController.deleteFolder);
router.delete("/folder-service/remove-all", auth, folderController.deleteAll);
router.get("/folder-service/info/:id", auth, folderController.getInfo);
router.get("/folder-service/subfolder-list", auth, folderController.getSubfolderList);
router.get("/folder-service/list", auth, folderController.getFolderList);
router.patch("/folder-service/rename", auth, folderController.renameFolder);
router.patch("/folder-service/move", auth, folderController.moveFolder);
module.exports = router;
backend/express-routers/folder.ts
+45
View File
@@ -0,0 +1,45 @@
import {Router} from "express";
import auth from "../middleware/auth";
import env from "../enviroment/env";
import MongoService from "../services/ChunkService/MongoService";
import FileSystemService from "../services/ChunkService/FileSystemService";
import S3Service from "../services/ChunkService/S3Service";
import FolderController from "../controllers/folder";
let folderController: FolderController;
if (env.dbType === "mongo") {
const mongoService = new MongoService();
folderController = new FolderController(mongoService);
} else if (env.dbType === "fs") {
const fileSystemService = new FileSystemService();
folderController = new FolderController(fileSystemService);
} else {
const s3Service = new S3Service();
folderController = new FolderController(s3Service);
}
const router = Router();
router.post("/folder-service/upload", auth, folderController.uploadFolder);
router.delete("/folder-service/remove", auth, folderController.deleteFolder);
router.delete("/folder-service/remove-all", auth, folderController.deleteAll);
router.get("/folder-service/info/:id", auth, folderController.getInfo);
router.get("/folder-service/subfolder-list", auth, folderController.getSubfolderList);
router.get("/folder-service/list", auth, folderController.getFolderList);
router.patch("/folder-service/rename", auth, folderController.renameFolder);
router.patch("/folder-service/move", auth, folderController.moveFolder);
export default router;
backend/express-routers/storage.js
-9
View File
@@ -1,9 +0,0 @@
const express = require("express");
const router = new express.Router;
const auth = require("../middleware/auth");
const StorageController = require("../controllers/storage");
const storageController = new StorageController();
router.get("/storage-service/info", auth, storageController.getStorageInfo);
module.exports = router;
backend/express-routers/storage.ts
+11
View File
@@ -0,0 +1,11 @@
import {Router} from "express";
import auth from "../middleware/auth";
import StorageController from "../controllers/storage";
const storageController = new StorageController();
const router = Router();
router.get("/storage-service/info", auth, storageController.getStorageInfo);
export default router;
backend/express-routers/user.js → backend/express-routers/user.ts
+7 -5
View File
@@ -1,9 +1,11 @@
const express = require("express");
const router = new express.Router;
const auth = require("../middleware/auth");
const UserController = require("../controllers/user");
import { Router } from "express";
import auth from "../middleware/auth";
import UserController from "../controllers/user";
const userController = new UserController();
const router = Router();
router.get("/user-service/user", auth, userController.getUser);
router.post("/user-service/login", userController.login);
@@ -16,4 +18,4 @@ router.post("/user-service/create", userController.createUser);
router.post("/user-service/change-password", auth, userController.changePassword);
module.exports = router;
export default router;
backend/key/getKey.ts
+33
View File
@@ -0,0 +1,33 @@
const prompt = require('password-prompt')
import env from "../enviroment/env";
import crypto from "crypto";
import getWebUIKey from "./getWebUIKey";
const getKey = async() => {
if (process.env.KEY) {
// For Docker
const password = process.env.KEY;
env.key = crypto.createHash("md5").update(password).digest("hex");
} else if (process.env.NODE_ENV === "production") {
let password = await getWebUIKey();
password = crypto.createHash("md5").update(password).digest("hex");
env.key = password;
} else {
let password = "1234";
password = crypto.createHash("md5").update(password).digest("hex");
env.key = password;
}
}
export default getKey;
backend/key/getWebUIKey.ts
+52
View File
@@ -0,0 +1,52 @@
import express, {Request, Response} from "express";
import http from "http";
import path from "path";
import bodyParser from "body-parser";
const app = express();
const getWebUIKey = () => {
const publicPath = path.join(__dirname, "..", "..", "webUI");
return new Promise<string>((resolve, reject) => {
app.use(express.static(publicPath));
app.use(express.json());
app.use(bodyParser.json({limit: "50mb"}));
app.use(bodyParser.urlencoded({limit: "50mb", extended: true, parameterLimit:50000}))
app.post("/submit", (req: Request, res: Response) => {
const password = req.body.password;
if (password && password.length > 0) {
console.log("Got WebUI key");
res.send();
server.close();
resolve(password);
}
})
app.get("*", (req: Request, res: Response) => {
res.sendFile(path.join(publicPath, "index.html"));
})
const port = process.env.HTTP_PORT || process.env.PORT || "3000";
const url = process.env.DOCKER ? undefined : "localhost";
const server = http.createServer(app) as any;
server.listen(port, url, () => {
console.log(`\nPlease navigate to http://localhost:${port} to enter encryption key\n`)
});
})
}
export default getWebUIKey;
backend/middleware/auth.js → backend/middleware/auth.ts
+21 -9
View File
@@ -1,18 +1,30 @@
const jwt = require("jsonwebtoken");
const User = require("../models/user");
const env = require("../enviroment/env");
import jwt from "jsonwebtoken";
import User, {UserInterface} from "../models/user";
import env from "../enviroment/env";
import {Request, Response, NextFunction} from "express";
const auth = async(req, res, next) => {
interface RequestType extends Request {
user?: UserInterface,
token?: string,
encryptedToken?: string,
}
type jwtType = {
iv: Buffer,
_id: string,
}
const auth = async(req: RequestType, res: Response, next: NextFunction) => {
try {
const token = req.header("Authorization").replace("Bearer ", "");
const token = req.header("Authorization")!.replace("Bearer ", "");
const decoded = await jwt.verify(token, env.password);
const decoded = jwt.verify(token, env.password!) as jwtType;
const iv = decoded.iv;
const user = await User.findOne({_id: decoded._id});
const user = await User.findOne({_id: decoded._id}) as UserInterface;
const encrpytionKey = user.getEncryptionKey();
const encryptedToken = user.encryptToken(token, encrpytionKey, iv);
@@ -46,4 +58,4 @@ const auth = async(req, res, next) => {
}
}
module.exports = auth;
export default auth;
backend/middleware/tempAuth.js → backend/middleware/tempAuth.ts
+20 -7
View File
@@ -1,18 +1,31 @@
const jwt = require("jsonwebtoken");
const User = require("../models/user");
const env = require("../enviroment/env");
import jwt from "jsonwebtoken";
import User, {UserInterface} from "../models/user";
import env from "../enviroment/env";
import {Request, Response, NextFunction} from "express";
const tempAuth = async(req, res, next) => {
interface RequestType extends Request {
user?: UserInterface,
auth?: boolean,
encryptedTempToken?: string,
}
type jwtType = {
iv: Buffer,
_id: string,
}
const tempAuth = async(req: RequestType, res: Response, next: NextFunction) => {
try {
const token = req.params.tempToken;
const decoded = await jwt.verify(token, env.password);
const decoded = jwt.verify(token, env.password!) as jwtType;
const iv = decoded.iv;
const user = await User.findOne({_id: decoded._id})
const user = await User.findOne({_id: decoded._id}) as UserInterface;
const encrpytionKey = user.getEncryptionKey();
const encryptedToken = user.encryptToken(token, encrpytionKey, iv);
@@ -54,4 +67,4 @@ const tempAuth = async(req, res, next) => {
}
}
module.exports = tempAuth;
export default tempAuth;
backend/middleware/tempAuthVideo.js → backend/middleware/tempAuthVideo.ts
+21 -7
View File
@@ -1,14 +1,28 @@
const jwt = require("jsonwebtoken");
const User = require("../models/user");
const env = require("../enviroment/env");
import jwt from "jsonwebtoken";
import User, {UserInterface} from "../models/user";
import env from "../enviroment/env";
import {Request, Response, NextFunction} from "express";
const tempAuthVideo = async(req, res, next) => {
interface RequestType extends Request {
user?: UserInterface,
auth?: boolean,
encryptedTempToken?: string,
}
type jwtType = {
iv: Buffer,
_id: string,
cookie: string
}
const tempAuthVideo = async(req: RequestType, res: Response, next: NextFunction) => {
try {
const token = req.params.tempToken;
const decoded = await jwt.verify(token, env.password);
const decoded = jwt.verify(token, env.password!) as jwtType;
const iv = decoded.iv;
@@ -17,7 +31,7 @@ const tempAuthVideo = async(req, res, next) => {
throw new Error("Cookie mismatch")
}
const user = await User.findOne({_id: decoded._id})
const user = await User.findOne({_id: decoded._id}) as UserInterface;
const encrpytionKey = user.getEncryptionKey();
const encryptedToken = user.encryptToken(token, encrpytionKey, iv);
@@ -53,4 +67,4 @@ const tempAuthVideo = async(req, res, next) => {
}
}
module.exports = tempAuthVideo;
export default tempAuthVideo;
backend/models/file.ts
+87
View File
@@ -0,0 +1,87 @@
import mongoose, {Document} from "mongoose";
import { Binary } from "mongodb";
const fileSchema = new mongoose.Schema({
length: {
type: Number,
required: true,
},
chunkSize: {
type: Number,
},
uploadDate: {
type: Date,
required: true
},
filename: {
type: String,
required: true
},
metadata: {
type: {
owner: {
type: String,
required: true
},
parent: {
type: String,
required: true
},
parentList: {
type: String,
required: true
},
hasThumbnail: {
type: Boolean,
required: true
},
isVideo: {
type: Boolean,
required: true
},
thumbnailID: String,
size: {
type: Number,
required: true,
},
IV: {
type: Binary,
required: true
},
linkType: String,
link: String,
filePath: String,
s3ID: String,
},
required: true
}
})
export interface FileInterface extends Document {
length: number,
chunkSize: number,
uploadDate: string,
filename: string,
lastErrorObject: {updatedExisting: any}
metadata: {
owner: string,
parent: string,
parentList: string,
hasThumbnail: boolean,
isVideo: boolean,
thumbnailID?: string,
size: number,
IV: Buffer,
linkType?: 'one' | 'public',
link?: string,
filePath?: string,
s3ID?: string,
}
}
const File = mongoose.model<FileInterface>("fs.files", fileSchema);
export default File;
backend/models/fileSystem.ts
+69
View File
@@ -0,0 +1,69 @@
import mongoose, {Document} from "mongoose";
const fileSystemSchema = new mongoose.Schema({
name: {
type: String,
required: true,
},
owner: {
type: String,
required: true
},
path: {
type: String,
required: true
},
parent: {
type: String,
required: true,
},
parentList: {
type: Array,
required: true
},
hasThumbnail: {
type: Boolean,
required: true
},
thumbnailID: {
type: String
},
originalSize: {
type: Number,
required: true
},
size: {
type: Number,
required: true
},
isVideo: {
type: Boolean,
required: true
},
IV: {
type: Buffer,
required: true
}
}, {
timestamps: true
})
export interface FileSystemInterface extends Document {
name: string,
owner: string,
path: string,
parent: string,
parentList: string[],
hasThumbnail: boolean,
thumbnailID?: string,
originalSize: number,
size: number,
isVideo: boolean,
IV: Buffer
}
const FileSystem = mongoose.model<FileSystemInterface>("FileSystem", fileSystemSchema);
export default FileSystem;
backend/models/folder.js → backend/models/folder.ts
+11 -3
View File
@@ -1,4 +1,4 @@
const mongoose = require("mongoose");
import mongoose, {Document} from "mongoose";
const folderSchema = new mongoose.Schema({
@@ -27,6 +27,14 @@ const folderSchema = new mongoose.Schema({
timestamps: true
})
const Folder = mongoose.model("Folder", folderSchema);
export interface FolderInterface extends Document {
name: string,
parent: string,
owner: string,
parentList: string[],
_doc?: any
}
module.exports = Folder;
const Folder = mongoose.model<FolderInterface>("Folder", folderSchema);
export default Folder;
backend/models/thumbnail.js
-25
View File
@@ -1,25 +0,0 @@
const mongoose = require("mongoose");
const thumbnailSchema = new mongoose.Schema({
name: {
type: String,
required: true,
},
owner: {
type: String,
required: true
},
data: {
type: Buffer,
required: true
}
}, {
timestamps: true
})
const Thumbnail = mongoose.model("Thumbnail", thumbnailSchema);
module.exports = Thumbnail;
backend/models/thumbnail.ts
+41
View File
@@ -0,0 +1,41 @@
import mongoose, {Document} from "mongoose";
const thumbnailSchema = new mongoose.Schema({
name: {
type: String,
required: true,
},
owner: {
type: String,
required: true
},
data: {
type: Buffer,
},
path: {
type: String
},
IV: {
type: Buffer,
},
s3ID: String
}, {
timestamps: true
})
export interface ThumbnailInterface extends Document {
_id: string,
name: string,
owner: string,
data?: any,
path?: string,
IV?: Buffer,
s3ID? : string,
}
const Thumbnail = mongoose.model<ThumbnailInterface>("Thumbnail", thumbnailSchema);
export default Thumbnail;
backend/models/user.js → backend/models/user.ts
+73 -73
View File
@@ -1,11 +1,11 @@
const mongoose = require("mongoose");
const validator = require("validator");
const crypto = require("crypto");
const bcrypt = require("bcrypt");
const jwt = require("jsonwebtoken");
const env = require("../enviroment/env");
import mongoose, {Document} from "mongoose";
import validator from "validator";
import crypto from "crypto";
import bcrypt from "bcrypt";
import jwt from "jsonwebtoken";
import env from "../enviroment/env";
const userSchema = mongoose.Schema({
const userSchema = new mongoose.Schema({
name:
{
@@ -18,7 +18,7 @@ const userSchema = mongoose.Schema({
trim: true,
unique: true,
lowercase: true,
validate(value) {
validate(value: any): any {
if (!validator.isEmail(value)) {
throw new Error("Email is invalid");
}
@@ -28,7 +28,7 @@ const userSchema = mongoose.Schema({
type: String,
trim: true,
required: true,
validate(value) {
validate(value: any): any {
if (value.length < 6) {
throw new Error("Password Length Not Sufficent");
}
@@ -57,12 +57,28 @@ const userSchema = mongoose.Schema({
timestamps: true
})
// userSchema.virtual("files", {
// ref: "fs.files",
// localField: "_id"
// })
export interface UserInterface extends Document {
_id: string,
name: string,
email: string,
password: string,
tokens: any[],
tempTokens: any[],
privateKey?: string,
publicKey?: string,
token?: string,
userSchema.pre("save", async function(next) {
getEncryptionKey: () => Buffer | undefined;
generateTempAuthToken: () => any;
generateTempAuthTokenVideo: (cookie: string) => any;
encryptToken: (tempToken: any, key: any, publicKey: any) => any;
findByCreds: (email: string, password: string) => UserInterface;
generateAuthToken: () => any;
generateEncryptionKeys: () => Promise<void>;
changeEncryptionKey: (randomKey: Buffer) => void;
}
userSchema.pre("save", async function(this: any, next: any) {
const user = this;
@@ -73,7 +89,7 @@ userSchema.pre("save", async function(next) {
next();
})
userSchema.statics.findByCreds = async(email, password) => {
userSchema.statics.findByCreds = async(email: string, password: string) => {
const user = await User.findOne({email});
@@ -110,8 +126,7 @@ userSchema.methods.generateAuthToken = async function() {
const iv = crypto.randomBytes(16);
const user = this;
let token = jwt.sign({_id:user._id.toString(), iv}, env.password);
const publicKey = user.publicKey;
let token = jwt.sign({_id:user._id.toString(), iv}, env.password!);
const encryptionKey = user.getEncryptionKey();
@@ -123,19 +138,18 @@ userSchema.methods.generateAuthToken = async function() {
return token;
}
userSchema.methods.encryptToken = function(token, key, iv) {
userSchema.methods.encryptToken = function(token: string, key: string, iv: any) {
iv = Buffer.from(iv, "hex")
const TOKEN_CIPHER_KEY = crypto.createHash('sha256').update(key).digest();
const cipher = crypto.createCipheriv('aes-256-cbc', TOKEN_CIPHER_KEY, iv);
let encryptedText = cipher.update(token);
encryptedText = Buffer.concat([encryptedText, cipher.final()]).toString("hex");
const encryptedText = cipher.update(token);
return encryptedText;
return Buffer.concat([encryptedText, cipher.final()]).toString("hex");;
}
userSchema.methods.decryptToken = function(encryptedToken, key, iv) {
userSchema.methods.decryptToken = function(encryptedToken: any, key: string, iv: any) {
encryptedToken = Buffer.from(encryptedToken, "hex");
iv = Buffer.from(iv, "hex")
@@ -143,17 +157,16 @@ userSchema.methods.decryptToken = function(encryptedToken, key, iv) {
const TOKEN_CIPHER_KEY = crypto.createHash('sha256').update(key).digest();
const decipher = crypto.createDecipheriv('aes-256-cbc', TOKEN_CIPHER_KEY, iv)
let tokenDecrypted = decipher.update(encryptedToken);
tokenDecrypted = Buffer.concat([tokenDecrypted, decipher.final()]).toString();
const tokenDecrypted = decipher.update(encryptedToken);
return tokenDecrypted;
return Buffer.concat([tokenDecrypted, decipher.final()]).toString();
}
userSchema.methods.generateEncryptionKeys = async function() {
const user = this;
const userPassword = user.password;
const masterPassword = env.key;
const masterPassword = env.key!;
const randomKey = crypto.randomBytes(32);
@@ -165,10 +178,9 @@ userSchema.methods.generateEncryptionKeys = async function() {
const MASTER_CIPHER_KEY = crypto.createHash('sha256').update(masterPassword).digest();
const masterCipher = crypto.createCipheriv('aes-256-cbc', MASTER_CIPHER_KEY, iv);
let masterEncryptedText = masterCipher.update(encryptedText);
masterEncryptedText = Buffer.concat([masterEncryptedText, masterCipher.final()]).toString("hex");
const masterEncryptedText = masterCipher.update(encryptedText);
user.privateKey = masterEncryptedText;
user.privateKey = Buffer.concat([masterEncryptedText, masterCipher.final()]).toString("hex");
user.publicKey = iv.toString("hex");
await user.save();
@@ -176,32 +188,39 @@ userSchema.methods.generateEncryptionKeys = async function() {
userSchema.methods.getEncryptionKey = function() {
const user = this;
const userPassword = user.password;
const masterEncryptedText = user.privateKey;
const masterPassword = env.key;
const iv = Buffer.from(user.publicKey, "hex");
try {
const user = this;
const userPassword = user.password;
const masterEncryptedText = user.privateKey;
const masterPassword = env.key!;
const iv = Buffer.from(user.publicKey, "hex");
const USER_CIPHER_KEY = crypto.createHash('sha256').update(userPassword).digest();
const MASTER_CIPHER_KEY = crypto.createHash('sha256').update(masterPassword).digest();
const USER_CIPHER_KEY = crypto.createHash('sha256').update(userPassword).digest();
const MASTER_CIPHER_KEY = crypto.createHash('sha256').update(masterPassword).digest();
const unhexMasterText = Buffer.from(masterEncryptedText, "hex");
const masterDecipher = crypto.createDecipheriv('aes-256-cbc', MASTER_CIPHER_KEY, iv)
let masterDecrypted = masterDecipher.update(unhexMasterText);
masterDecrypted = Buffer.concat([masterDecrypted, masterDecipher.final()])
const unhexMasterText = Buffer.from(masterEncryptedText, "hex");
const masterDecipher = crypto.createDecipheriv('aes-256-cbc', MASTER_CIPHER_KEY, iv)
let masterDecrypted = masterDecipher.update(unhexMasterText);
masterDecrypted = Buffer.concat([masterDecrypted, masterDecipher.final()])
let decipher = crypto.createDecipheriv('aes-256-cbc', USER_CIPHER_KEY, iv);
let decrypted = decipher.update(masterDecrypted);
decrypted = Buffer.concat([decrypted, decipher.final()]);
let decipher = crypto.createDecipheriv('aes-256-cbc', USER_CIPHER_KEY, iv);
let decrypted = decipher.update(masterDecrypted);
decrypted = Buffer.concat([decrypted, decipher.final()]);
return decrypted;
return decrypted;
} catch (e) {
console.log("Get Encryption Key Error", e);
return undefined;
}
}
userSchema.methods.changeEncryptionKey = async function(randomKey) {
userSchema.methods.changeEncryptionKey = async function(randomKey: Buffer) {
const user = this;
const userPassword = user.password;
const masterPassword = env.key;
const masterPassword = env.key!;
const iv = crypto.randomBytes(16);
const USER_CIPHER_KEY = crypto.createHash('sha256').update(userPassword).digest();
@@ -211,10 +230,9 @@ userSchema.methods.changeEncryptionKey = async function(randomKey) {
const MASTER_CIPHER_KEY = crypto.createHash('sha256').update(masterPassword).digest();
const masterCipher = crypto.createCipheriv('aes-256-cbc', MASTER_CIPHER_KEY, iv);
let masterEncryptedText = masterCipher.update(encryptedText);
masterEncryptedText = Buffer.concat([masterEncryptedText, masterCipher.final()]).toString("hex");
const masterEncryptedText = masterCipher.update(encryptedText);
user.privateKey = masterEncryptedText;
user.privateKey = Buffer.concat([masterEncryptedText, masterCipher.final()]).toString("hex");
user.publicKey = iv.toString("hex");
await user.save();
@@ -222,25 +240,10 @@ userSchema.methods.changeEncryptionKey = async function(randomKey) {
userSchema.methods.generateTempAuthToken = async function() {
// const user = this;
// let token = jwt.sign({_id:user._id.toString()}, env.password);
// const publicKey = user.publicKey;
// const encryptionKey = user.getEncryptionKey();
// const encryptedToken = user.encryptToken(token, encryptionKey, publicKey);
// user.tokens = user.tokens.concat({token: encryptedToken});
// await user.save();
// return token;
const iv = crypto.randomBytes(16);
const user = this;
const token = jwt.sign({_id:user._id.toString(), iv}, env.password, {expiresIn:2});
const publicKey = user.publicKey;
const user = this as UserInterface;
const token = jwt.sign({_id:user._id.toString(), iv}, env.password!, {expiresIn: "3000ms"});
const encryptionKey = user.getEncryptionKey();
const encryptedToken = user.encryptToken(token, encryptionKey, iv);
@@ -251,25 +254,22 @@ userSchema.methods.generateTempAuthToken = async function() {
return token;
}
userSchema.methods.generateTempAuthTokenVideo = async function(cookie) {
userSchema.methods.generateTempAuthTokenVideo = async function(cookie: string) {
const iv = crypto.randomBytes(16);
const user = this;
const token = jwt.sign({_id:user._id.toString(), cookie, iv}, env.password, {expiresIn:"5h"});
//const publicKey = user.publicKey;
const token = jwt.sign({_id:user._id.toString(), cookie, iv}, env.password!, {expiresIn:"5h"});
const encryptionKey = user.getEncryptionKey();
const encryptedToken = user.encryptToken(token, encryptionKey, iv);
//user.tempTokens = user.tempTokens.concat({token});
user.tempTokens = user.tempTokens.concat({token: encryptedToken});
await user.save();
return token;
}
const User = mongoose.model<UserInterface>("User", userSchema);
const User = mongoose.model("User", userSchema);
module.exports = User;
export default User;
backend/server/server.ts
Executable
+67
View File
@@ -0,0 +1,67 @@
import express, {Request, Response} from "express";
import path from "path";
import userRouter from "../express-routers/user";
import fileRouter from "../express-routers/file";
import folderRouter from "../express-routers/folder";
import storageRouter from "../express-routers/storage";
import bodyParser from "body-parser";
import https from "https";
import fs from "fs";
import helmet from "helmet";
import busboy from "connect-busboy";
import compression from "compression";
import http from "http";
const app = express();
const publicPath = path.join(__dirname, "..", "..", "public");
let server: any;
let serverHttps: any;
if (process.env.SSL === 'true') {
const cert = fs.readFileSync("certificate.crt")
const ca = fs.readFileSync("certificate.ca-bundle");
const key = fs.readFileSync("certificate.key");
const options = {
cert,
ca,
key
}
serverHttps = https.createServer( options, app );
}
server = http.createServer(app);
require("../db/mongoose");
app.use(helmet())
app.use(compression());
app.use(express.json());
app.use(express.static(publicPath));
app.use(bodyParser.json({limit: "50mb"}));
app.use(bodyParser.urlencoded({limit: "50mb", extended: true, parameterLimit:50000}))
app.use(busboy({
highWaterMark: 2 * 1024 * 1024,
}));
app.use(userRouter, fileRouter, folderRouter, storageRouter);
const nodeMode = process.env.NODE_ENV ? "Production" : "Development/Testing";
console.log("Node Enviroment Mode:", nodeMode);
app.get("*", (_: Request, res: Response) => {
res.sendFile(path.join(publicPath, "index.html"))
})
export default {server, serverHttps};
backend/server/serverStart.ts
+42
View File
@@ -0,0 +1,42 @@
import getEnvVariables from "../enviroment/getEnvVariables";
getEnvVariables();
import getKey from "../key/getKey";
import servers from "./server";
const {server, serverHttps} = servers;
const serverStart = async() => {
await getKey();
console.log("ENV", process.env.NODE_ENV);
if (process.env.NODE_ENV === 'production' && process.env.SSL === "true") {
server.listen(process.env.HTTP_PORT,process.env.URL, () => {
console.log("Http Server Running On Port:", process.env.HTTP_PORT);
})
serverHttps.listen(process.env.HTTPS_PORT, function () {
console.log( 'Https Server Running On Port:', process.env.HTTPS_PORT);
} );
} else if (process.env.NODE_ENV === 'production') {
const port = process.env.HTTP_PORT || process.env.PORT;
server.listen(port, process.env.URL, () => {
console.log("Http Server (No-SSL) Running On Port:", port);
})
} else {
const port = process.env.HTTP_PORT || process.env.PORT;
server.listen(port, process.env.URL, () => {
console.log("Development Server Running On Port:", port);
})
}
}
serverStart();
backend/services/ChunkService/FileSystemService.ts
+398
View File
@@ -0,0 +1,398 @@
import { Response, Request } from "express";
import ChunkInterface from "./utils/ChunkInterface";
import { UserInterface } from "../../models/user";
import NotAuthorizedError from "../../utils/NotAuthorizedError";
import NotFoundError from "../../utils/NotFoundError";
import crypto from "crypto";
import getBusboyData from "./utils/getBusboyData";
import videoChecker from "../../utils/videoChecker";
import fs from "fs";
import uuid from "uuid";
import awaitUploadStreamFS from "./utils/awaitUploadStreamFS";
import File, { FileInterface } from "../../models/file";
import getFileSize from "./utils/getFileSize";
import DbUtilFile from "../../db/utils/fileUtils/index";
import awaitStream from "./utils/awaitStream";
import createThumbnailFS from "./utils/createThumbnailFS";
import imageChecker from "../../utils/imageChecker";
import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail";
import streamToBuffer from "../../utils/streamToBuffer";
import User from "../../models/user";
import env from "../../enviroment/env";
import removeChunksFS from "./utils/removeChunksFS";
import getPrevIVFS from "./utils/getPrevIVFS";
import awaitStreamVideo from "./utils/awaitStreamVideo";
import fixStartChunkLength from "./utils/fixStartChunkLength";
import Folder from "../../models/folder";
const dbUtilsFile = new DbUtilFile();
class FileSystemService implements ChunkInterface {
constructor() {
}
uploadFile = async(user: UserInterface, busboy: any, req: Request) => {
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const initVect = crypto.randomBytes(16);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const cipher = crypto.createCipheriv('aes256', CIPHER_KEY, initVect);
const {file, filename, formData} = await getBusboyData(busboy);
const parent = formData.get("parent") || "/"
const parentList = formData.get("parentList") || "/";
const size = formData.get("size") || ""
let hasThumbnail = false;
let thumbnailID = ""
const isVideo = videoChecker(filename)
const systemFileName = uuid.v4();
const metadata = {
owner: user._id,
parent,
parentList,
hasThumbnail,
thumbnailID,
isVideo,
size,
IV: initVect,
filePath: env.fsDirectory + systemFileName
}
const fileWriteStream = fs.createWriteStream(metadata.filePath);
const totalStreamsToErrorCatch = [file, cipher, fileWriteStream];
await awaitUploadStreamFS(file.pipe(cipher), fileWriteStream, req, metadata.filePath, totalStreamsToErrorCatch);
const date = new Date();
const encryptedFileSize = await getFileSize(metadata.filePath);
const currentFile = new File({
filename,
uploadDate: date.toISOString(),
length: encryptedFileSize,
metadata
});
await currentFile.save();
console.log(currentFile);
const imageCheck = imageChecker(currentFile.filename);
if (currentFile.length < 15728640 && imageCheck) {
const updatedFile = await createThumbnailFS(currentFile, filename, user);
return updatedFile;
} else {
return currentFile;
}
}
downloadFile = async(user: UserInterface, fileID: string, res: Response) => {
const currentFile: FileInterface = await dbUtilsFile.getFileInfo(fileID, user._id);
if (!currentFile) throw new NotFoundError("Download File Not Found");
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const filePath = currentFile.metadata.filePath!;
const IV = currentFile.metadata.IV.buffer as Buffer;
const readStream = fs.createReadStream(filePath);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + currentFile.filename + '"');
res.set('Content-Length', currentFile.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
}
getThumbnail = async(user: UserInterface, id: string) => {
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const thumbnail = await Thumbnail.findById(id) as ThumbnailInterface;
if (thumbnail.owner !== user._id.toString()) {
throw new NotAuthorizedError('Thumbnail Unauthorized Error');
}
const iv = thumbnail.IV!;
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv);
const readStream = fs.createReadStream(thumbnail.path!);
const allStreamsToErrorCatch = [readStream, decipher];
const bufferData = await streamToBuffer(readStream.pipe(decipher), allStreamsToErrorCatch);
return bufferData;
}
getFullThumbnail = async(user: UserInterface, fileID: string, res: Response) => {
const userID = user._id;
const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("File Thumbnail Not Found");
const password = user.getEncryptionKey();
const IV = file.metadata.IV.buffer as Buffer;
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const readStream = fs.createReadStream(file.metadata.filePath!);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
console.log("Sending Full Thumbnail...")
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
console.log("Full thumbnail sent");
}
streamVideo = async(user: UserInterface, fileID: string, headers: any, res: Response, req: Request) => {
const userID = user._id;
const currentFile: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!currentFile) throw new NotFoundError("Video File Not Found");
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const fileSize = currentFile.metadata.size;
const range = headers.range
const parts = range.replace(/bytes=/, "").split("-")
let start = parseInt(parts[0], 10)
let end = parts[1]
? parseInt(parts[1], 10)
: fileSize-1
const IV = currentFile.metadata.IV.buffer as Buffer;
const chunksize = (end-start)+1
let head = {
'Content-Range': 'bytes ' + start + '-' + end + '/' + fileSize,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/mp4'}
let currentIV = IV;
let fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start);
if (+start === 0) {
fixedStart = 0;
}
const fixedEnd = currentFile.length;
const differenceStart = start - fixedStart;
if (fixedStart !== 0 && start !== 0) {
currentIV = await getPrevIVFS(fixedStart - 16, currentFile.metadata.filePath!) as Buffer;
}
const readStream = fs.createReadStream(currentFile.metadata.filePath!, {
start: fixedStart,
end: fixedEnd,
});
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, currentIV);
decipher.setAutoPadding(false);
res.writeHead(206, head);
const allStreamsToErrorCatch = [readStream, decipher];
readStream.pipe(decipher);
// req.on("close", () => {
// // console.log("req closed");
// readStream.destroy();
// })
const tempUUID = req.params.uuid;
await awaitStreamVideo(start, end, differenceStart, decipher, res, req, tempUUID, allStreamsToErrorCatch);
console.log("Video Stream Finished");
readStream.destroy();
}
getPublicDownload = async(fileID: string, tempToken: any, res: Response) => {
const file: FileInterface = await dbUtilsFile.getPublicFile(fileID);
if (!file || !file.metadata.link || file.metadata.link !== tempToken) {
throw new NotAuthorizedError("File Not Public");
}
const user = await User.findById(file.metadata.owner) as UserInterface;
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key");
const IV = file.metadata.IV.buffer as Buffer;
const readStream = fs.createReadStream(file.metadata.filePath!);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
if (file.metadata.linkType === "one") {
console.log("removing public link");
await dbUtilsFile.removeOneTimePublicLink(fileID);
}
}
deleteFile = async(userID: string, fileID: string) => {
const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("Delete File Not Found Error");
if (file.metadata.thumbnailID) {
const thumbnail = await Thumbnail.findById(file.metadata.thumbnailID) as ThumbnailInterface;
const thumbnailPath = thumbnail.path!;
await removeChunksFS(thumbnailPath);
await Thumbnail.deleteOne({_id: file.metadata.thumbnailID});
}
await removeChunksFS(file.metadata.filePath!);
await File.deleteOne({_id: file._id});
}
deleteFolder = async(userID: string, folderID: string, parentList: string[]) => {
const parentListString = parentList.toString()
await Folder.deleteMany({"owner": userID, "parentList": { $all: parentList}})
await Folder.deleteMany({"owner": userID, "_id": folderID});
const fileList = await dbUtilsFile.getFileListByParent(userID, parentListString);
if (!fileList) throw new NotFoundError("Delete File List Not Found");
for (let i = 0; i < fileList.length; i++) {
const currentFile = fileList[i];
try {
if (currentFile.metadata.thumbnailID) {
const thumbnail = await Thumbnail.findById(currentFile.metadata.thumbnailID) as ThumbnailInterface;
const thumbnailPath = thumbnail.path!;
await removeChunksFS(thumbnailPath);
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID});
}
await removeChunksFS(currentFile.metadata.filePath!);
await File.deleteOne({_id: currentFile._id});
} catch (e) {
console.log("Could not delete file", currentFile.filename, currentFile._id);
}
}
}
deleteAll = async(userID: string) => {
console.log("remove all request")
await Folder.deleteMany({"owner": userID});
const fileList = await dbUtilsFile.getFileListByOwner(userID);
if (!fileList) throw new NotFoundError("Delete All File List Not Found Error");
for (let i = 0; i < fileList.length; i++) {
const currentFile = fileList[i];
try {
if (currentFile.metadata.thumbnailID) {
const thumbnail = await Thumbnail.findById(currentFile.metadata.thumbnailID) as ThumbnailInterface;
const thumbnailPath = thumbnail.path!;
await removeChunksFS(thumbnailPath);
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID});
}
await removeChunksFS(currentFile.metadata.filePath!);
await File.deleteOne({_id: currentFile._id});
} catch (e) {
console.log("Could Not Remove File", currentFile.filename, currentFile._id);
}
}
}
}
export default FileSystemService;
backend/services/ChunkService/MongoService.ts
+382
View File
@@ -0,0 +1,382 @@
import mongoose from "../../db/mongoose";
import { Response, Request } from "express";
import DbUtilFolder from "../../db/utils/folderUtils";
import DbUtilFile from "../../db/utils/fileUtils";
import crypto from "crypto";
import videoChecker from "../../utils/videoChecker"
import imageChecker from "../../utils/imageChecker"
import { ObjectID } from "mongodb";
import createThumbnail from "./utils/createThumbnail";
import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail";
import NotAuthorizedError from "../../utils/NotAuthorizedError";
import NotFoundError from "../../utils/NotFoundError";
import {GridFSBucketWriteStream} from "mongodb";
import awaitStream from "./utils/awaitStream";
import awaitUploadStream from "./utils/awaitUploadStream";
import User, { UserInterface } from "../../models/user";
import Folder from "../../models/folder";
import { FileInterface } from "../../models/file";
import getBusboyData from "./utils/getBusboyData";
import ChunkInterface from "./utils/ChunkInterface";
import fixStartChunkLength from "./utils/fixStartChunkLength";
import getPrevIVMongo from "./utils/getPrevIVMongo";
import awaitStreamVideo from "./utils/awaitStreamVideo";
const conn = mongoose.connection;
const dbUtilsFile = new DbUtilFile();
class MongoService implements ChunkInterface {
constructor() {
}
uploadFile = async(user: UserInterface, busboy: any, req: Request) => {
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
let bucketStream: GridFSBucketWriteStream;
const initVect = crypto.randomBytes(16);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const cipher = crypto.createCipheriv('aes256', CIPHER_KEY, initVect);
const {file, filename, formData} = await getBusboyData(busboy);
const parent = formData.get("parent") || "/"
const parentList = formData.get("parentList") || "/";
const size = formData.get("size") || ""
let hasThumbnail = false;
let thumbnailID = ""
const isVideo = videoChecker(filename)
const metadata = {
owner: user._id,
parent,
parentList,
hasThumbnail,
thumbnailID,
isVideo,
size,
IV: initVect
}
let bucket = new mongoose.mongo.GridFSBucket(conn.db);
bucketStream = bucket.openUploadStream(filename, {metadata});
const allStreamsToErrorCatch = [file, cipher, bucketStream];
const finishedFile = await awaitUploadStream(file.pipe(cipher), bucketStream, req, allStreamsToErrorCatch) as FileInterface;
const imageCheck = imageChecker(filename);
if (finishedFile.length < 15728640 && imageCheck) {
const updatedFile = await createThumbnail(finishedFile, filename, user);
return updatedFile;
} else {
return finishedFile;
}
}
downloadFile = async(user: UserInterface, fileID: string, res: Response) => {
const currentFile = await dbUtilsFile.getFileInfo(fileID, user._id);
if (!currentFile) throw new NotFoundError("Download File Not Found");
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const bucket = new mongoose.mongo.GridFSBucket(conn.db);
const IV = currentFile.metadata.IV.buffer as Buffer;
const readStream = bucket.openDownloadStream(new ObjectID(fileID));
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + currentFile.filename + '"');
res.set('Content-Length', currentFile.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
}
getThumbnail = async(user: UserInterface, id: string) => {
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const thumbnail = await Thumbnail.findById(id) as ThumbnailInterface;
if (thumbnail.owner !== user._id.toString()) {
throw new NotAuthorizedError('Thumbnail Unauthorized Error');
}
const iv = thumbnail.data.slice(0, 16);
const chunk = thumbnail.data.slice(16);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv);
const decryptedThumbnail = Buffer.concat([decipher.update(chunk), decipher.final()]);
return decryptedThumbnail;
}
getFullThumbnail = async(user: UserInterface, fileID: string, res: Response) => {
const userID = user._id;
const file = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("File Thumbnail Not Found");
const bucket = new mongoose.mongo.GridFSBucket(conn.db);
const password = user.getEncryptionKey();
const IV = file.metadata.IV.buffer as Buffer;
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const readStream = bucket.openDownloadStream(new ObjectID(fileID))
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size.toString());
console.log("Sending Full Thumbnail...")
const allStreamsToErrorCatch = [readStream, decipher];
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
console.log("Full thumbnail sent");
}
getPublicDownload = async(fileID: string, tempToken: any, res: Response) => {
const file = await dbUtilsFile.getPublicFile(fileID);
if (!file || !file.metadata.link || file.metadata.link !== tempToken) {
throw new NotAuthorizedError("File Not Public");
}
const user = await User.findById(file.metadata.owner) as UserInterface;
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key");
const bucket = new mongoose.mongo.GridFSBucket(conn.db);
const IV = file.metadata.IV.buffer as Buffer;
const readStream = bucket.openDownloadStream(new ObjectID(fileID))
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
if (file.metadata.linkType === "one") {
console.log("removing public link");
await dbUtilsFile.removeOneTimePublicLink(fileID);
}
}
streamVideo = async(user: UserInterface, fileID: string, headers: any, res: Response, req: Request) => {
const userID = user._id;
const currentFile = await dbUtilsFile.getFileInfo(fileID, userID);
if (!currentFile) throw new NotFoundError("Video File Not Found");
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const fileSize = currentFile.metadata.size;
const range = headers.range
const parts = range.replace(/bytes=/, "").split("-")
let start = parseInt(parts[0], 10)
let end = parts[1]
? parseInt(parts[1], 10)
: fileSize-1
const chunksize = (end-start)+1
const IV = currentFile.metadata.IV.buffer as Buffer;
let head = {
'Content-Range': 'bytes ' + start + '-' + end + '/' + fileSize,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/mp4'}
let currentIV = IV;
let fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start);
if (+start === 0) {
fixedStart = 0;
}
const fixedEnd = currentFile.length;
const differenceStart = start - fixedStart;
if (fixedStart !== 0 && start !== 0) {
currentIV = await getPrevIVMongo(fixedStart - 16, fileID) as Buffer;
}
const bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024
});
const readStream = bucket.openDownloadStream(new ObjectID(fileID), {
start: fixedStart,
end: fixedEnd,
});
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, currentIV);
decipher.setAutoPadding(false);
res.writeHead(206, head);
const allStreamsToErrorCatch = [readStream, decipher];
readStream.pipe(decipher);
// req.on("close", () => {
// // console.log("req closed");
// readStream.destroy();
// })
const tempUUID = req.params.uuid;
await awaitStreamVideo(start, end, differenceStart, decipher, res, req, tempUUID, allStreamsToErrorCatch);
readStream.destroy();
}
deleteFile = async(userID: string, fileID: string) => {
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255,
});
const file = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("Delete File Not Found Error");
if (file.metadata.thumbnailID) {
await Thumbnail.deleteOne({_id: file.metadata.thumbnailID});
}
await bucket.delete(new ObjectID(fileID));
}
deleteFolder = async(userID: string, folderID: string, parentList: string[]) => {
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255
});
const parentListString = parentList.toString()
await Folder.deleteMany({"owner": userID, "parentList": { $all: parentList}})
await Folder.deleteMany({"owner": userID, "_id": folderID});
const fileList = await dbUtilsFile.getFileListByParent(userID, parentListString);
if (!fileList) throw new NotFoundError("Delete File List Not Found");
for (let i = 0; i < fileList.length; i++) {
const currentFile = fileList[i];
try {
if (currentFile.metadata.thumbnailID) {
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID});
}
await bucket.delete(new ObjectID(currentFile._id));
} catch (e) {
console.log("Could not delete file", currentFile.filename, currentFile._id);
}
}
}
deleteAll = async(userID: string) => {
console.log("remove all request")
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255
});
await Folder.deleteMany({"owner": userID});
const fileList = await dbUtilsFile.getFileListByOwner(userID);
if (!fileList) throw new NotFoundError("Delete All File List Not Found Error");
for (let i = 0; i < fileList.length; i++) {
const currentFile = fileList[i];
try {
if (currentFile.metadata.thumbnailID) {
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID})
}
await bucket.delete(new ObjectID(currentFile._id));
} catch (e) {
console.log("Could Not Remove File", currentFile.filename, currentFile._id);
}
}
}
}
export default MongoService;
backend/services/ChunkService/S3Service.ts
+429
View File
@@ -0,0 +1,429 @@
import File, { FileInterface } from "../../models/file";
import User, {UserInterface} from "../../models/user";
import s3 from "../../db/s3";
import env from "../../enviroment/env";
import { Response, Request } from "express";
import ChunkInterface from "./utils/ChunkInterface";
import NotAuthorizedError from "../../utils/NotAuthorizedError";
import NotFoundError from "../../utils/NotFoundError";
import crypto from "crypto";
import getBusboyData from "./utils/getBusboyData";
import videoChecker from "../../utils/videoChecker";
import uuid from "uuid";
import awaitUploadStreamS3 from "./utils/awaitUploadStreamS3";
import awaitStream from "./utils/awaitStream";
import createThumbnailS3 from "./utils/createThumbnailS3";
import imageChecker from "../../utils/imageChecker";
import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail";
import streamToBuffer from "../../utils/streamToBuffer";
import removeChunksS3 from "./utils/removeChunksS3";
import fixStartChunkLength from "./utils/fixStartChunkLength";
import fixEndChunkLength from "./utils/fixEndChunkLength";
import getPrevIVS3 from "./utils/getPrevIVS3";
import awaitStreamVideo from "./utils/awaitStreamVideo";
import Folder from "../../models/folder";
import DbUtilFile from "../../db/utils/fileUtils/index";
const dbUtilsFile = new DbUtilFile();
class S3Service implements ChunkInterface {
constructor() {
}
uploadFile = async(user: UserInterface, busboy: any, req: Request) => {
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const initVect = crypto.randomBytes(16);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const cipher = crypto.createCipheriv('aes256', CIPHER_KEY, initVect);
const {file, filename, formData} = await getBusboyData(busboy);
const parent = formData.get("parent") || "/"
const parentList = formData.get("parentList") || "/";
const size = formData.get("size") || ""
let hasThumbnail = false;
let thumbnailID = ""
const isVideo = videoChecker(filename)
const randomS3ID = uuid.v4();
const metadata = {
owner: user._id,
parent,
parentList,
hasThumbnail,
thumbnailID,
isVideo,
size,
IV: initVect,
s3ID: randomS3ID
}
const params = {
Bucket: env.s3Bucket,
Body : file.pipe(cipher),
Key : randomS3ID
};
await awaitUploadStreamS3(params);
const date = new Date();
const encryptedFileSize = size;
const currentFile = new File({
filename,
uploadDate: date.toISOString(),
length: encryptedFileSize,
metadata
});
await currentFile.save();
const imageCheck = imageChecker(currentFile.filename);
if (currentFile.length < 15728640 && imageCheck) {
console.log("Creating thumbnail...")
const updatedFile = await createThumbnailS3(currentFile, filename, user);
return updatedFile;
} else {
return currentFile;
}
}
downloadFile = async(user: UserInterface, fileID: string, res: Response) => {
const currentFile: FileInterface = await dbUtilsFile.getFileInfo(fileID, user._id);
if (!currentFile) throw new NotFoundError("Download File Not Found");
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const IV = currentFile.metadata.IV.buffer as Buffer;
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + currentFile.filename + '"');
res.set('Content-Length', currentFile.metadata.size.toString());
const params: any = {Bucket: env.s3Bucket, Key: currentFile.metadata.s3ID!};
const s3ReadStream = s3.getObject(params).createReadStream();
const allStreamsToErrorCatch = [s3ReadStream, decipher];
await awaitStream(s3ReadStream.pipe(decipher), res, allStreamsToErrorCatch);
}
streamVideo = async(user: UserInterface, fileID: string, headers: any, res: Response, req: Request) => {
const userID = user._id;
const currentFile: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!currentFile) throw new NotFoundError("Video File Not Found");
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const fileSize = currentFile.metadata.size;
const range = headers.range
const parts = range.replace(/bytes=/, "").split("-")
let start = parseInt(parts[0], 10)
let end = parts[1]
? parseInt(parts[1], 10)
: fileSize-1
const chunksize = (end-start)+1
const IV = currentFile.metadata.IV.buffer as Buffer;
let head = {
'Content-Range': 'bytes ' + start + '-' + end + '/' + fileSize,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/mp4'}
let currentIV = IV;
let fixedStart = start % 16 === 0 ? start : fixStartChunkLength(start);
if (+start === 0) {
fixedStart = 0;
}
const fixedEnd = fileSize % 16 === 0 ? fileSize : fixEndChunkLength(fileSize); //end % 16 === 0 ? end + 15: (fixEndChunkLength(end) - 1) + 16;
const differenceStart = start - fixedStart;
if (fixedStart !== 0 && start !== 0) {
currentIV = await getPrevIVS3(fixedStart - 16, currentFile.metadata.s3ID!) as Buffer;
}
const params: any = {Bucket: env.s3Bucket, Key: currentFile.metadata.s3ID!, Range: `bytes=${fixedStart}-${fixedEnd}`};
const s3ReadStream = s3.getObject(params).createReadStream();
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, currentIV);
res.writeHead(206, head);
const allStreamsToErrorCatch = [s3ReadStream, decipher];
s3ReadStream.pipe(decipher);
const tempUUID = req.params.uuid;
// s3ReadStream.on("data", () => {
// console.log("data", tempUUID);
// })
// req.on("close", () => {
// // console.log("Destoying read stream");
// // s3ReadStream.destroy();
// // console.log("Read Stream Destroyed");
// })
// req.on("end", () => {
// console.log("ending stream");
// s3ReadStream.destroy();
// console.log("ended stream")
// })
// req.on("error", () => {
// console.log("req error");
// })
// req.on("pause", () => {
// console.log("req pause")
// })
// req.on("close", () => {
// // console.log("req closed");
// s3ReadStream.destroy();
// })
//req.on("")
// req.on("end", () => {
// console.log("req end");
// })
await awaitStreamVideo(start, end, differenceStart, decipher, res, req, tempUUID, allStreamsToErrorCatch);
console.log("Video stream finished");
s3ReadStream.destroy();
}
getThumbnail = async(user: UserInterface, id: string) => {
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const thumbnail = await Thumbnail.findById(id) as ThumbnailInterface;
if (thumbnail.owner !== user._id.toString()) {
throw new NotAuthorizedError('Thumbnail Unauthorized Error');
}
const iv = thumbnail.IV!;
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv);
const params: any = {Bucket: env.s3Bucket, Key: thumbnail.s3ID!};
const readStream = s3.getObject(params).createReadStream();
const allStreamsToErrorCatch = [readStream, decipher];
const bufferData = await streamToBuffer(readStream.pipe(decipher), allStreamsToErrorCatch);
return bufferData;
}
getFullThumbnail = async(user: UserInterface, fileID: string, res: Response) => {
const userID = user._id;
const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("File Thumbnail Not Found");
const password = user.getEncryptionKey();
const IV = file.metadata.IV.buffer as Buffer;
if (!password) throw new NotAuthorizedError("Invalid Encryption Key")
const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!};
const readStream = s3.getObject(params).createReadStream();
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
console.log("Sending Full Thumbnail...")
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
console.log("Full thumbnail sent");
}
getPublicDownload = async(fileID: string, tempToken: any, res: Response) => {
const file: FileInterface = await dbUtilsFile.getPublicFile(fileID);
if (!file || !file.metadata.link || file.metadata.link !== tempToken) {
throw new NotAuthorizedError("File Not Public");
}
const user = await User.findById(file.metadata.owner) as UserInterface;
const password = user.getEncryptionKey();
if (!password) throw new NotAuthorizedError("Invalid Encryption Key");
const IV = file.metadata.IV.buffer as Buffer;
const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!};
const readStream = s3.getObject(params).createReadStream();
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
if (file.metadata.linkType === "one") {
console.log("removing public link");
await dbUtilsFile.removeOneTimePublicLink(fileID);
}
}
deleteFile = async(userID: string, fileID: string) => {
const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("Delete File Not Found Error");
if (file.metadata.thumbnailID) {
const thumbnail = await Thumbnail.findById(file.metadata.thumbnailID) as ThumbnailInterface;
const paramsThumbnail: any = {Bucket: env.s3Bucket, Key: thumbnail.s3ID!};
await removeChunksS3(paramsThumbnail);
await Thumbnail.deleteOne({_id: file.metadata.thumbnailID});
}
const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!};
await removeChunksS3(params);
await File.deleteOne({_id: file._id});
}
deleteFolder = async(userID: string, folderID: string, parentList: string[]) => {
const parentListString = parentList.toString()
await Folder.deleteMany({"owner": userID, "parentList": { $all: parentList}})
await Folder.deleteMany({"owner": userID, "_id": folderID});
const fileList = await dbUtilsFile.getFileListByParent(userID, parentListString);
if (!fileList) throw new NotFoundError("Delete File List Not Found");
for (let i = 0; i < fileList.length; i++) {
const currentFile = fileList[i];
try {
if (currentFile.metadata.thumbnailID) {
const thumbnail = await Thumbnail.findById(currentFile.metadata.thumbnailID) as ThumbnailInterface;
const paramsThumbnail: any = {Bucket: env.s3Bucket, Key: thumbnail.s3ID!};
await removeChunksS3(paramsThumbnail);
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID});
}
const params: any = {Bucket: env.s3Bucket, Key: currentFile.metadata.s3ID!};
await removeChunksS3(params);
await File.deleteOne({_id: currentFile._id});
} catch (e) {
console.log("Could not delete file", currentFile.filename, currentFile._id);
}
}
}
deleteAll = async(userID: string) => {
console.log("remove all request")
await Folder.deleteMany({"owner": userID});
const fileList = await dbUtilsFile.getFileListByOwner(userID);
if (!fileList) throw new NotFoundError("Delete All File List Not Found Error");
for (let i = 0; i < fileList.length; i++) {
const currentFile = fileList[i];
try {
if (currentFile.metadata.thumbnailID) {
const thumbnail = await Thumbnail.findById(currentFile.metadata.thumbnailID) as ThumbnailInterface;
const paramsThumbnail: any = {Bucket: env.s3Bucket, Key: thumbnail.s3ID!};
await removeChunksS3(paramsThumbnail);
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID});
}
const params: any = {Bucket: env.s3Bucket, Key: currentFile.metadata.s3ID!};
await removeChunksS3(params);
await File.deleteOne({_id: currentFile._id});
} catch (e) {
console.log("Could Not Remove File", currentFile.filename, currentFile._id);
}
}
}
}
export default S3Service;
backend/services/ChunkService/utils/ChunkInterface.ts
+14
View File
@@ -0,0 +1,14 @@
import { UserInterface } from "../../../models/user";
import { FileInterface } from "../../../models/file";
import { Request, Response } from "express";
interface ChunkInterface {
uploadFile: (user: UserInterface, busboy: any, req: Request) => Promise<FileInterface>;
downloadFile: (user: UserInterface, fileID: string, res: Response) => void;
getThumbnail: (user: UserInterface, id: string) => Promise<Buffer>;
getFullThumbnail: (user: UserInterface, fileID: string, res: Response) => void;
getPublicDownload: (fileID: string, tempToken: any, res: Response) => void;
streamVideo: (user: UserInterface, fileID: string, headers: any, res: Response, req: Request) => void;
}
export default ChunkInterface;
backend/services/ChunkService/utils/awaitStream.ts
+24
View File
@@ -0,0 +1,24 @@
const awaitStream = <T>(inputSteam: any, outputStream: any, allStreamsToErrorCatch: any[]) => {
return new Promise<T>((resolve, reject) => {
allStreamsToErrorCatch.forEach((currentStream: any) => {
currentStream.on("error", (e: Error) => {
reject({
message: "Await Stream Input Error",
code: 500,
error: e
})
})
})
inputSteam.pipe(outputStream).on("finish", (data: T) => {
console.log("await stream finished")
resolve(data);
})
})
}
export default awaitStream;
backend/services/ChunkService/utils/awaitStreamVideo.ts
+124
View File
@@ -0,0 +1,124 @@
import {Response, Request} from "express"
import tempStorage from "../../../tempStorage/tempStorage";
import uuid from "uuid";
const awaitStreamVideo = (start: number, end:number, differenceStart: number,
decipher: any, res: Response, req: Request, tempUUID: string, streamsToErrorCatch: any[]) => {
const currentUUID = uuid.v4();
tempStorage[tempUUID] = currentUUID;
return new Promise((resolve, reject) => {
let firstBytesRemoved = false;
//let sizeCounter = 0;
req.on("close", () => {
//console.log("close resolved");
streamsToErrorCatch.forEach((stream) => {
stream.destroy();
})
resolve();
})
req.on("end", () => {
//console.log("End resolved");
resolve();
})
decipher.on("data", (data: Buffer | string) => {
//console.log("check", tempStorage[tempUUID]);
//console.log("check b", tempStorage[tempUUID] !== currentUUID && tempStorage[tempUUID] !== undefined);
//console.log(tempStorage[tempUUID] !== currentUUID, tempStorage[tempUUID])
if (tempStorage[tempUUID] !== currentUUID) {
//console.log("New Stream Requested, Desroying old stream");
streamsToErrorCatch.forEach((stream) => {
stream.destroy();
})
//console.log("Old Stream Desroyed");
//console.log("Old Stream Resolved");
resolve();
//delete tempStorage[tempUUID];
}
//console.log("data", currentUUID);
//console.log("stream passed", currentUUID);
// if (tempStorage[tempUUID] !== undefined && tempStorage[tempUUID] !== currentUUID) {
// console.log("New Stream Requested, Desroying old stream");
// streamsToErrorCatch[0].destroy();
// console.log("Old Stream Desroyed");
// delete tempStorage[tempUUID];
// } else {
// tempStorage[tempUUID] = currentUUID;
// }
if (+start === 0 && +end === 1) {
const dataCoverted = data.toString("hex");
let neededData = dataCoverted.substring(0, 4);
const dataBack = Buffer.from(neededData, "hex");
res.write(dataBack);
//res.flush();
return;
}
if (!firstBytesRemoved) {
const dataCoverted = data.toString("hex");
let neededData = dataCoverted.substring(differenceStart * 2);
const dataBack = Buffer.from(neededData, "hex");
firstBytesRemoved = true;
//sizeCounter += dataBack.length;
res.write(dataBack);
//res.flush();
return;
}
res.write(data);
//res.flush();
//sizeCounter += data.length;
})
decipher.on("end", () => {
//console.log("decipher resolved");
res.end();
resolve();
})
streamsToErrorCatch.forEach((currentStream) => {
currentStream.on("error", (e: Error) => {
reject({
message: "Await Video Stream Input Error",
code: 500,
error: e
})
})
})
})
}
export default awaitStreamVideo;
backend/services/ChunkService/utils/awaitUploadStream.ts
+35
View File
@@ -0,0 +1,35 @@
import removeChunks from "./removeChunks";
import {Request} from "express"
const awaitUploadStream = <T>(inputSteam: any, outputStream: any, req: Request, allStreamsToErrorCatch: any[]) => {
return new Promise<T>((resolve, reject) => {
allStreamsToErrorCatch.forEach((currentStream: any) => {
currentStream.on("error", (e: Error) => {
removeChunks(outputStream);
reject({
message: "Await Stream Input Error",
code: 500,
error: e
})
})
})
req.on("aborted", () => {
console.log("Upload Request Cancelling...");
removeChunks(outputStream);
})
inputSteam.pipe(outputStream).on("finish", (data: T) => {
resolve(data);
})
})
}
export default awaitUploadStream;
backend/services/ChunkService/utils/awaitUploadStreamFS.ts
+35
View File
@@ -0,0 +1,35 @@
import {Request} from "express"
import removeChunksFS from "./removeChunksFS";
const awaitUploadStream = <T>(inputSteam: any, outputStream: any, req: Request, path: string, allStreamsToCatchError: any[]) => {
return new Promise<T>((resolve, reject) => {
allStreamsToCatchError.forEach((currentStream: any) => {
currentStream.on("error", (e: Error) => {
removeChunksFS(path);
reject({
message: "Await Stream Input Error",
code: 500,
error: e
})
})
})
req.on("aborted", () => {
console.log("Upload Request Cancelling...");
removeChunksFS(path);
})
inputSteam.pipe(outputStream).on("finish", (data: T) => {
resolve(data);
})
})
}
export default awaitUploadStream;
backend/services/ChunkService/utils/awaitUploadStreamS3.ts
+19
View File
@@ -0,0 +1,19 @@
import s3 from "../../../db/s3";
const awaitUploadStreamS3 = (params: any) => {
return new Promise((resolve, reject) => {
s3.upload(params, (err: any, data: any) => {
if (err) {
reject("Amazon upload error");
}
resolve();
})
})
}
export default awaitUploadStreamS3;
backend/services/FileService/utils/createThumbnail.js → backend/services/ChunkService/utils/createThumbnail.ts
+24 -24
View File
@@ -1,69 +1,69 @@
const mongoose = require("../../../db/mongoose")
import mongoose from "../../../db/mongoose";
import crypto from "crypto";
import Thumbnail from "../../../models/thumbnail";
import { ObjectID } from "mongodb";
import sharp from "sharp";
import concat from "concat-stream";
import { FileInterface } from "../../../models/file";
import { UserInterface } from "../../../models/user";
const conn = mongoose.connection;
const crypto= require("crypto");
const env = require("../../../enviroment/env");
const Thumbnail = require("../../../models/thumbnail");
const ObjectID = require('mongodb').ObjectID
const sharp = require("sharp");
const concat = require("concat-stream")
const createThumbnail = async(file, filename, user) => {
const createThumbnail = (file: FileInterface, filename: string, user: UserInterface) => {
return new Promise((resolve) => {
return new Promise<FileInterface>((resolve) => {
const password = user.getEncryptionKey();
let CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
let CIPHER_KEY = crypto.createHash('sha256').update(password!).digest()
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255
})
const readStream = bucket.openDownloadStream(ObjectID(file._id))
const readStream = bucket.openDownloadStream(new ObjectID(file._id))
readStream.on("error", (e) => {
readStream.on("error", (e: Error) => {
console.log("File service upload thumbnail error", e);
resolve(file);
})
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, file.metadata.IV);
decipher.on("error", (e) => {
decipher.on("error", (e: Error) => {
console.log("File service upload thumbnail decipher error", e);
resolve(file)
})
try {
const concatStream = concat(async(bufferData) => {
const concatStream = concat(async(bufferData: Buffer) => {
const thumbnailIV = crypto.randomBytes(16);
//CIPHER_KEY = crypto.createHash('sha256').update(process.env.newPassword).digest()
const thumbnailIV = crypto.randomBytes(16);
const thumbnailCipher = crypto.createCipheriv("aes256", CIPHER_KEY, thumbnailIV);
bufferData = Buffer.concat([thumbnailIV, thumbnailCipher.update(bufferData), thumbnailCipher.final()]); //thumbnailCipher.update(bufferData)
bufferData = Buffer.concat([thumbnailIV, thumbnailCipher.update(bufferData), thumbnailCipher.final()]);
const thumbnailModel = new Thumbnail({name: filename, owner: user._id, data: bufferData});
await thumbnailModel.save();
let updatedFile = await conn.db.collection("fs.files")
const getUpdatedFile = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}})
updatedFile = updatedFile.value;
let updatedFile: FileInterface = getUpdatedFile.value;
updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}}
updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}} as FileInterface;
resolve(updatedFile);
}).on("error", (e) => {
}).on("error", (e: Error) => {
console.log("File service upload concat stream error", e);
resolve(file);
})
const imageResize = sharp().resize(300).on("error", (e) => {
const imageResize = sharp().resize(300).on("error", (e: Error) => {
console.log("resize error", e);
resolve(file);
@@ -81,4 +81,4 @@ const createThumbnail = async(file, filename, user) => {
})
}
module.exports = createThumbnail;
export default createThumbnail;
backend/services/ChunkService/utils/createThumbnailFS.ts
+81
View File
@@ -0,0 +1,81 @@
import mongoose from "../../../db/mongoose";
import crypto from "crypto";
import Thumbnail from "../../../models/thumbnail";
import sharp from "sharp";
import {FileInterface} from "../../../models/file";
import {UserInterface} from "../../../models/user";
import fs from "fs";
import uuid from "uuid";
import env from "../../../enviroment/env";
const conn = mongoose.connection;
const createThumbnailFS = (file: FileInterface, filename: string, user: UserInterface) => {
return new Promise<FileInterface>((resolve) => {
const password = user.getEncryptionKey();
let CIPHER_KEY = crypto.createHash('sha256').update(password!).digest()
const thumbnailFilename = uuid.v4();
const readStream = fs.createReadStream(file.metadata.filePath!);
const writeStream = fs.createWriteStream(env.fsDirectory + thumbnailFilename);
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, file.metadata.IV);
readStream.on("error", (e: Error) => {
console.log("File service upload thumbnail error", e);
resolve(file);
})
writeStream.on("error", (e: Error) => {
console.log("File service upload write thumbnail error", e);
resolve(file);
})
decipher.on("error", (e: Error) => {
console.log("File service upload thumbnail decipher error", e);
resolve(file)
})
try {
const thumbnailIV = crypto.randomBytes(16);
const thumbnailCipher = crypto.createCipheriv("aes256", CIPHER_KEY, thumbnailIV);
const imageResize = sharp().resize(300).on("error", (e: Error) => {
console.log("resize error", e);
resolve(file);
})
readStream.pipe(decipher).pipe(imageResize).pipe(thumbnailCipher).pipe(writeStream);
writeStream.on("finish", async() => {
const thumbnailModel = new Thumbnail({name: filename, owner: user._id, IV: thumbnailIV, path: env.fsDirectory + thumbnailFilename});
await thumbnailModel.save();
const getUpdatedFile = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}})
let updatedFile: FileInterface = getUpdatedFile.value;
updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}} as FileInterface
resolve(updatedFile);
})
} catch (e) {
console.log("Thumbnail error", e);
resolve(file);
}
})
}
export default createThumbnailFS;
backend/services/ChunkService/utils/createThumbnailS3.ts
+85
View File
@@ -0,0 +1,85 @@
import mongoose from "../../../db/mongoose";
import crypto from "crypto";
import Thumbnail from "../../../models/thumbnail";
import sharp from "sharp";
import { FileInterface } from "../../../models/file";
import { UserInterface } from "../../../models/user";
import uuid from "uuid";
import env from "../../../enviroment/env";
import s3 from "../../../db/s3";
const conn = mongoose.connection;
const createThumbnailS3 = (file: FileInterface, filename: string, user: UserInterface) => {
return new Promise<FileInterface>((resolve, reject) => {
const password = user.getEncryptionKey();
let CIPHER_KEY = crypto.createHash('sha256').update(password!).digest()
const thumbnailFilename = uuid.v4();
const params: any = {Bucket: env.s3Bucket, Key: file.metadata.s3ID!};
const readStream = s3.getObject(params).createReadStream();
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, file.metadata.IV);
readStream.on("error", (e: Error) => {
console.log("File service upload thumbnail error", e);
resolve(file);
})
decipher.on("error", (e: Error) => {
console.log("File service upload thumbnail decipher error", e);
resolve(file)
})
try {
const thumbnailIV = crypto.randomBytes(16);
const thumbnailCipher = crypto.createCipheriv("aes256", CIPHER_KEY, thumbnailIV);
const imageResize = sharp().resize(300).on("error", (e: Error) => {
console.log("resize error", e);
resolve(file);
})
const paramsWrite: any = {
Bucket: env.s3Bucket,
Body : readStream.pipe(decipher).pipe(imageResize).pipe(thumbnailCipher),
Key : thumbnailFilename
};
s3.upload(paramsWrite, async(err: any, data: any) => {
if (err) {
console.log("Amazon Upload Error", err);
reject("Amazon upload error");
}
const thumbnailModel = new Thumbnail({name: thumbnailFilename, owner: user._id, IV: thumbnailIV, s3ID: thumbnailFilename});
await thumbnailModel.save();
const getUpdatedFile = await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": file._id}, {"$set": {"metadata.hasThumbnail": true, "metadata.thumbnailID": thumbnailModel._id}})
let updatedFile: FileInterface = getUpdatedFile.value;
updatedFile = {...updatedFile, metadata: {...updatedFile.metadata, hasThumbnail: true, thumbnailID: thumbnailModel._id}} as FileInterface;
resolve(updatedFile);
})
} catch (e) {
console.log("Thumbnail error", e);
resolve(file);
}
})
}
export default createThumbnailS3;
backend/services/ChunkService/utils/fixEndChunkLength.ts
+6
View File
@@ -0,0 +1,6 @@
const fixEndChunkLength = (length: number) => {
return Math.floor((length-1) / 16) * 16 + 16;
}
export default fixEndChunkLength;
backend/services/ChunkService/utils/fixStartChunkLength.ts
+6
View File
@@ -0,0 +1,6 @@
const fixStartChunkLength = (length: number) => {
return Math.floor((length-1) / 16) * 16 - 16;
}
export default fixStartChunkLength;
backend/services/ChunkService/utils/getBusboyData.ts
+33
View File
@@ -0,0 +1,33 @@
import { Stream } from "stream";
const getBusboyData = (busboy: any) => {
type dataType = {
file: Stream,
filename: string,
formData: Map<any, any>
}
return new Promise<dataType>((resolve, reject) => {
const formData = new Map();
busboy.on("field", (field: any, val: any) => {
formData.set(field, val)
});
busboy.on("file", async(_: string, file: Stream, filename: string) => {
resolve({
file,
filename,
formData
})
})
})
}
export default getBusboyData;
backend/services/ChunkService/utils/getFileSize.ts
+19
View File
@@ -0,0 +1,19 @@
import fs from "fs";
const getFileSize = (path: string) => {
return new Promise((resolve, reject) => {
fs.stat(path, (error, stats) => {
if (error) {
resolve(0);
}
resolve(stats.size);
});
})
}
export default getFileSize;
backend/services/ChunkService/utils/getPrevIVFS.ts
+19
View File
@@ -0,0 +1,19 @@
import fs from "fs";
const getPrevIV = (start: number, path: string) => {
return new Promise<Buffer | string>((resolve, reject) => {
const stream = fs.createReadStream(path, {
start,
end: start + 15
})
stream.on("data", (data) => {
resolve(data);
})
})
}
export default getPrevIV;
backend/services/ChunkService/utils/getPrevIVMongo.ts
+23
View File
@@ -0,0 +1,23 @@
import mongoose from "../../../db/mongoose";
import { ObjectID } from "mongodb"
const conn = mongoose.connection;
const getPrevIV = (start: number, fileID: string) => {
return new Promise<Buffer | string>((resolve, reject) => {
const bucket = new mongoose.mongo.GridFSBucket(conn.db);
const stream = bucket.openDownloadStream(new ObjectID(fileID), {
start: start,
end: start + 16
});
stream.on("data", (data) => {
resolve(data);
})
})
}
export default getPrevIV;
backend/services/ChunkService/utils/getPrevIVS3.ts
+19
View File
@@ -0,0 +1,19 @@
import s3 from "../../../db/s3";
import env from "../../../enviroment/env";
const getPrevIV = (start: number, key: string) => {
return new Promise<Buffer | string>((resolve, reject) => {
const params: any = {Bucket: env.s3Bucket, Key: key, Range: `bytes=${start}-${start + 15}`};
const stream = s3.getObject(params).createReadStream();
stream.on("data", (data) => {
resolve(data);
})
})
}
export default getPrevIV;
backend/services/FileService/utils/removeChunks.js → backend/services/ChunkService/utils/removeChunks.ts
+7 -5
View File
@@ -1,11 +1,13 @@
const DbUtilsFile = require("../../../db/utils/fileUtils");
import { GridFSBucketWriteStream } from "mongodb";
import DbUtilsFile from "../../../db/utils/fileUtils";
const dbUtilsFile = new DbUtilsFile();
const removeChunks = async(bucketStream) => {
const removeChunks = async(bucketStream: GridFSBucketWriteStream) => {
const uploadID = bucketStream.id as string;
try {
const uploadID = bucketStream.id;
if (!uploadID || uploadID.length === 0) {
@@ -23,4 +25,4 @@ const removeChunks = async(bucketStream) => {
}
}
module.exports = removeChunks;
export default removeChunks;
backend/services/ChunkService/utils/removeChunksFS.ts
+20
View File
@@ -0,0 +1,20 @@
import fs from "fs";
const removeChunksFS = (path: string) => {
return new Promise((resolve, reject) => {
fs.unlink(path, (err) => {
if (err) {
console.log("Could not remove fs file", err);
resolve();
}
console.log("File Removed");
resolve();
})
})
}
export default removeChunksFS;
backend/services/ChunkService/utils/removeChunksS3.ts
+21
View File
@@ -0,0 +1,21 @@
import s3 from "../../../db/s3";
const removeChunksS3 = (parmas: any) => {
return new Promise((resolve, reject) => {
s3.deleteObject(parmas, (err, data) => {
if (err) {
console.log("Could not remove S3 file");
reject("Could Not Remove S3 File");
}
console.log("Delete S3 file");
resolve();
})
})
}
export default removeChunksS3;
backend/services/ChunkService/utils/removeTempToken.ts
+13
View File
@@ -0,0 +1,13 @@
import { UserInterface } from "../../../models/user";
const removeTempToken = async(user: UserInterface, tempToken: any,) => {
user.tempTokens = user.tempTokens.filter((filterToken) => {
return filterToken.token !== tempToken;
})
await user.save();
}
export default removeTempToken;
backend/services/FileService/index.js
-899
View File
@@ -1,899 +0,0 @@
const imageChecker = require("../../../src/utils/imageChecker");
const crypto= require("crypto");
const videoChecker = require("../../../src/utils/videoChecker");
const mongoose = require("../../db/mongoose")
const conn = mongoose.connection;
const createThumbnail = require("../../services/FileService/utils/createThumbnail");
const Thumbnail = require("../../models/thumbnail");
const ObjectID = require('mongodb').ObjectID
const NotAuthorizedError = require("../../../src/utils/NotAuthorizedError");
const NotFoundError = require("../../../src/utils/NotFoundError");
const env = require("../../enviroment/env");
const jwt = require("jsonwebtoken");
const removeChunks = require("./utils/removeChunks");
const User = require("../../models/user");
const Folder = require("../../models/folder");
const sortBySwitch = require("../../../src/utils/sortBySwitch")
const createQuery = require("../../../src/utils/createQuery");
const ffmpeg = require("fluent-ffmpeg");
const temp = require("temp").track();
const progress = require("progress-stream");
const fs = require("fs")
const DbUtilFile = require("../../db/utils/fileUtils");
const DbUtilFolder = require("../../db/utils/folderUtils");
const dbUtilsFile = new DbUtilFile();
const dbUtilsFolder = new DbUtilFolder();
const FileService = function() {
this.upload = (user, busboy, req) => {
return new Promise((resolve, reject) => {
const password = user.getEncryptionKey();
let bucketStream;
const initVect = crypto.randomBytes(16);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const cipher = crypto.createCipheriv('aes256', CIPHER_KEY, initVect);
cipher.on("error", async(e) => {
await removeChunks(bucketStream);
reject({
message: "File service upload cipher error",
exception: e,
code: 500
})
})
const formData = new Map();
busboy.on("error", async(e) => {
await removeChunks(bucketStream);
reject({
message: "File service upload busboy error",
exception: e,
code: 500
})
})
busboy.on("file", async(fieldname, file, filename) => {
const parent = formData.get("parent") || "/"
const parentList = formData.get("parentList") || "/";
const size = formData.get("size") || ""
let hasThumbnail = false;
let thumbnailID = ""
const isVideo = videoChecker(filename)
const metadata = {
owner: user._id,
parent,
parentList,
hasThumbnail,
thumbnailID,
isVideo,
size,
IV: initVect
}
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255
});
bucketStream = bucket.openUploadStream(filename, {metadata})
bucketStream.on("error", async(e) => {
await removeChunks(bucketStream);
reject({
message: "Cannot upload file to database",
exception: e,
code: 500
})
})
req.on("aborted", async() => {
console.log("Upload Request Cancelling...");
await removeChunks(bucketStream);
})
file.pipe(cipher).pipe(bucketStream);
bucketStream.on("finish", async(file) => {
const imageCheck = imageChecker(filename);
if (file.length < 15728640 && imageCheck) {
let updatedFile = await createThumbnail(file, filename, user);
resolve(updatedFile);
} else {
resolve(file);
}
})
}).on("field", (field, val) => {
formData.set(field, val)
})
})
}
this.getThumbnail = async(user, id) => {
const password = user.getEncryptionKey();
const thumbnail = await Thumbnail.findById(id);
if (user._id.toString() !== thumbnail.owner) {
throw new NotAuthorizedError("Thumbnail Unauthorized Error");
}
const iv = thumbnail.data.slice(0, 16);
const chunk = thumbnail.data.slice(16);
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv);
const decryptedThumbnail = Buffer.concat([decipher.update(chunk), decipher.final()]);
//console.log("decry", decryptedThumbnail);
return decryptedThumbnail;
}
this.getFullThumbnail = async(user, fileID, res) => {
return new Promise((resolve, reject) => {
const userID = user._id;
dbUtilsFile.getFileInfo(fileID, userID).then( async(file) => {
if (!file) {
reject({
code: 401,
message: "File For Full Thumbnail Not Found",
exception: undefined
})
}
const bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255,
})
const password = user.getEncryptionKey();
const IV = file.metadata.IV.buffer
const readStream = bucket.openDownloadStream(ObjectID(fileID))
readStream.on("error", (e) => {
reject({
code: 500,
message: "File service Full Thumbnail stream error",
exception: e
})
})
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
decipher.on("error", (e) => {
reject({
code: 500,
message: "File service Full Thumbnail decipher error",
exception: e
})
})
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size);
readStream.pipe(decipher).pipe(res).on("finish", async() => {
console.log("Sent Full Thumbnail");
resolve();
});
});
})
}
this.publicDownload = (ID, tempToken, res) => {
return new Promise((resolve, reject) => {
dbUtilsFile.getPublicFile(ID).then(async(file) => {
if (!file || !file.metadata.link || file.metadata.link !== tempToken) {
reject({
code: 401,
message: "File not public/Not found",
exception: undefined
})
} else {
const user = await User.findById(file.metadata.owner);
const bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255,
})
const password = user.getEncryptionKey();
const IV = file.metadata.IV.buffer
const readStream = bucket.openDownloadStream(ObjectID(ID))
readStream.on("error", (e) => {
reject({
code: 500,
message: "File service public download decipher error",
exception: e
})
})
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
decipher.on("error", (e) => {
reject({
code: 500,
message: "File service public download decipher error",
exception: e
})
})
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size);
readStream.pipe(decipher).pipe(res).on("finish", async() => {
console.log("removing public link");
await this.removePublicOneTimeLink(file);
resolve();
});
}
})
})
}
this.removePublicOneTimeLink = async(currentFile) => {
const ID = currentFile._id;
if (currentFile.metadata.linkType === "one") {
await dbUtilsFile.removeOneTimePublicLink(ID);
}
}
this.removeLink = async(userID, fileID) => {
const file = await dbUtilsFile.removeLink(fileID, userID);
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Remove Link File Not Found Error")
}
this.makePublic = async(user, fileID) => {
const userID = user._id;
const token = await jwt.sign({_id: userID.toString()}, env.password);
const file = await dbUtilsFile.makePublic(fileID, userID, token, "public");
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Make Public File Not Found Error");
return token;
}
this.getPublicInfo = async(fileID, tempToken) => {
const file = await dbUtilsFile.getPublicInfo(fileID, tempToken);
if (!file || !file.metadata.link || file.metadata.link !== tempToken) {
throw new NotFoundError("Public Info Not Found");
} else {
return file;
}
}
this.makeOneTimePublic = async(userID, fileID) => {
const token = await jwt.sign({_id: userID.toString()}, env.password);
const file = await dbUtilsFile.makeOneTimePublic(fileID, userID, token);
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Make One Time Public Not Found Error");
return token;
}
this.getFileInfo = async(userID, fileID) => {
let currentFile = await dbUtilsFile.getFileInfo(fileID, userID)
if (!currentFile) throw new NotFoundError("Get File Info Not Found Error");
const parentID = currentFile.metadata.parent
let parentName = "";
if (parentID === "/") {
parentName = "Home"
} else {
const parentFolder = await Folder.findOne({"owner": userID, "_id": parentID});
if (parentFolder) {
parentName = parentFolder.name;
} else {
parentName = "Unknown"
}
}
return {...currentFile, parentName}
}
this.getQuickList = async(userID) => {
const quickList = await dbUtilsFile.getQuickList(userID);
if (!quickList) throw new NotFoundError("Quick List Not Found Error");
return quickList;
}
this.getList = async(userID, query) => {
let searchQuery = query.search || "";
const parent = query.parent || "/";
let limit = query.limit || 50;
let sortBy = query.sortby || "DEFAULT"
const startAt = query.startAt || undefined
const startAtDate = query.startAtDate || "0"
const startAtName = query.startAtName || ""
sortBy = sortBySwitch(sortBy)
limit = parseInt(limit)
const queryObj = createQuery(userID, parent, query.sortby,startAt, startAtDate, searchQuery, startAtName)
const fileList = await dbUtilsFile.getList(queryObj, sortBy, limit);
if (!fileList) throw new NotFoundError("File List Not Found");
return fileList;
}
this.getDownloadToken = async(user) => {
const tempToken = await user.generateTempAuthToken();
if (!tempToken) throw new NotAuthorizedError("Get Download Token Not Authorized Error");
return tempToken;
}
this.getDownloadTokenVideo = async(user, cookie) => {
if (!cookie) throw new NotAuthorizedError("Get Download Token Video Cookie Not Authorized Error");
const tempToken = await user.generateTempAuthTokenVideo(cookie);
if (!tempToken) throw new NotAuthorizedError("Get Download Token Video Not Authorized Error");
return tempToken;
}
this.removeTempToken = async(user, tempToken) => {
const key = user.getEncryptionKey();
const decoded = await jwt.verify(tempToken, env.password);
const publicKey = decoded.iv;
const encryptedToken = user.encryptToken(tempToken, key, publicKey);
const removedTokenUser = await dbUtilsFile.removeTempToken(user, encryptedToken);
if (!removedTokenUser) throw new NotFoundError("Remove Temp Token User Not Found Errors");
await removedTokenUser.save();
}
this.transcodeVideo = (user, body) => {
return new Promise((resolve, reject) => {
const id = body.file._id;
const userID = user._id;
conn.db.collection("fs.files")
.findOne({"_id": ObjectID(id), "metadata.owner": userID}).then((currentFile) => {
if (!currentFile) {
reject({
message: "Transcode Video Not Found Error",
exception: undefined,
code: 401
})
} else {
const password = user.getEncryptionKey(); //env.key;
const IV = currentFile.metadata.IV.buffer
const bucket = new mongoose.mongo.GridFSBucket(conn.db);
const readStream = bucket.openDownloadStream(ObjectID(id));
readStream.on("error", (e) => {
reject({
message: "File service transcode video stream error",
exception: e,
code: 500
})
})
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
decipher.on("error", (e) => {
reject({
message: "File service transcode video decipher error",
exception: e,
code: 500
})
})
const tempStream = temp.createWriteStream()
tempStream.on("error", (e) => {
reject({
message: "File service transcode video temp error",
exception: e,
code: 500
})
})
readStream.pipe(decipher).pipe(tempStream).on("finish", () => {
const writeBucket = new mongoose.mongo.GridFSBucket(conn.db, {
bucketName: "videos",
});
const gfsWrite = writeBucket.openUploadStream(currentFile.filename, {
_id: id,
})
gfsWrite.on("error", (e) => {
reject({
message: "File service transcode video write stream error",
exception: e,
code: 500
})
})
const initVect = crypto.randomBytes(16);
const cipher = crypto.createCipheriv('aes256', CIPHER_KEY, initVect);
cipher.on("error", (e) => {
reject({
message: "File service transcode video cipher write error",
exception: e,
code: 500
})
})
const str = progress({time: 100})
str.on("progress", () => {
});
str.on("finish", () => {
})
const tempStream2 = temp.createWriteStream()
tempStream2.on("error", (e) => {
reject({
message: "File service transcode video temp write error",
exception: e,
code: 500
})
})
const ffmpegCommand = ffmpeg()
.input(tempStream.path).outputOptions(['-movflags frag_keyframe+empty_moov', '-movflags +faststart'])
.outputFormat('mp4').on("error", (err, stdout, stderr) => {
console.log('An error occurred: ' + err.message, err, stderr);
reject({
message: "Video Transcode Failed Error",
exception: err.message,
code: 500
})
})
.videoCodec('libx264')
.audioBitrate('128k')
.videoBitrate(8000, true)
.pipe(tempStream2).on("finish", () => {
fs.createReadStream(tempStream2.path).pipe(cipher).pipe(gfsWrite).on("finish", async(file) => {
console.log("transcode finished");
await conn.db.collection("fs.files")
.findOneAndUpdate({"_id": ObjectID(id)}, {"$set": {"metadata.transcoded": true, "metadata.transcodedIV": initVect, "metadata.transcoded_size": tempStream2.bytesWritten, "metadata.transcodedID": file._id}})
resolve();
}).on("error", (e) => {
reject({
message: "File service transcode read stream error",
exception: e,
code: 500
})
})
})
ffmpegCommand.on("error", (e) => {
reject({
message: "File service FFMPEG error",
exception: e,
code: 500
})
})
})
}
})
})
}
this.removeTranscodeVideo = async(userID, fileID) => {
const parentFile = await dbUtilsFile.getFileInfo(fileID, userID);
const transcodedVideoID = parentFile.metadata.transcodedID;
const bucket = new mongoose.mongo.GridFSBucket(conn.db, {
bucketName: "videos"
});
try {
await bucket.delete(ObjectID(transcodedVideoID));
} catch (e) {
console.log("Could Not Find Transcoded Video");
}
const file = await dbUtilsFile.removeTranscodeVideo(fileID, userID);
if (!file) throw new NotFoundError("Remove Transcode Video Error");
}
this.streamTranscodedVideo = (userID, fileID, headers, res) => {
return new Promise((resolve, reject) => {
dbUtilsFile.getFileInfo(fileID, userID).then((file) => {
if (!file) {
reject({
code: 401,
message: "Stream Transcoded File Not Found",
exception: undefined
})
} else {
const fileSize = file.metadata.transcoded_size;
const transcodedVideoID = file.metadata.transcodedID;
const password = user.getEncryptionKey(); //env.key;
const range = headers.range
const parts = range.replace(/bytes=/, "").split("-")
const start = parseInt(parts[0], 10)
const end = parts[1]
? parseInt(parts[1], 10)
: fileSize-1
const chunksize = (end-start)+1
const IV = file.metadata.transcodedIV.buffer
let head = {
'Content-Range': 'bytes ' + start + '-' + end + '/' + fileSize,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/mp4'
}
const bucket = new mongoose.mongo.GridFSBucket(conn.db, {
bucketName: "videos",
chunkSizeBytes: chunksize
});
const readStream = bucket.openDownloadStream(ObjectID(transcodedVideoID), {
start: start,
end: end
});
readStream.on("error", (e) => {
reject({
code: 500,
message: "File service transcode video stream error",
exception: e
})
})
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
decipher.setAutoPadding(false);
decipher.on("error", (e) => {
reject({
code: 500,
message: "File service transcode video decipher error",
exception: e
})
});
res.writeHead(206, head);
readStream.pipe(decipher).pipe(res).on("finish", () => {
resolve();
});
}
})
})
}
this.streamVideo = (user, fileID, headers, res) => {
return new Promise((resolve, reject) => {
const userID = user._id;
dbUtilsFile.getFileInfo(fileID, userID).then((currentFile) => {
if (!currentFile) {
reject({
code: 401,
message: "Video Steam Not Found Error",
exception: undefined
})
} else {
const password = user.getEncryptionKey();
const fileSize = currentFile.metadata.size;
const range = headers.range
const parts = range.replace(/bytes=/, "").split("-")
let start = parseInt(parts[0], 10)
let end = parts[1]
? parseInt(parts[1], 10)
: fileSize-1
const chunksize = (end-start)+1
const IV = currentFile.metadata.IV.buffer
let head = {
'Content-Range': 'bytes ' + start + '-' + end + '/' + fileSize,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': 'video/mp4'}
const bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024
});
const readStream = bucket.openDownloadStream(ObjectID(fileID), {
start: start,
end: end.length
});
readStream.on("error", (e) => {
reject({
code: 500,
message: "File service stream video stream error",
exception: e
})
})
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
decipher.on("error", (e) => {
reject({
code: 500,
message: "File service stream video decipher error",
exception: e
})
})
res.writeHead(206, head);
readStream.pipe(decipher).pipe(res).on("finish", () => {
resolve();
});
}
})
})
}
this.downloadFile = (user, fileID, res) => {
return new Promise((resolve, reject) => {
const userID = user._id;
dbUtilsFile.getFileInfo(fileID, userID).then((file) => {
if (!file) {
reject({
code: 401,
message: "Download File Not Found Error",
exception: undefined
})
} else {
const password = user.getEncryptionKey();
const bucket = new mongoose.mongo.GridFSBucket(conn.db);
const IV = file.metadata.IV.buffer
const readStream = bucket.openDownloadStream(ObjectID(fileID))
readStream.on("error", (e) => {
reject({
code: 500,
message: "File service download decipher error",
exception: e
})
})
const CIPHER_KEY = crypto.createHash('sha256').update(password).digest()
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, IV);
decipher.on("error", (e) => {
reject({
code: 500,
message: "File service download decipher error",
exception: e
})
})
res.set('Content-Type', 'binary/octet-stream');
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
res.set('Content-Length', file.metadata.size);
readStream.pipe(decipher).pipe(res).on("finish", () => {
resolve();
});
}
})
})
}
this.getSuggestedList = async(userID, searchQuery) => {
searchQuery = new RegExp(searchQuery, 'i')
const fileList = await dbUtilsFile.getFileSearchList(userID, searchQuery);
const folderList = await dbUtilsFolder.getFolderSearchList(userID, searchQuery);
if (!fileList || !folderList) throw new NotFoundError("Suggested List Not Found Error");
return {
fileList,
folderList
}
}
this.renameFile = async(userID, fileID, title) => {
const file = await dbUtilsFile.renameFile(fileID, userID, title);
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Rename File Not Found Error");
return file;
}
this.moveFile = async(userID, fileID, parentID) => {
let parentList = ["/"];
if (parentID.length !== 1) {
const parentFile = await dbUtilsFolder.getFolderInfo(parentID, userID);
if (!parentFile) throw new NotFoundError("Rename Parent File Not Found Error")
const parentList = parentFile.parentList;
parentList.push(parentID);
}
const file = await dbUtilsFile.moveFile(fileID, userID, parentID, parentList.toString());
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Rename File Not Found Error");
return file;
}
this.deleteFile = async(userID, fileID) => {
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255,
});
const videoBucket = new mongoose.mongo.GridFSBucket(conn.db, {
bucketName: "videos"
});
const file = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("Delete File Not Found Error");
if (file.metadata.thumbnailID) {
await Thumbnail.deleteOne({_id: file.metadata.thumbnailID});
}
if (file.metadata.isVideo && file.metadata.transcoded) {
try {
await bucket.delete(ObjectID(file.metadata.transcodedID));
} catch (e) {
console.log("Could Not Find Transcoded Video");
}
}
await bucket.delete(ObjectID(fileID));
}
}
module.exports = FileService;
backend/services/FileService/index.ts
+223
View File
@@ -0,0 +1,223 @@
import NotAuthorizedError from "../../utils/NotAuthorizedError";
import NotFoundError from "../../utils/NotFoundError";
import env from "../../enviroment/env";
import jwt from "jsonwebtoken";
import Folder from "../../models/folder";
import sortBySwitch from "../../utils/sortBySwitch";
import createQuery from "../../utils/createQuery";
import DbUtilFile from "../../db/utils/fileUtils/index";
import DbUtilFolder from "../../db/utils/folderUtils";
import { UserInterface } from "../../models/user";
import { FileInterface } from "../../models/file";
import tempStorage from "../../tempStorage/tempStorage";
import uuid from "uuid";
const dbUtilsFile = new DbUtilFile();
const dbUtilsFolder = new DbUtilFolder();
class MongoFileService {
constructor() {
}
removePublicOneTimeLink = async(currentFile: FileInterface) => {
const fileID = currentFile._id;
if (currentFile.metadata.linkType === "one") {
await dbUtilsFile.removeOneTimePublicLink(fileID);
}
}
removeLink = async(userID: string, fileID: string) => {
const file = await dbUtilsFile.removeLink(fileID, userID);
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Remove Link File Not Found Error")
}
makePublic = async(user: UserInterface, fileID: string) => {
const userID = user._id;
const token = await jwt.sign({_id: userID.toString()}, env.password!);
const file = await dbUtilsFile.makePublic(fileID, userID, token);
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Make Public File Not Found Error");
return token;
}
getPublicInfo = async(fileID: string, tempToken: string) => {
const file: FileInterface = await dbUtilsFile.getPublicInfo(fileID, tempToken);
if (!file || !file.metadata.link || file.metadata.link !== tempToken) {
throw new NotFoundError("Public Info Not Found");
} else {
return file;
}
}
makeOneTimePublic = async(userID: string, fileID: string) => {
const token = await jwt.sign({_id: userID.toString()}, env.password!);
const file = await dbUtilsFile.makeOneTimePublic(fileID, userID, token);
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Make One Time Public Not Found Error");
return token;
}
getFileInfo = async(userID: string, fileID: string) => {
let currentFile = await dbUtilsFile.getFileInfo(fileID, userID)
if (!currentFile) throw new NotFoundError("Get File Info Not Found Error");
const parentID = currentFile.metadata.parent
let parentName = "";
if (parentID === "/") {
parentName = "Home"
} else {
const parentFolder = await Folder.findOne({"owner": userID, "_id": parentID});
if (parentFolder) {
parentName = parentFolder.name;
} else {
parentName = "Unknown"
}
}
return {...currentFile, parentName}
}
getQuickList = async(userID: string) => {
const quickList = await dbUtilsFile.getQuickList(userID);
if (!quickList) throw new NotFoundError("Quick List Not Found Error");
return quickList;
}
getList = async(userID: string, query: any) => {
let searchQuery = query.search || "";
const parent = query.parent || "/";
let limit = query.limit || 50;
let sortBy = query.sortby || "DEFAULT"
const startAt = query.startAt || undefined
const startAtDate = query.startAtDate || "0"
const startAtName = query.startAtName || ""
sortBy = sortBySwitch(sortBy)
limit = parseInt(limit)
const queryObj = createQuery(userID, parent, query.sortby,startAt, startAtDate, searchQuery, startAtName)
const fileList = await dbUtilsFile.getList(queryObj, sortBy, limit);
if (!fileList) throw new NotFoundError("File List Not Found");
return fileList;
}
getDownloadToken = async(user: UserInterface) => {
const tempToken = await user.generateTempAuthToken();
if (!tempToken) throw new NotAuthorizedError("Get Download Token Not Authorized Error");
return tempToken;
}
getDownloadTokenVideo = async(user: UserInterface, cookie: string) => {
if (!cookie) throw new NotAuthorizedError("Get Download Token Video Cookie Not Authorized Error");
const tempToken = await user.generateTempAuthTokenVideo(cookie);
if (!tempToken) throw new NotAuthorizedError("Get Download Token Video Not Authorized Error");
return tempToken;
}
removeTempToken = async(user: UserInterface, tempToken: any, currentUUID: string) => {
const key = user.getEncryptionKey();
const decoded = await jwt.verify(tempToken, env.password!) as any;
const publicKey = decoded.iv;
const encryptedToken = user.encryptToken(tempToken, key, publicKey);
const removedTokenUser = await dbUtilsFile.removeTempToken(user, encryptedToken);
if (!removedTokenUser) throw new NotFoundError("Remove Temp Token User Not Found Errors");
delete tempStorage[currentUUID];
await removedTokenUser.save();
}
getSuggestedList = async(userID: string, searchQuery: any) => {
searchQuery = new RegExp(searchQuery, 'i')
const fileList = await dbUtilsFile.getFileSearchList(userID, searchQuery);
const folderList = await dbUtilsFolder.getFolderSearchList(userID, searchQuery);
if (!fileList || !folderList) throw new NotFoundError("Suggested List Not Found Error");
return {
fileList,
folderList
}
}
renameFile = async(userID: string, fileID: string, title: string) => {
const file = await dbUtilsFile.renameFile(fileID, userID, title);
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Rename File Not Found Error");
return file;
}
moveFile = async(userID: string, fileID: string, parentID: string) => {
let parentList = ["/"];
if (parentID.length !== 1) {
const parentFile = await dbUtilsFolder.getFolderInfo(parentID, userID);
if (!parentFile) throw new NotFoundError("Rename Parent File Not Found Error")
const parentList = parentFile.parentList;
parentList.push(parentID);
}
const file = await dbUtilsFile.moveFile(fileID, userID, parentID, parentList.toString());
if (!file.lastErrorObject.updatedExisting) throw new NotFoundError("Rename File Not Found Error");
return file;
}
}
export default MongoFileService;
backend/services/FolderService/index.js → backend/services/FolderService/index.ts
+16 -118
View File
@@ -1,20 +1,16 @@
const Folder = require("../../models/folder");
const InternalServerError = require("../../../src/utils/InternalServerError");
const NotFoundError = require("../../../src/utils/NotFoundError");
const UtilsFile = require("../../db/utils/fileUtils");
const ObjectID = require('mongodb').ObjectID
const UtilsFolder = require("../../db/utils/folderUtils");
const mongoose = require("../../db/mongoose")
const conn = mongoose.connection;
const Thumbnail = require("../../models/thumbnail");
const sortBySwitch = require("../../../src/utils/sortBySwitchFolder")
import Folder from "../../models/folder";
import InternalServerError from "../../utils/InternalServerError";
import NotFoundError from "../../utils/NotFoundError";
import UtilsFile from "../../db/utils/fileUtils";
import UtilsFolder from "../../db/utils/folderUtils";
import sortBySwitch from "../../utils/sortBySwitchFolder";
const utilsFile = new UtilsFile();
const utilsFolder = new UtilsFolder();
const FolderService = function() {
class FolderService {
this.uploadFolder = async(data) => {
uploadFolder = async(data: any) => {
const folder = new Folder(data);
@@ -24,105 +20,8 @@ const FolderService = function() {
return folder;
}
this.deleteFolder = async(userID, folderID, parentList) => {
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255
});
const videoBucket = new mongoose.mongo.GridFSBucket(conn.db, {
bucketName: "videos"
});
const parentListString = parentList.toString()
await Folder.deleteMany({"owner": userID, "parentList": { $all: parentList}})
await Folder.deleteMany({"owner": userID, "_id": folderID});
const fileList = await utilsFile.getFileListByParent(userID, parentListString);
if (!fileList) throw new NotFoundError("Delete File List Not Found");
for (let i = 0; i < fileList.length; i++) {
const currentFile = fileList[i];
try {
if (currentFile.metadata.thumbnailID) {
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID});
}
if (currentFile.metadata.isVideo && currentFile.metadata.transcoded) {
try {
await videoBucket.delete(ObjectID(currentFile.metadata.transcodedID))
} catch (e) {console.log("Could Not Find Transcoded Video")}
}
await bucket.delete(ObjectID(currentFile._id));
} catch (e) {
console.log("Could not delete file", currentFile.filename, currentFile._id);
}
}
}
this.deleteAll = async(userID) => {
console.log("remove all request")
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255
});
const videoBucket = new mongoose.mongo.GridFSBucket(conn.db, {
bucketName: "videos"
});
await Folder.deleteMany({"owner": userID});
const fileList = await utilsFile.getFileListByOwner(userID);
if (!fileList) throw new NotFoundError("Delete All File List Not Found Error");
for (let i = 0; i < fileList.length; i++) {
const currentFile = fileList[i];
try {
if (currentFile.metadata.thumbnailID) {
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID})
}
if (currentFile.metadata.isVideo && currentFile.metadata.transcoded) {
try {
await videoBucket.delete(ObjectID(currentFile.metadata.transcodedID))
} catch (e) {
console.log("Cannot Find Transcoded Video");
}
}
await bucket.delete(ObjectID(currentFile._id));
} catch (e) {
console.log("Could Not Remove File", currentFile.filename, currentFile._id);
}
}
}
this.getFolderInfo = async(userID, folderID) => {
getFolderInfo = async(userID: string, folderID: string) => {
let currentFolder = await utilsFolder.getFolderInfo(folderID, userID);
@@ -160,8 +59,8 @@ const FolderService = function() {
return currentFolder;
}
this.getFolderSublist = async(userID, folderID) => {
getFolderSublist = async(userID: string, folderID: string) => {
const folder = await utilsFolder.getFolderInfo(folderID, userID);
if (!folder) throw new NotFoundError("Folder Sublist Not Found Error");
@@ -196,10 +95,9 @@ const FolderService = function() {
folderIDList,
folderNameList
}
}
this.getFolderList = async(userID, query) => {
getFolderList = async(userID: string, query: any) => {
let searchQuery = query.search || "";
const parent = query.parent || "/";
@@ -225,14 +123,14 @@ const FolderService = function() {
}
}
this.renameFolder = async(userID, folderID, title) => {
renameFolder = async(userID: string, folderID: string, title: string) => {
const folder = await utilsFolder.renameFolder(folderID, userID, title);
if (!folder) throw new NotFoundError("Rename Folder Not Found");
}
this.moveFolder = async(userID, folderID, parentID) => {
moveFolder = async(userID: string, folderID: string, parentID: string) => {
let parentList = ["/"];
@@ -268,7 +166,7 @@ const FolderService = function() {
fileChildren.map( async(currentFileChild) => {
let currentFileChildParentList = currentFileChild.metadata.parentList;
let currentFileChildParentList: string | string[] = currentFileChild.metadata.parentList;
currentFileChildParentList = currentFileChildParentList.split(",");
@@ -284,4 +182,4 @@ const FolderService = function() {
}
}
module.exports = FolderService;
export default FolderService;
backend/services/UserService/index.js → backend/services/UserService/index.ts
+32 -16
View File
@@ -1,18 +1,33 @@
const User = require("../../models/user");
const bcrypt = require("bcrypt");
const crypto = require("crypto");
const NotFoundError = require("../../../src/utils/NotFoundError");
const InternalServerError = require("../../../src/utils/InternalServerError");
const NotAuthorizedError = require("../../../src/utils/NotAuthorizedError");
import User, {UserInterface} from "../../models/user";
import bcrypt from "bcrypt";
import NotFoundError from "../../utils/NotFoundError";
import InternalServerError from "../../utils/InternalServerError";
import NotAuthorizedError from "../../utils/NotAuthorizedError";
const UserService = function() {
type UserDataType = {
email: string,
password: string,
}
this.login = async(userData) => {
const uknownUserType = User as unknown;
const UserStaticType = uknownUserType as {
findByCreds: (email: string, password: string) => Promise<UserInterface>;
};
class UserService {
constructor() {
}
login = async(userData: UserDataType) => {
const email = userData.email;
const password = userData.password;
const user = await User.findByCreds(email, password);
const user = await UserStaticType.findByCreds(email, password);
const token = await user.generateAuthToken();
@@ -21,7 +36,7 @@ const UserService = function() {
return {user, token}
}
this.logout = async(user, userToken) => {
logout = async(user: UserInterface, userToken: any) => {
user.tokens = user.tokens.filter((token) => {
return token.token !== userToken;
@@ -30,15 +45,15 @@ const UserService = function() {
await user.save();
}
this.logoutAll = async(user) => {
logoutAll = async(user: UserInterface) => {
user.tokens = []
user.tempTokens = [];
await user.save();
}
this.create = async(userData) => {
create = async(userData: any) => {
console.log("Create");
@@ -54,7 +69,7 @@ const UserService = function() {
return {user, token}
}
this.changePassword = async(user, oldPassword, newPassword) => {
changePassword = async(user: UserInterface, oldPassword: string, newPassword: string) => {
const isMatch = await bcrypt.compare(oldPassword, user.password);
@@ -68,12 +83,13 @@ const UserService = function() {
user.tempTokens = [];
await user.save();
await user.changeEncryptionKey(encryptionKey);
await user.changeEncryptionKey(encryptionKey!);
const newToken = await user.generateAuthToken();
return newToken;
}
}
module.exports = UserService;
export default UserService;
backend/tempStorage/tempStorage.ts
+3
View File
@@ -0,0 +1,3 @@
const tempStorage: any = {};
export default tempStorage;
backend/utils/InternalServerError.ts
+12
View File
@@ -0,0 +1,12 @@
class InternalServerError extends Error {
code: number;
constructor(args: any) {
super(args);
this.code = 500;
}
}
export default InternalServerError;
backend/utils/NotAuthorizedError.ts
+12
View File
@@ -0,0 +1,12 @@
class NotAuthorizedError extends Error {
code: number;
constructor(args: any) {
super(args);
this.code = 401;
}
}
export default NotAuthorizedError;
backend/utils/NotFoundError.ts
+12
View File
@@ -0,0 +1,12 @@
class NotFoundError extends Error {
code: number;
constructor(args: any) {
super(args);
this.code = 404;
}
}
export default NotFoundError;
backend/utils/createQuery.ts
+54
View File
@@ -0,0 +1,54 @@
export interface QueryInterface {
"metadata.owner": string,
"metadata.parent"?: string,
filename?: string | RegExp | {
$lt?: string,
$gt?: string,
},
uploadDate?: {
$lt?: Date,
$gt?: Date,
},
}
const createQuery = (owner: string, parent: string, sortBy: string, startAt: number, startAtDate: number, searchQuery: string | RegExp, startAtName: string) => {
let query: QueryInterface = {"metadata.owner": owner}
if (searchQuery !== "") {
searchQuery = new RegExp(searchQuery, 'i')
query = {...query, filename: searchQuery}
} else {
query = {...query, "metadata.parent": parent}
}
if (startAt) {
if (sortBy === "date_desc" || sortBy === "DEFAULT") {
query = {...query, "uploadDate": {$lt: new Date(startAtDate)}}
} else if (sortBy === "date_asc") {
query = {...query, "uploadDate": {$gt: new Date(startAtDate)}}
} else if (sortBy === "alp_desc") {
query = {...query, "filename": {$lt: startAtName}}
} else {
query = {...query, "filename": {$gt: startAtName}}
}
}
return query;
}
export default createQuery;
backend/utils/imageChecker.ts
+31
View File
@@ -0,0 +1,31 @@
const imageExtList = [
"jpeg",
"jpg",
"png",
"gif",
"svg",
"tiff",
"bmp"
]
const imageChecker = (filename: string) => {
if (filename.length < 1 || !filename.includes(".")) {
return false;
}
const extSplit = filename.split(".");
if (extSplit.length <= 1) {
return false;
}
const ext = extSplit[extSplit.length - 1];
return imageExtList.includes(ext.toLowerCase());
}
export default imageChecker;
backend/utils/mobileCheck.ts
+8
View File
@@ -0,0 +1,8 @@
declare let window:any
const mobilecheck = () => {
var check = false;
(function(a){if(/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(a)||/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test(a.substr(0,4))) check = true;})(navigator.userAgent||navigator.vendor||window.opera);
return check;
};
export default mobilecheck;
backend/utils/sortBySwitch.ts
+15
View File
@@ -0,0 +1,15 @@
const sortBySwitch = (sortBy: string) => {
switch(sortBy) {
case "alp_asc":
return {filename: 1}
case "alp_desc":
return {filename: -1}
case "date_asc":
return {uploadDate: 1}
default:
return {uploadDate: -1}
}
}
export default sortBySwitch;
backend/utils/sortBySwitchFolder.ts
+15
View File
@@ -0,0 +1,15 @@
const sortBySwitchFolder = (sortBy: string) => {
switch(sortBy) {
case "alp_asc":
return {name: 1}
case "alp_desc":
return {name: -1}
case "date_asc":
return {createdAt: 1}
default:
return {createdAt: -1}
}
}
export default sortBySwitchFolder;
backend/utils/streamToBuffer.ts
+27
View File
@@ -0,0 +1,27 @@
const streamToBuffer = (stream: any, allStreamsToErrorCatch: any[]) => {
const chunks: any[] = []
return new Promise<Buffer>((resolve, reject) => {
allStreamsToErrorCatch.forEach((currentStream) => {
currentStream.on("error", (e: Error) => {
console.log("Stream To Buffer Error", e);
reject({
message: "stream to buffer error",
code: 500,
error: e
})
})
})
stream.on('data', (chunk: any) => chunks.push(chunk));
stream.on('error', reject);
stream.on('end', () => resolve(Buffer.concat(chunks)));
})
}
export default streamToBuffer;
backend/utils/videoChecker.ts
+57
View File
@@ -0,0 +1,57 @@
const videoExtList = [
"3g2",
"3gp",
"aaf",
"asf",
"avchd",
"avi",
"drc",
"flv",
"m2v",
"m4p",
"m4v",
"mkv",
"mng",
"mov",
"mp2",
"mp4",
"mpe",
"mpeg",
"mpg",
"mpv",
"mxf",
"nsv",
"ogg",
"ogv",
"qt",
"rm",
"rmvb",
"roq",
"svi",
"vob",
"webm",
"wmv",
"yuv"
]
const videoChecker = (filename: string) => {
if (filename.length < 1 || !filename.includes(".")) {
return false;
}
const extSplit = filename.split(".");
if (extSplit.length <= 1) {
return false;
}
const ext = extSplit[extSplit.length - 1];
return videoExtList.includes(ext.toLowerCase());
}
export default videoChecker;
docker-compose-no-mongo.yml
+13
View File
@@ -0,0 +1,13 @@
version: '3'
services:
app:
container_name: mydrive-node-no-mongo
restart: always
build: .
ports:
- '3000:3000'
- '8080:8080'
env_file:
- docker-variables.env
docker-compose.yml
+24
View File
@@ -0,0 +1,24 @@
version: '3'
services:
app:
container_name: mydrive-node
restart: always
build: .
ports:
- '3000:3000'
- '8080:8080'
env_file:
- docker-variables.env
links:
- mongo
mongo:
container_name: mongo
image: mongo
ports:
- '27017:27017'
volumes:
- 'mongodb_data_volume:/data/db'
volumes:
mongodb_data_volume:
external: true
github_images/folder-screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 167 KiB

github_images/image-screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 362 KiB

github_images/mobile-screenshot.jpeg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 200 KiB

github_images/move-screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 235 KiB

github_images/mydrivehome.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 394 KiB

github_images/mydrivehome2.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 278 KiB

github_images/mydrivelogogit.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

github_images/search-screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 292 KiB

github_images/share-screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 63 KiB

github_images/upload-screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 342 KiB

github_images/video-screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 566 KiB

key/getKey.js
+8 -2
View File
@@ -1,13 +1,19 @@
const prompt = require('password-prompt')
const env = require("../backend/enviroment/env")
import env from "../backend/enviroment/env";
const crypto = require("crypto");
// NOT IN USE
const getKey = async() => {
if (process.env.KEY) {
// For Docker
env.key = process.env.KEY
const password = process.env.KEY;
env.key = password;
//console.log("Docker Key", env.key);
} else if (process.env.NODE_ENV) {
key/getNewKey.js
+1 -1
View File
@@ -1,5 +1,5 @@
const prompt = require('password-prompt')
const env = require("../backend/enviroment/env")
import env from "../backend/enviroment/env";
const crypto = require("crypto");
const getKey = async() => {
package.json
+35 -25
View File
@@ -1,41 +1,50 @@
{
"name": "my-drive",
"version": "1.0.0",
"version": "1.0.1",
"main": "index.js",
"license": "GNU General Public License v3.0",
"engines": {
"node": "13.11.0"
},
"scripts": {
"server": "live-server public/",
"build": "cross-env NODE_ENV=production webpack -p --env production",
"build:no-ssl": "cross-env NODE_ENV=production-no-ssl webpack -p --env production-no-ssl",
"build:dev": "webpack",
"build": "tsc && webpack -p --env production && webpack --config webUI.config.js",
"build:docker": "export NODE_OPTIONS=--max_old_space_size=8000 && tsc && webpack -p --env production && webpack --config webUI.config.js",
"build:webUI": "webpack --config webUI.config.js",
"build:dev": "tsc && webpack --env development",
"dev-server": "webpack-dev-server",
"test": "cross-env NODE_ENV=test env-cmd -f ./config/test.env jest --config=jest.config.json --watchAll --runInBand",
"start:dev": "env-cmd -f ./config/dev.env node server/serverStart.js",
"start": "cross-env NODE_ENV=production env-cmd -f ./config/prod.env node server/serverStart.js --env production",
"start:no-ssl": "cross-env NODE_ENV=production-no-ssl env-cmd -f ./config/prod.env node server/serverStart.js --env production-no-ssl",
"clean-database": "env-cmd -f ./config/prod.env node serverUtils/cleanDatabase.js",
"clean-database:dev": "env-cmd -f ./config/dev.env node serverUtils/cleanDatabase.js",
"backup-database": "env-cmd -f ./config/prod.env node serverUtils/backupDatabase.js",
"backup-database:dev": "env-cmd -f ./config/dev.env node serverUtils/backupDatabase.js",
"restore-database": "env-cmd -f ./config/prod.env node serverUtils/restoreDatabase.js",
"restore-database:dev": "env-cmd -f ./config/dev.env node serverUtils/restoreDatabase.js",
"delete-temp-database": "env-cmd -f ./config/prod.env node serverUtils/deleteTempDatabase.js",
"delete-temp-database:dev": "env-cmd -f ./config/dev.env node serverUtils/deleteTempDatabase.js",
"delete-database": "env-cmd -f ./config/prod.env node serverUtils/deleteDatabase.js",
"delete-database:dev": "env-cmd -f ./config/dev.env node serverUtils/deleteDatabase.js",
"change-password-database": "env-cmd -f ./config/prod.env node serverUtils/changeEncryptionPassword.js",
"change-password-database:dev": "env-cmd -f ./config/dev.env node serverUtils/changeEncryptionPassword.js",
"create-indexes-database": "env-cmd -f ./config/prod.env node serverUtils/createIndexes.js",
"create-indexes-database:dev": "env-cmd -f ./config/dev.env node serverUtils/createIndexes.js",
"init": "node serverUtils/initServer.js",
"heroku-postbuild": "yarn run build:prod"
"test": "NODE_ENV=test env-cmd -f ./config/test.env jest --config=jest.config.json --watchAll --runInBand",
"start:dev": "NODE_ENV=dev node dist/server/serverStart.js",
"start": "node dist/server/serverStart.js --env production",
"create-indexes-database": "node serverUtils/createIndexes.js",
"increase-node-memory": "export NODE_OPTIONS=--max_old_space_size=8000",
"create-indexes-database:dev": "node serverUtils/createIndexes.js",
"setup": "node serverUtils/setupServer.js",
"debug": "env-cmd -f ./config/prod.env node --expose-gc --inspect=9222 dist/server/serverStart.js --env production",
"heroku-postbuild": "npm run build"
},
"dependencies": {
"@babel/core": "^7.8.4",
"@babel/parser": "^7.9.4",
"@babel/plugin-proposal-class-properties": "^7.8.3",
"@babel/plugin-proposal-object-rest-spread": "^7.8.3",
"@babel/preset-env": "^7.8.4",
"@babel/preset-react": "^7.8.3",
"@babel/types": "^7.9.5",
"@types/bcrypt": "^3.0.0",
"@types/compression": "^1.7.0",
"@types/concat-stream": "^1.6.0",
"@types/connect-busboy": "0.0.2",
"@types/express": "^4.17.6",
"@types/helmet": "0.0.45",
"@types/jsonwebtoken": "^8.3.9",
"@types/mongodb": "^3.5.5",
"@types/mongoose": "^5.7.10",
"@types/node": "^13.13.5",
"@types/sharp": "^0.25.0",
"@types/uuid": "^7.0.2",
"@types/validator": "^13.0.0",
"aws-sdk": "^2.657.0",
"axios": "^0.19.2",
"babel-polyfill": "^6.26.0",
"bcrypt": "^3.0.8",
@@ -49,6 +58,7 @@
"core-js": "^3.6.4",
"crypto": "^1.0.1",
"diskusage": "^1.1.3",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"fluent-ffmpeg": "^2.1.2",
"helmet": "^3.21.2",
@@ -76,6 +86,7 @@
"sharp": "^0.23.4",
"sweetalert2": "^9.7.2",
"temp": "^0.9.1",
"typescript": "^3.8.3",
"uuid": "^3.4.0",
"validator": "^12.2.0",
"webpack": "^4.41.6"
@@ -88,7 +99,6 @@
"cross-env": "^6.0.3",
"css-loader": "^3.4.2",
"dart-sass": "^1.25.0",
"dotenv": "^8.2.0",
"env-cmd": "^10.1.0",
"enzyme": "^3.11.0",
"enzyme-adapter-react-16": "^1.15.2",
public/images/icon.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

public/index.html
+4
View File
@@ -6,6 +6,10 @@
<title>MyDrive</title>
<link rel="icon" href="/images/icon.png">
<link rel="shortcut icon" type="image/png" href="/images/icon.png">
<link rel="shortcut icon" sizes="192x192" href="/images/icon.png">
<link rel="apple-touch-icon" href="/images/icon.png">
<link rel="stylesheet" type="text/css" href="/dist/styles.css">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
readme.md
+132 -70
View File
@@ -1,117 +1,179 @@
# ![MyDrive Homepage](github_images/mydrivehome2.png)
# MyDrive
MyDrive is an Open Source Cloud Server (Similar To Google Drive), built with Node.JS, Express, React, and MongoDB.
MyDrive is an Open Source Cloud Server (Similar To Google Drive), the service uses mongoDB to store file/folder metadata, and supports multiple databases to store the file chunks, such as Amazon S3, the Filesystem, or just MongoDB. MyDrive is built using Node.js, and Typescript. The service now even supports Docker images!
Demo: https://mydrive-demo.herokuapp.com/
- Note: The Upload and Download Features, and other core features, are disabled in the demo.
## Index
### Wiki
* [Features](#features)
* [Installation](#installation)
* [WebUI For Encryption Key](#webui-for-encryption-key)
* [Docker](#docker)
* [Environment Variables](#environment-variables)
* [Screenshots](#screenshots)
* [Wiki](https://github.com/subnub/myDrive/wiki)
* [Video](#video)
* [Demo](#demo)
For a more detailed list of myDrive features, including examples with images, visit the wiki here: https://github.com/subnub/myDrive/wiki
## Features
MyDrive Features:
- Upload Files
- Download Files
- Create Folders
- Share Files
- Creates Image Thumbnails
- Photo-Viewer
- Stream Video
- Transcode Video
- Create One-Time Download links for files
- Move Files/Folders
- AES256 Encryption (For file chunks, tokens, and more!)
- Search For Files/Folders
- Mobile Friendly (Including Uploading!)
- Advanced Filter Options
* Upload Files
* Download Files
* Share Files
* Multiple DB Support (MongoDB, S3, Filesystem)
* Photo Viewer
* Video Viewer
* Thumbnails
* One-time download links
* Move Folder/Files
* Mobile Support
* Docker Support
* Search/Filter Options
* AES256 Encryption
## Installation
Required:
- Node.js (13.9+ Is Recommended)
- MongoDB
- Node.js (13+ Recommended)
- MongoDB (Unless using a service like Atlas)
Windows users will usually need both the microsoft visual build tools, and python 2. These are required to build the sharp module.
Windows users will usually need both the microsoft visual build tools, and python 2. These are required to build the sharp module:
- Visual Tools: http://go.microsoft.com/fwlink/?LinkId=691126
- Python 2: https://www.python.org/downloads/release/python-2717/
Optional:
- FFMPEG (For Video Transcoding, Does not work very well because of encryption ATM, But non-transcoded video streams work much better, I do not recommend using transcoding yet.)
Linux users will need to make sure they have 'build-essential' installed:
```bash
sudo apt-get install build-essential
```
Setup:
- “npm install”, Installs all dependencies.
>Install Node Modules
``` javascript
npm install
```
- "npm run init", Asks user for some environment variable details, pressing enter on these fields will instead use the default values.
>Create Environment Variables, Users can use the built in command to easily create the needed Environment files, or view the Environment Variables section to see how to manually create the files.
``` javascript
npm run setup
```
- "npm run build" or "npm run build:no-ssl", This will look for the HTTPS Certificate, Place this HTTPS certificate at the root of the project, with the naming convention (certificate.ca-bundle, certificate.crt, and certificate.key).
Note: If you do not have or want to use a HTTPS certificate, run "npm run build:no-ssl". This will run the server in production mode, without SSL encryption, this is vulnerable to man in the middle attacks!
>Run the build command
``` javascript
npm run build
```
- “npm run create-indexes-database”, This will create indexes for mongoDB, this is greatly recommended, it will improve mongoDBs speed in retrieving files.
>(Optional) Create the MongoDB indexes, this increases performance. MongoDB must be running for this command to work.
```javascript
npm run create-indexes-database
```
- "npm start" or "npm run start:no-ssl". If you do not want SSL Encryption use "npm run start:no-ssl".
>Start the server
``` javascript
npm run start
```
- It will then ask for an encryption key, this can be whatever you would like, this key is later hashed. Do not lose this key, there is NO way to recover data from a lost key!
## WebUI For Encryption Key
- That's It! Now just Create a new account.
MyDrive will first host a server on http://localhost:3000 in order to safely get the encryption key, just navigate to this URL in a browser, and enter the encryption key.
## Enviroment Variables
If you're using a service like SSH or a Droplet, you can forward the localhost connection safely like so:
```bash
ssh -L localhost:3000:localhost:3000 username@ip_address
```
The npm run init command will create the needed env variables, but if you would like to create them manually, structure the files like so.
Note: You can also disable using the webUI for the encryption key by providing a key in the server enviroment variables (e.g. KEY=password), but this is not recommended because it greatly reduces security.
Backend variables: Stored in /config/prod.env
## Docker
- MONGODB_URL=
- PASSWORD=
- HTTP_PORT=
- HTTPS_PORT=
- URL=
- FULL_URL=
- ROOT=
MyDrive has built in Docker support, there are two options when using Docker, users can either use the Docker image that has MongoDB built in, or use the Docker image that just has the MyDrive image (If you're using a service like Atlas).
Optional:
- KEY=
Create the Docker environment variables by running the 'npm run setup' command as seen in the installation section. Or by manually creating the file (e.g. docker-variables.env on the root of the project, see the enviroment section for more infomation).
The KEY variable is optional, without it the server will prompt you for a password on startup, with a KEY the server will skip prompting for the password, use a KEY if you want to use myDrive with docker, or similar tools.
Docker with mongoDB image:
```bash
docker-compose build
```
Frontend variables: Stored on the root of the project .env.production
Docker without mongoDB image:
```bash
docker-compose -f docker-compose-no-mongo.yml build
```
Start the Docker Image:
```bash
docker-compose up
```
- PORT=
- REMOTE_URL=
## Screenshots
## Built In Server Tools
Upload Files
![MyDrive Upload](github_images/upload-screenshot.png)
MyDrive comes with some built in NPM scripts for server management, this includes:
- Backup Database: Command “npm run backup-database", creates a temporary backup of the database inside of mongoDB, please note you can only have one backup at a time, for better backups use mongoExport.
Image Viewer
![Image Viewer](github_images/image-screenshot.png)
- Restore Database: Command “npm run restore-database”, Restores Database from backup.
Video Viewer
![Video Viewer](github_images/video-screenshot.png)
- Clean Database: Command “npm run clean-database”, Cleans database, removing any orphaned chunks that do not have a file associated with them, this command will automatically run backup database, incase anything fails.
Search For Files/Folders
![Search](github_images/search-screenshot.png)
- Change Encryption Password: Command “change-password-database”, Changed the encryption password, that the server first asks for on start up, this requires the old encryption password. This will also create a new backup, incase anything fails.
Move File/Folders
![Move](github_images/move-screenshot.png)
- Delete Database: Command “npm run delete-database”, Deletes the main database, does not delete the temporary backup.
Share
![Share](github_images/share-screenshot.png)
- Delete Temp Database: Command “npm run delete-temp-database”, Deletes the database backup.
Folders
![Folders](github_images/folder-screenshot.png)
- Create Indexes: Command “npm run create-indexes-database", Creates Indexes for mongoDB, without this mongoDB will need to search through every single file on request, run this before using the server.
Mobile
<div>
<img src="github_images/mobile-screenshot.jpeg" width="150">
</div>
- Run Tests: Command "npm run test", Starts unit testing.
## Environment Variables
Create a config folder on the root of the project, and create a file with the name prod.env for the server. For the client variables create a .env.production file in the root of the project.
Server Environment Variables:
- MONGODB_URL (Required): Sets the MongoDB URL, this should also work with DocumentDB.
- HTTP_PORT (Required): Sets the HTTP port number.
- HTTPS_PORT (Required): Sets the HTTPS port number.
- PASSWORD (Required): Sets the JWT password.
- DB_TYPE (Required): Sets the Database Type, options include s3/mongo/fs.
- NODE_ENV (Required): Must be set to 'production'.
- SSL (Optional): Enables SSL, place certificate.crt, certificate.ca-bundle, and certificate.key at the root of the project.
- KEY (Optional): Encryption key for data, this is not recommended, please use the built in webUI for setting the key.
- DOCKER (Optional/Required): Sets the server to use docker, set this to true.
- FS_DIRECTORY (Optional/Required): Sets the directory for file data on the file system.
- S3_ID (Optional/Required): Sets the Amazon S3 ID.
- S3_KEY (Optional/Required): Sets the Amazon S3 Key.
- S3_BUCKET (Optional/Required): Sets the Amazon Bucket.
- ROOT (Optional): Uses a filesystem path, is used for storage space.
- URL (Optional): Allows to specify URL to host on, this is usually not needed.
- USE_DOCUMENT_DB (Optional): Enables documentDB, this is experimental.
- DOCUMENT_DB_BUNDLE (Optional): Enables SSL with documentDB.
- BLOCK_CREATE_ACCOUNT (Optional): Blocks the ability to create accounts.
Client Environment Variables
- REMOTE_URL (Required): Sets the Remote URL for the client.
- DISABLE_STORAGE (Optional): Disables storage, use this if you're not using ROOT on the server.
## Wiki
For a more detailed list of myDrive features, including examples with images, visit the wiki here: https://github.com/subnub/myDrive/wiki
## Video
I created a short YouTube video, showing off myDrives design and features: https://youtu.be/0YKU5CZHG4I
## Security
MyDrive encrypts all file chunks, tokens, and temp tokens. These items are first encrypted with a randomly generated 32 byte private key, and random 16 byte public key (A different random public key is used for different items).The private key is encrypted with the users salted password, and then the hashed server key (Acquired on server startup). Note: Running the command to change the servers key, will generate new private and public keys for each user, and will have to re-encrypt all chunks.
## Demo
### Using myDrive for personal use?
After you create your account, disable the ability to create a new account by adding the following value to the prod.env file (Located inside on the config folder).
BLOCK_CREATE_ACCOUNT=true
Demo: https://mydrive-demo.herokuapp.com/
- Note: The Upload and Download Features, and other core features, are disabled in the demo.
## Questions? Feature Requests? Hiring? Contact Me!
Contact Email: kyle.hoell@gmail.com
## Follow my twitter account
Twitter: https://twitter.com/subnub2
rootpath.js
-1
View File
@@ -1 +0,0 @@
module.exports = __dirname
server/server.js
-83
View File
@@ -1,83 +0,0 @@
const express = require("express");
const app = express();
const path = require("path")
const publicPath = path.join(__dirname, "..", "public");
const userRouter = require("../backend/express-routers/user")
const fileRouter = require("../backend/express-routers/file")
const folderRouter = require("../backend/express-routers/folder");
const storageRouter = require("../backend/express-routers/storage");
const bodyParser = require('body-parser');
const https = require("https");
const fs = require("fs");
const helmet = require("helmet");
const busboy = require("connect-busboy")
const compression = require("compression");
const http = require("http");
let server;
let serverHttps;
if (process.env.NODE_ENV === 'production') {
const cert = fs.readFileSync("certificate.crt")
const ca = fs.readFileSync("certificate.ca-bundle");
const key = fs.readFileSync("certificate.key");
const options = {
cert,
ca,
key
}
serverHttps = https.createServer( options, app );
}
server = http.createServer(app);
require("../backend/db/mongoose");
app.use(helmet())
app.use(compression());
app.use(express.json());
app.use(express.static(publicPath));
app.use(bodyParser.json({limit: "50mb"}));
app.use(bodyParser.urlencoded({limit: "50mb", extended: true, parameterLimit:50000}))
app.use(busboy({
highWaterMark: 2 * 1024 * 1024,
}));
app.use(userRouter, fileRouter, folderRouter, storageRouter);
const nodeMode = process.env.NODE_ENV ? "Production" : "Development/Testing";
console.log("Node Enviroment Mode:", nodeMode);
app.get("*", (req, res) => {
res.sendFile(path.join(publicPath, "index.html"))
})
if (process.env.NODE_ENV === 'production') {
module.exports = {server, serverHttps};
} else {
module.exports = server;
}
server/serverStart.js
-41
View File
@@ -1,41 +0,0 @@
const getKey = require("../key/getKey");
const serverStart = async() => {
await getKey();
if (process.env.NODE_ENV === 'production') {
const {server, serverHttps} = require("./server");
server.listen(process.env.HTTP_PORT,process.env.URL, () => {
console.log("Http Server Running On Port:", process.env.HTTP_PORT);
})
serverHttps.listen(process.env.HTTPS_PORT, function () {
console.log( 'Https Server Running On Port:', process.env.HTTPS_PORT);
} );
} else if (process.env.NODE_ENV === 'production-no-ssl') {
const server = require("./server");
server.listen(process.env.HTTP_PORT,process.env.URL, () => {
console.log("Http Server (No-SSL) Running On Port:", process.env.HTTP_PORT);
})
} else {
const server = require("./server");
server.listen(process.env.HTTP_PORT,process.env.URL, () => {
console.log("Development Server Running On Port:", process.env.HTTP_PORT);
})
}
}
serverStart();
serverUtils/createIndexes.js
+3 -1
View File
@@ -1,4 +1,6 @@
const mongoose = require("../backend/db/mongoose");
const getEnvVariables = require("./getEnvVaribables");
getEnvVariables();
const mongoose = require("./mongoServerUtil");
const conn = mongoose.connection;
const waitForDatabase = () => {
serverUtils/createThumbnailBuffer.js
+2 -2
View File
@@ -1,7 +1,7 @@
const mongoose = require("../backend/db/mongooseServerUtils")
import mongoose from "../backend/db/mongoose";
const conn = mongoose.connection;
const crypto= require("crypto");
const env = require("../backend/enviroment/env");
import env from "../backend/enviroment/env";
const Thumbnail = require("../backend/models/thumbnail");
const ObjectID = require('mongodb').ObjectID
const sharp = require("sharp");

Some files were not shown because too many files have changed in this diff Show More