oxmc/myDrive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added path option for storing files on file system, started to add s3 support

Browse Source
This commit is contained in:
subnub
2020-04-17 00:13:12 -04:00
parent 7bfba3b499
commit 4821c5a408
10 changed files with 66 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/controllers/file.ts
+1 -1
View File
@@ -578,7 +578,7 @@ class FileController {
const userID = req.user._id;
const fileID = req.body.id;
await fileService.deleteFile(userID, fileID);
await mongoService.deleteFile(userID, fileID);
res.send()
backend/controllers/fileSystem.ts
+1 -1
View File
@@ -576,7 +576,7 @@ class FileSystemController {
const userID = req.user._id;
const fileID = req.body.id;
await fileService.deleteFile(userID, fileID);
await fileSystemService.deleteFile(userID, fileID);
res.send()
backend/db/s3.ts
+6
View File
@@ -0,0 +1,6 @@
import AWS from "aws-sdk";
AWS.config.update({
accessKeyId: "<Access Key Here>",
secretAccessKey: "<Secret Access Key Here>"
});
backend/enviroment/env.js
+2 -1
View File
@@ -6,5 +6,6 @@ module.exports = {
root: process.env.ROOT,
url: process.env.URL,
mongoURL: process.env.MONGODB_URL,
dbType: process.env.DB_TYPE
dbType: process.env.DB_TYPE,
fsDirectory: process.env.FS_DIRECTORY
}
backend/services/ChunkService/FileSystemService.ts
+22 -1
View File
@@ -18,6 +18,8 @@ import imageChecker from "../../utils/imageChecker";
import Thumbnail, {ThumbnailInterface} from "../../models/thumbnail";
import streamToBuffer from "../../utils/streamToBuffer";
import User from "../../models/user";
import env from "../../enviroment/env";
import { removeChunksFS } from "./utils/awaitUploadStreamFS";
const dbUtilsFile = new DbUtilFile();
@@ -62,7 +64,7 @@ class FileSystemService {
isVideo,
size,
IV: initVect,
filePath: `/Users/kylehoell/Documents/fstestdata/${systemFileName}`
filePath: env.fsDirectory + systemFileName
}
const fileWriteStream = fs.createWriteStream(metadata.filePath);
@@ -255,6 +257,25 @@ class FileSystemService {
await dbUtilsFile.removeOneTimePublicLink(fileID);
}
}
deleteFile = async(userID: string, fileID: string) => {
const file: FileInterface = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("Delete File Not Found Error");
if (file.metadata.thumbnailID) {
const thumbnail = await Thumbnail.findById(file.metadata.thumbnailID) as ThumbnailInterface;
const thumbnailPath = thumbnail.path!;
await removeChunksFS(thumbnailPath);
await Thumbnail.deleteOne({_id: file.metadata.thumbnailID});
}
await removeChunksFS(file.metadata.filePath!);
await File.deleteOne({_id: file._id});
}
}
export default FileSystemService;
backend/services/ChunkService/MongoService.ts
+25
View File
@@ -250,6 +250,31 @@ class MongoService implements ChunkInterface {
await awaitStream(readStream.pipe(decipher), res);
}
deleteFile = async(userID: string, fileID: string) => {
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255,
});
const file = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("Delete File Not Found Error");
if (file.metadata.thumbnailID) {
await Thumbnail.deleteOne({_id: file.metadata.thumbnailID});
}
if (file.metadata.isVideo && file.metadata.transcoded) {
try {
await bucket.delete(new ObjectID(file.metadata.transcodedID));
} catch (e) {
console.log("Could Not Find Transcoded Video");
}
}
await bucket.delete(new ObjectID(fileID));
}
}
export default MongoService;
backend/services/ChunkService/utils/awaitUploadStreamFS.ts
+5 -4
View File
@@ -1,7 +1,7 @@
import {Request} from "express"
import fs from "fs";
const removeChunks = (path: string) => {
export const removeChunksFS = (path: string) => {
return new Promise((resolve, reject) => {
@@ -12,6 +12,7 @@ const removeChunks = (path: string) => {
}
console.log("File Removed");
resolve();
})
})
}
@@ -22,7 +23,7 @@ const awaitUploadStream = <T>(inputSteam: any, outputStream: any, req: Request,
inputSteam.on("error", (e: Error) => {
removeChunks(path);
removeChunksFS(path);
reject({
message: "Await Stream Input Error",
@@ -33,7 +34,7 @@ const awaitUploadStream = <T>(inputSteam: any, outputStream: any, req: Request,
outputStream.on("error", (e: Error) => {
removeChunks(path);
removeChunksFS(path);
reject({
message: "Await Stream Output Error",
@@ -46,7 +47,7 @@ const awaitUploadStream = <T>(inputSteam: any, outputStream: any, req: Request,
console.log("Upload Request Cancelling...");
removeChunks(path);
removeChunksFS(path);
})
inputSteam.pipe(outputStream).on("finish", (data: T) => {
backend/services/FileService/indexnew.ts
-26
View File
@@ -231,32 +231,6 @@ class MongoFileService {
return file;
}
deleteFile = async(userID: string, fileID: string) => {
let bucket = new mongoose.mongo.GridFSBucket(conn.db, {
chunkSizeBytes: 1024 * 255,
});
const file = await dbUtilsFile.getFileInfo(fileID, userID);
if (!file) throw new NotFoundError("Delete File Not Found Error");
if (file.metadata.thumbnailID) {
await Thumbnail.deleteOne({_id: file.metadata.thumbnailID});
}
if (file.metadata.isVideo && file.metadata.transcoded) {
try {
await bucket.delete(ObjectID(file.metadata.transcodedID));
} catch (e) {
console.log("Could Not Find Transcoded Video");
}
}
await bucket.delete(ObjectID(fileID));
}
}
export default MongoFileService;
backend/services/FileService/utils/createThumbnailFS.ts
+3 -3
View File
@@ -1,7 +1,6 @@
const mongoose = require("../../../db/mongoose")
const conn = mongoose.connection;
const crypto= require("crypto");
const env = require("../../../enviroment/env");
import Thumbnail from "../../../models/thumbnail";
const ObjectID = require('mongodb').ObjectID
const sharp = require("sharp");
@@ -10,6 +9,7 @@ import {FileInterface} from "../../../models/file";
import {UserInterface} from "../../../models/user";
import fs from "fs";
import uuid from "uuid";
import env from "../../../enviroment/env";
const createThumbnailFS = (file: FileInterface, filename: string, user: UserInterface) => {
@@ -22,7 +22,7 @@ const createThumbnailFS = (file: FileInterface, filename: string, user: UserInte
const thumbnailFilename = uuid.v4();
const readStream = fs.createReadStream(file.metadata.filePath!);
const writeStream = fs.createWriteStream(`/Users/kylehoell/Documents/fstestdata/${thumbnailFilename}`);
const writeStream = fs.createWriteStream(env.fsDirectory + thumbnailFilename);
const decipher = crypto.createDecipheriv('aes256', CIPHER_KEY, file.metadata.IV);
readStream.on("error", (e: Error) => {
@@ -58,7 +58,7 @@ const createThumbnailFS = (file: FileInterface, filename: string, user: UserInte
writeStream.on("finish", async() => {
console.log("Thumbnail written");
const thumbnailModel = new Thumbnail({name: filename, owner: user._id, IV: thumbnailIV, path: `/Users/kylehoell/Documents/fstestdata/${thumbnailFilename}`});
const thumbnailModel = new Thumbnail({name: filename, owner: user._id, IV: thumbnailIV, path: env.fsDirectory + thumbnailFilename});
await thumbnailModel.save();
package.json
+1
View File
@@ -43,6 +43,7 @@
"@types/mongoose": "^5.7.10",
"@types/node": "^13.11.1",
"@types/uuid": "^7.0.2",
"aws-sdk": "^2.657.0",
"axios": "^0.19.2",
"babel-polyfill": "^6.26.0",
"bcrypt": "^3.0.8",