oxmc/myDrive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

continued dep injectiong

Browse Source
This commit is contained in:
subnub
2024-07-02 15:10:03 -04:00
parent 09cb81f96d
commit 7bd81a9aa5
16 changed files with 413 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/controllers/file.ts
+92 -15
View File
@@ -9,6 +9,7 @@ import {
removeStreamVideoCookie,
} from "../cookies/createCookies";
import ChunkService from "../services/ChunkService";
import streamToBuffer from "../utils/streamToBuffer";
const fileService = new FileService();
@@ -30,10 +31,8 @@ interface RequestType extends Request {
encryptedToken?: string;
}
type ChunkServiceType = FileSystemService | S3Service;
class FileController {
chunkService: ChunkServiceType;
chunkService;
constructor() {
this.chunkService = new ChunkService();
@@ -43,19 +42,43 @@ class FileController {
if (!req.user) {
return;
}
let responseSent = false;
try {
const user = req.user;
const id = req.params.id;
const decryptedThumbnail = await this.chunkService.getThumbnail(user, id);
const { readStream, decipher } = await this.chunkService.getThumbnail(
user,
id
);
res.send(decryptedThumbnail);
readStream.on("error", (e: Error) => {
console.log("Get thumbnail read stream error", e);
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error getting thumbnail");
}
});
decipher.on("error", (e: Error) => {
console.log("Get thumbnail decipher error", e);
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error getting thumbnail");
}
});
const bufferData = await streamToBuffer(readStream.pipe(decipher));
res.send(bufferData);
} catch (e: unknown) {
if (e instanceof Error) {
console.log("\nGet Thumbnail Error File Route:", e.message);
}
res.status(500).send("Server error getting thumbnail");
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error getting thumbnail");
}
}
};
@@ -63,12 +86,38 @@ class FileController {
if (!req.user) {
return;
}
let responseSent = false;
try {
const user = req.user;
const fileID = req.params.id;
await this.chunkService.getFullThumbnail(user, fileID, res);
const { decipher, readStream, file } =
await this.chunkService.getFullThumbnail(user, fileID);
readStream.on("error", (e: Error) => {
console.log("Get full thumbnail read stream error", e);
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error getting full thumbnail");
}
});
decipher.on("error", (e: Error) => {
console.log("Get full thumbnail decipher error", e);
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error gettingfull thumbnail");
}
});
res.set("Content-Type", "binary/octet-stream");
res.set(
"Content-Disposition",
'attachment; filename="' + file.filename + '"'
);
res.set("Content-Length", file.metadata.size.toString());
readStream.pipe(decipher).pipe(res);
} catch (e: unknown) {
if (e instanceof Error) {
console.log("\nGet Thumbnail Full Error File Route:", e.message);
@@ -379,18 +428,46 @@ class FileController {
if (!req.user) {
return;
}
let responseSent = false;
try {
const user = req.user;
const fileID = req.params.id;
await this.chunkService.downloadFile(user, fileID, res);
const { readStream, decipher, file } =
await this.chunkService.downloadFile(user, fileID, res);
readStream.on("error", (e: Error) => {
console.log("read stream error", e);
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error downloading file");
}
});
decipher.on("error", (e: Error) => {
console.log("decipher stream error", e);
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error downloading file");
}
});
res.set("Content-Type", "binary/octet-stream");
res.set(
"Content-Disposition",
'attachment; filename="' + file.filename + '"'
);
res.set("Content-Length", file.metadata.size.toString());
readStream.pipe(decipher).pipe(res);
} catch (e: unknown) {
if (e instanceof Error) {
console.log("\nDownload File Error File Route:", e.message);
}
res.status(500).send("Server error downloading file");
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error downloading file");
}
}
};
@@ -586,7 +663,7 @@ class FileController {
const userID = req.user._id;
const items = req.body.items;
await this.chunkService.trashMulti(userID, items);
await fileService.trashMulti(userID, items);
res.send();
} catch (e: unknown) {
@@ -607,7 +684,7 @@ class FileController {
const userID = req.user._id;
const items = req.body.items;
await this.chunkService.restoreMulti(userID, items);
await fileService.restoreMulti(userID, items);
res.send();
} catch (e: unknown) {
backend/services/ChunkService/FileSystemActions.ts
+16
View File
@@ -12,6 +12,22 @@ class FilesystemActions implements IStorageActions {
const fsReadableStream = fs.createReadStream(params.filePath);
return fsReadableStream;
}
async removeChunks(params: AuthParams) {
return new Promise<void>((resolve, reject) => {
if (!params.filePath) {
reject("File path not configured");
return;
}
fs.unlink(params.filePath, (err) => {
if (err) {
reject("Error removing file");
return;
}
resolve();
});
});
}
}
export { FilesystemActions };
backend/services/ChunkService/S3Actions.ts
+17
View File
@@ -17,6 +17,23 @@ class S3Actions implements IStorageActions {
.createReadStream();
return s3ReadableStream;
}
removeChunks(params: AuthParams) {
return new Promise<void>((resolve, reject) => {
if (!params.Key) {
reject("S3 not configured");
return;
}
const { s3Storage, bucket } = this.getAuth();
s3Storage.deleteObject({ Key: params.Key, Bucket: bucket }, (err) => {
if (err) {
reject("Error removing file");
return;
}
resolve();
});
});
}
}
export { S3Actions };
backend/services/ChunkService/StoreTypes.ts
+1
View File
@@ -12,4 +12,5 @@ export interface IStorageActions {
createReadStream(
params: AuthParams
): NodeJS.ReadableStream | internal.Readable;
removeChunks(params: AuthParams): Promise<void>;
}
backend/services/ChunkService/index.ts
+84 -184
View File
@@ -34,16 +34,15 @@ import MongoFileService from "../FileService";
import createVideoThumbnailFS from "./utils/createVideoThumbnailFS";
import { S3Actions } from "./S3Actions";
import { FilesystemActions } from "./FileSystemActions";
import { createGenericParams } from "./utils/storageHelper";
const dbUtilsFile = new DbUtilFile();
const dbUtilsFolder = new DbUtilFolder();
const folderService = new FolderService();
const fileService = new MongoFileService();
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
class StorageService implements ChunkInterface {
class StorageService {
constructor() {}
uploadFile = async (user: UserInterface, busboy: any, req: Request) => {
@@ -145,56 +144,13 @@ class StorageService implements ChunkInterface {
}
};
getFileWriteStream = async (
user: UserInterface,
file: FileInterface,
parentFolder: FolderInterface
) => {
const password = user.getEncryptionKey();
if (!password) throw new ForbiddenError("Invalid Encryption Key");
const initVect = crypto.randomBytes(16);
const CIPHER_KEY = crypto.createHash("sha256").update(password).digest();
const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect);
const filename = file.filename;
const parent = parentFolder._id;
const parentList = [...parentFolder.parentList, parentFolder._id];
const size = file.metadata.size;
const personalFile = file.metadata.personalFile ? true : false;
let hasThumbnail = file.metadata.hasThumbnail;
let thumbnailID = file.metadata.thumbnailID;
const isVideo = file.metadata.isVideo;
const systemFileName = uuid.v4();
const metadata = {
owner: user._id,
parent,
parentList,
hasThumbnail,
thumbnailID,
isVideo,
size,
IV: file.metadata.IV,
filePath: env.fsDirectory + systemFileName,
};
const fileWriteStream = fs.createWriteStream(metadata.filePath);
return fileWriteStream;
};
// INJECTED
downloadFile = async (user: UserInterface, fileID: string, res: Response) => {
console.log("DEP INJECT :)");
const currentFile = await dbUtilsFile.getFileInfo(
fileID,
user._id.toString()
);
console.log(fileID, user._id);
if (!currentFile) throw new NotFoundError("Download File Not Found");
const password = user.getEncryptionKey();
@@ -208,12 +164,10 @@ class StorageService implements ChunkInterface {
IV = IV.buffer;
}
const readStreamParams = {} as any;
if (env.dbType === "fs") {
readStreamParams.filePath = currentFile.metadata.filePath!;
} else {
readStreamParams.Key = currentFile.metadata.s3ID!;
}
const readStreamParams = createGenericParams({
filePath: currentFile.metadata.filePath,
Key: currentFile.metadata.s3ID,
});
const readStream = storageActions.createReadStream(readStreamParams);
@@ -221,43 +175,14 @@ class StorageService implements ChunkInterface {
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV);
res.set("Content-Type", "binary/octet-stream");
res.set(
"Content-Disposition",
'attachment; filename="' + currentFile.filename + '"'
);
res.set("Content-Length", currentFile.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
};
getFileReadStream = async (user: UserInterface, fileID: string) => {
const currentFile = await dbUtilsFile.getFileInfo(
fileID,
user._id.toString()
);
if (!currentFile) throw new NotFoundError("Download File Not Found");
const password = user.getEncryptionKey();
if (!password) throw new ForbiddenError("Invalid Encryption Key");
const filePath = currentFile.metadata.filePath!;
const IV = currentFile.metadata.IV.buffer as Buffer;
const readStream = fs.createReadStream(filePath);
const CIPHER_KEY = crypto.createHash("sha256").update(password).digest();
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV);
return readStream;
return {
readStream,
decipher,
file: currentFile,
};
};
// INJECTED
getThumbnail = async (user: UserInterface, id: string) => {
const password = user.getEncryptionKey();
@@ -281,25 +206,21 @@ class StorageService implements ChunkInterface {
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, iv);
const readStreamParams = {} as any;
const readStreamParams = createGenericParams({
filePath: thumbnail.path,
Key: thumbnail.s3ID,
});
const readStream = fs.createReadStream(thumbnail.path!);
const readStream = storageActions.createReadStream(readStreamParams);
const allStreamsToErrorCatch = [readStream, decipher];
const bufferData = await streamToBuffer(
readStream.pipe(decipher),
allStreamsToErrorCatch
);
return bufferData;
return {
readStream,
decipher,
};
};
getFullThumbnail = async (
user: UserInterface,
fileID: string,
res: Response
) => {
// INJECTED
getFullThumbnail = async (user: UserInterface, fileID: string) => {
const userID = user._id;
const file = await dbUtilsFile.getFileInfo(fileID, userID.toString());
@@ -307,26 +228,31 @@ class StorageService implements ChunkInterface {
if (!file) throw new NotFoundError("File Thumbnail Not Found");
const password = user.getEncryptionKey();
const IV = file.metadata.IV;
// TODO: Fix this type
let IV = file.metadata.IV as any;
if (!(IV instanceof Buffer)) {
IV = IV.buffer;
}
if (!password) throw new ForbiddenError("Invalid Encryption Key");
const readStream = fs.createReadStream(file.metadata.filePath!);
const readStreamParams = createGenericParams({
filePath: file.metadata.filePath,
Key: file.metadata.s3ID,
});
const readStream = storageActions.createReadStream(readStreamParams);
const CIPHER_KEY = crypto.createHash("sha256").update(password).digest();
const decipher = crypto.createDecipheriv("aes256", CIPHER_KEY, IV);
res.set("Content-Type", "binary/octet-stream");
res.set(
"Content-Disposition",
'attachment; filename="' + file.filename + '"'
);
res.set("Content-Length", file.metadata.size.toString());
const allStreamsToErrorCatch = [readStream, decipher];
await awaitStream(readStream.pipe(decipher), res, allStreamsToErrorCatch);
return {
readStream,
decipher,
file,
};
};
streamVideo = async (
@@ -490,60 +416,7 @@ class StorageService implements ChunkInterface {
}
};
trashMulti = async (
userID: string,
items: {
type: "file" | "folder" | "quick-item";
id: string;
file?: FileInterface;
folder?: FolderInterface;
}[]
) => {
const fileList = items.filter(
(item) => item.type === "file" || item.type === "quick-item"
);
const folderList = items
.filter((item) => item.type === "folder")
.sort((a, b) => {
if (!a.folder || !b.folder) return 0;
return b.folder.parentList.length - a.folder.parentList.length;
});
for (const file of fileList) {
await fileService.trashFile(userID, file.id);
}
for (const folder of folderList) {
await folderService.trashFolder(userID, folder.id);
}
};
restoreMulti = async (
userID: string,
items: {
type: "file" | "folder" | "quick-item";
id: string;
file?: FileInterface;
folder?: FolderInterface;
}[]
) => {
const fileList = items.filter(
(item) => item.type === "file" || item.type === "quick-item"
);
const folderList = items
.filter((item) => item.type === "folder")
.sort((a, b) => {
if (!a.folder || !b.folder) return 0;
return b.folder.parentList.length - a.folder.parentList.length;
});
for (const file of fileList) {
await fileService.restoreFile(userID, file.id);
}
for (const folder of folderList) {
await folderService.restoreFolder(userID, folder.id);
}
};
// INJECTED
deleteMulti = async (
userID: string,
items: {
@@ -571,6 +444,7 @@ class StorageService implements ChunkInterface {
}
};
// INJECTED
deleteFile = async (userID: string, fileID: string) => {
const file = await dbUtilsFile.getFileInfo(fileID, userID);
@@ -580,21 +454,27 @@ class StorageService implements ChunkInterface {
const thumbnail = (await Thumbnail.findById(
file.metadata.thumbnailID
)) as ThumbnailInterface;
const thumbnailPath = thumbnail.path!;
await removeChunksFS(thumbnailPath);
const removeChunksParams = createGenericParams({
filePath: thumbnail.path,
Key: thumbnail.s3ID,
});
await storageActions.removeChunks(removeChunksParams);
await Thumbnail.deleteOne({ _id: file.metadata.thumbnailID });
}
await removeChunksFS(file.metadata.filePath!);
const removeChunksParams = createGenericParams({
filePath: file.metadata.filePath,
Key: file.metadata.s3ID,
});
await storageActions.removeChunks(removeChunksParams);
await File.deleteOne({ _id: file._id });
await subtractFromStorageSize(
userID,
file.length,
file.metadata.personalFile!
);
};
// INJECTED
deleteFolder = async (userID: string, folderID: string) => {
const folder = await dbUtilsFolder.getFolderInfo(folderID, userID);
@@ -623,13 +503,23 @@ class StorageService implements ChunkInterface {
const thumbnail = (await Thumbnail.findById(
currentFile.metadata.thumbnailID
)) as ThumbnailInterface;
const thumbnailPath = thumbnail.path!;
await removeChunksFS(thumbnailPath);
const removeChunksParams = createGenericParams({
filePath: thumbnail.path,
Key: thumbnail.s3ID,
});
await storageActions.removeChunks(removeChunksParams);
await Thumbnail.deleteOne({ _id: currentFile.metadata.thumbnailID });
}
await removeChunksFS(currentFile.metadata.filePath!);
const removeChunksParams = createGenericParams({
filePath: currentFile.metadata.filePath,
Key: currentFile.metadata.s3ID,
});
await storageActions.removeChunks(removeChunksParams);
await File.deleteOne({ _id: currentFile._id });
} catch (e) {
console.log(
@@ -641,6 +531,7 @@ class StorageService implements ChunkInterface {
}
};
// INJECTED
deleteAll = async (userID: string) => {
await Folder.deleteMany({ owner: userID });
@@ -657,13 +548,22 @@ class StorageService implements ChunkInterface {
const thumbnail = (await Thumbnail.findById(
currentFile.metadata.thumbnailID
)) as ThumbnailInterface;
const thumbnailPath = thumbnail.path!;
await removeChunksFS(thumbnailPath);
const removeChunksParams = createGenericParams({
filePath: thumbnail.path,
Key: thumbnail.s3ID,
});
await storageActions.removeChunks(removeChunksParams);
await Thumbnail.deleteOne({ _id: currentFile.metadata.thumbnailID });
}
await removeChunksFS(currentFile.metadata.filePath!);
const removeChunksParams = createGenericParams({
filePath: currentFile.metadata.filePath,
Key: currentFile.metadata.s3ID,
});
await storageActions.removeChunks(removeChunksParams);
await File.deleteOne({ _id: currentFile._id });
} catch (e) {
console.log(
backend/services/ChunkService/utils/ChunkInterface.ts
+1
View File
@@ -2,6 +2,7 @@ import { UserInterface } from "../../../models/user";
import { FileInterface } from "../../../models/file";
import { Request, Response } from "express";
import { FolderInterface } from "../../../models/folder";
import crypto from "crypto";
interface ChunkInterface {
uploadFile: (
backend/services/ChunkService/utils/awaitStream.ts
+21 -21
View File
@@ -1,23 +1,23 @@
const awaitStream = <T>(inputSteam: any, outputStream: any, allStreamsToErrorCatch: any[]) => {
const awaitStream = <T>(
inputSteam: any,
outputStream: any,
allStreamsToErrorCatch: any[]
) => {
return new Promise<T>((resolve, reject) => {
allStreamsToErrorCatch.forEach((currentStream: any) => {
currentStream.on("error", (e: Error) => {
reject({
message: "Await Stream Input Error",
code: 500,
error: e,
});
});
});
return new Promise<T>((resolve, reject) => {
inputSteam.pipe(outputStream).on("finish", (data: T) => {
resolve(data);
});
});
};
allStreamsToErrorCatch.forEach((currentStream: any) => {
currentStream.on("error", (e: Error) => {
reject({
message: "Await Stream Input Error",
code: 500,
error: e
})
})
})
inputSteam.pipe(outputStream).on("finish", (data: T) => {
resolve(data);
})
})
}
export default awaitStream;
export default awaitStream;
backend/services/ChunkService/utils/awaitUploadStreamS3.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { ManagedUpload } from "aws-sdk/clients/s3";
import s3 from "../../../db/s3";
const awaitUploadStreamS3 = (
@@ -6,7 +7,7 @@ const awaitUploadStreamS3 = (
s3Data: { id: string; key: string; bucket: string }
) => {
return new Promise<void>((resolve, reject) => {
s3.upload(params, (err: any, data: any) => {
s3.upload(params, (err: any, data: ManagedUpload.SendData) => {
if (err) {
console.log("Amazon upload err", err);
reject("Amazon upload error");
backend/services/ChunkService/utils/storageHelper.ts
+22
View File
@@ -0,0 +1,22 @@
import env from "../../../enviroment/env";
type GenericParmasType = {
filePath?: string;
Key?: string;
Bucket?: string;
};
export const createGenericParams = ({ filePath, Key }: GenericParmasType) => {
// TODO: Remove file split after migration
if (env.dbType === "fs") {
const filePathSplit = filePath!.split("/");
const fileName = filePathSplit[filePathSplit.length - 1];
return {
filePath: env.fsDirectory + fileName,
};
} else {
return {
Key,
};
}
};
backend/services/FileService/index.ts
+56
View File
@@ -10,9 +10,11 @@ import DbUtilFolder from "../../db/utils/folderUtils";
import { UserInterface } from "../../models/user";
import { FileInterface } from "../../models/file";
import tempStorage from "../../tempStorage/tempStorage";
import FolderService from "../FolderService";
const dbUtilsFile = new DbUtilFile();
const dbUtilsFolder = new DbUtilFolder();
const folderService = new FolderService();
type userAccessType = {
_id: string;
@@ -249,12 +251,66 @@ class MongoFileService {
return trashedFile;
};
trashMulti = async (
userID: string,
items: {
type: "file" | "folder" | "quick-item";
id: string;
file?: FileInterface;
folder?: FolderInterface;
}[]
) => {
const fileList = items.filter(
(item) => item.type === "file" || item.type === "quick-item"
);
const folderList = items
.filter((item) => item.type === "folder")
.sort((a, b) => {
if (!a.folder || !b.folder) return 0;
return b.folder.parentList.length - a.folder.parentList.length;
});
for (const file of fileList) {
await this.trashFile(userID, file.id);
}
for (const folder of folderList) {
await folderService.trashFolder(userID, folder.id);
}
};
restoreFile = async (userID: string, fileID: string) => {
const restoredFile = await dbUtilsFile.restoreFile(fileID, userID);
if (!restoredFile) throw new NotFoundError("Restore File Not Found Error");
return restoredFile;
};
restoreMulti = async (
userID: string,
items: {
type: "file" | "folder" | "quick-item";
id: string;
file?: FileInterface;
folder?: FolderInterface;
}[]
) => {
const fileList = items.filter(
(item) => item.type === "file" || item.type === "quick-item"
);
const folderList = items
.filter((item) => item.type === "folder")
.sort((a, b) => {
if (!a.folder || !b.folder) return 0;
return b.folder.parentList.length - a.folder.parentList.length;
});
for (const file of fileList) {
await this.restoreFile(userID, file.id);
}
for (const folder of folderList) {
await folderService.restoreFolder(userID, folder.id);
}
};
renameFile = async (userID: string, fileID: string, title: string) => {
const file = await dbUtilsFile.renameFile(fileID, userID, title);
backend/utils/streamToBuffer.ts
+8 -25
View File
@@ -1,27 +1,10 @@
const streamToBuffer = (stream: any, allStreamsToErrorCatch: any[]) => {
const chunks: any[] = []
const streamToBuffer = (stream: any) => {
return new Promise<Buffer>((resolve, reject) => {
const chunks: any[] = [];
stream.on("data", (chunk: any) => chunks.push(chunk));
stream.on("error", reject);
stream.on("end", () => resolve(Buffer.concat(chunks)));
});
};
allStreamsToErrorCatch.forEach((currentStream) => {
currentStream.on("error", (e: Error) => {
console.log("Stream To Buffer Error", e);
reject({
message: "stream to buffer error",
code: 500,
error: e
})
})
})
stream.on('data', (chunk: any) => chunks.push(chunk));
stream.on('error', reject);
stream.on('end', () => resolve(Buffer.concat(chunks)));
})
}
export default streamToBuffer;
export default streamToBuffer;