oxmc/myDrive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleaned up file uploads

Browse Source
This commit is contained in:
subnub
2024-12-07 18:55:56 -05:00
parent 7e134b300f
commit e466ec0d07
10 changed files with 250 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/controllers/file-controller.ts
+10 -106
View File
@@ -131,126 +131,30 @@ class FileController {
}
};
uploadFile = async (req: RequestTypeFullUser, res: Response) => {
uploadFile = async (
req: RequestTypeFullUser,
res: Response,
next: NextFunction
) => {
if (!req.user) {
return;
}
let responseSent = false;
const handleError = () => {
if (!responseSent) {
responseSent = true;
res.writeHead(500, { Connection: "close" });
res.end();
}
};
const handleFinish = async (
filename: string,
metadata: FileMetadateInterface
) => {
try {
const user = req.user;
if (!user) throw new NotAuthorizedError("User Not Authorized");
const date = new Date();
let length = 0;
if (env.dbType === "fs" && metadata.filePath) {
length = (await getFileSize(metadata.filePath)) as number;
} else {
// TODO: Fix this we should be using the encrypted file size
length = metadata.size;
}
const currentFile = new File({
filename,
uploadDate: date.toISOString(),
length,
metadata,
});
await currentFile.save();
const imageCheck = imageChecker(currentFile.filename);
const videoCheck = videoChecker(currentFile.filename);
if (videoCheck) {
const updatedFile = await createVideoThumbnail(
currentFile,
filename,
user
);
res.send(updatedFile);
} else if (currentFile.length < 15728640 && imageCheck) {
const updatedFile = await createThumbnail(
currentFile,
filename,
user
);
res.send(updatedFile);
} else {
res.send(currentFile);
}
} catch (e: unknown) {
if (!responseSent) {
res.writeHead(500, { Connection: "close" });
res.end();
}
}
};
try {
const user = req.user;
const busboy = req.busboy;
busboy.on("error", (e: Error) => {
console.log("busboy error", e);
handleError();
req.on("error", (e: Error) => {
next(e);
});
req.pipe(busboy);
const { cipher, fileWriteStream, emitter, metadata, filename } =
await this.chunkService.uploadFile(user, busboy, req);
const file = await this.chunkService.uploadFile(user, busboy, req);
cipher.on("error", (e: Error) => {
console.log("cipher error", e);
handleError();
});
fileWriteStream.on("error", (e: Error) => {
console.log("file write stream error", e);
handleError();
});
if (emitter) {
emitter.on("finish", async () => {
await handleFinish(filename, metadata);
});
} else {
fileWriteStream.on("finish", async () => {
await handleFinish(filename, metadata);
});
}
cipher.pipe(fileWriteStream);
res.send(file);
} catch (e: unknown) {
if (!responseSent) {
res.writeHead(500, { Connection: "close" });
if (e instanceof Error) {
console.log("\nUploading File Error File Route:", e.message);
res.end(e.message);
} else {
console.log("\nUploading File Error File Route:", e);
res.end("Server error uploading file");
}
}
next(e);
}
};
backend/db/mongoDB/fileDB.ts
+21
View File
@@ -120,6 +120,27 @@ class DbUtil {
// UPDATE
updateFileUploadedFile = async (
fileID: string,
userID: string,
parent: string,
parentList: string
) => {
const file = await File.findOneAndUpdate(
{ _id: new ObjectId(fileID), "metadata.owner": userID },
{
$set: {
"metadata.parent": parent,
"metadata.parentList": parentList,
},
$unset: { "metadata.processingFile": null },
},
{ new: true }
);
return file;
};
updateFolderUploadedFile = async (
fileID: string,
userID: string,
backend/services/chunk-service/actions/helper-actions.ts
+11
View File
@@ -0,0 +1,11 @@
import { S3Actions } from "../actions/S3-actions";
import { FilesystemActions } from "../actions/file-system-actions";
import env from "../../../enviroment/env";
export const getStorageActions = () => {
if (env.dbType === "s3") {
return new S3Actions();
} else {
return new FilesystemActions();
}
};
backend/services/chunk-service/chunk-service.ts
+12 -81
View File
@@ -3,7 +3,7 @@ import { UserInterface } from "../../models/user-model";
import NotAuthorizedError from "../../utils/NotAuthorizedError";
import NotFoundError from "../../utils/NotFoundError";
import crypto from "crypto";
import getBusboyData from "./utils/getBusboyData";
import uploadFileToStorage from "./utils/getBusboyData";
import videoChecker from "../../utils/videoChecker";
import uuid from "uuid";
import { FileInterface, FileMetadateInterface } from "../../models/file-model";
@@ -23,65 +23,20 @@ import fixEndChunkLength from "./utils/fixEndChunkLength";
import archiver from "archiver";
import async from "async";
import getFolderBusboyData from "./utils/getFolderUploadBusboyData";
import { getStorageActions } from "./actions/helper-actions";
const fileDB = new FileDB();
const folderDB = new FolderDB();
const thumbnailDB = new ThumbnailDB();
const userDB = new UserDB();
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const storageActions = getStorageActions();
class StorageService {
constructor() {}
uploadFile = async (user: UserInterface, busboy: any, req: Request) => {
const password = user.getEncryptionKey();
if (!password) throw new ForbiddenError("Invalid Encryption Key");
const initVect = crypto.randomBytes(16);
const CIPHER_KEY = crypto.createHash("sha256").update(password).digest();
const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect);
const { file, filename: fileInfo, formData } = await getBusboyData(busboy);
const filename = fileInfo.filename;
const parent = formData.get("parent") || "/";
const size = +formData.get("size");
const hasThumbnail = false;
const thumbnailID = "";
const isVideo = videoChecker(filename);
const errors = [];
if (!filename || filename.length === 0 || filename.length > 255) {
errors.push({
msg: "Filename is required and length must be greater than 0 and less than 255",
param: "filename",
});
}
if (!file) {
errors.push({ msg: "File is required", param: "file" });
}
// @prettier-ignore
if (!(file instanceof Readable)) {
errors.push({ msg: "File must be a readable stream", param: "file" });
}
if (!size || size < 0) {
errors.push({ msg: "Size must be a positive integer", param: "size" });
}
if (errors.length > 0) {
throw new Error(
`Invalid request parameters ${errors.map((e) => e.msg).join(", ")}`
);
}
const { parent, file } = await uploadFileToStorage(busboy, user);
const parentList = [];
@@ -96,38 +51,14 @@ class StorageService {
parentList.push("/");
}
const randomFilenameID = uuid.v4();
const metadata = {
owner: user._id.toString(),
await fileDB.updateFileUploadedFile(
file._id!.toString(),
user._id.toString(),
parent,
parentList: parentList.toString(),
hasThumbnail,
thumbnailID,
isVideo,
size,
IV: initVect,
} as FileMetadateInterface;
if (env.dbType === "fs") {
metadata.filePath = env.fsDirectory + randomFilenameID;
} else {
metadata.s3ID = randomFilenameID;
}
const { writeStream, emitter } = storageActions.createWriteStream(
metadata,
file.pipe(cipher),
randomFilenameID
parentList.toString()
);
return {
cipher,
fileWriteStream: writeStream,
emitter,
metadata,
filename,
};
return file;
};
uploadFolder = async (
@@ -202,7 +133,7 @@ class StorageService {
if (parentDirectory && foldersCreated[parentDirectory]) {
tempParentList.push(
...foldersCreated[parentDirectory].parentList,
foldersCreated[parentDirectory]._id.toString()
foldersCreated[parentDirectory]._id!.toString()
);
} else {
tempParentList.push(...parentList);
@@ -230,10 +161,10 @@ class StorageService {
await fileDB.updateFolderUploadedFile(
currentFile.uploadedFileId,
user._id.toString(),
foldersCreated[parentDirectory]._id.toString(),
foldersCreated[parentDirectory]._id!.toString(),
[
...foldersCreated[parentDirectory].parentList,
foldersCreated[parentDirectory]._id.toString(),
foldersCreated[parentDirectory]._id!.toString(),
].toString()
);
} else {
backend/services/chunk-service/utils/createImageThumbnail.ts
+2 -2
View File
@@ -10,11 +10,11 @@ import { S3Actions } from "../actions/S3-actions";
import { FilesystemActions } from "../actions/file-system-actions";
import { EventEmitter } from "stream";
import FileDB from "../../../db/mongoDB/fileDB";
import { getStorageActions } from "../actions/helper-actions";
const fileDB = new FileDB();
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const storageActions = getStorageActions();
const processData = (
file: FileInterface,
backend/services/chunk-service/utils/createVideoThumbnail.ts
+2 -2
View File
@@ -14,9 +14,9 @@ import tempCreateVideoThumbnailFS from "./tempCreateVideoThumbnailFS";
import { S3Actions } from "../actions/S3-actions";
import { FilesystemActions } from "../actions/file-system-actions";
import { createGenericParams } from "./storageHelper";
import { getStorageActions } from "../actions/helper-actions";
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const storageActions = getStorageActions();
const attemptToRemoveChunks = async (
file: FileInterface,
backend/services/chunk-service/utils/getBusboyData.ts
+185 -19
View File
@@ -1,20 +1,170 @@
import { Stream } from "stream";
import { EventEmitter, Stream } from "stream";
import { UserInterface } from "../../../models/user-model";
import File, {
FileInterface,
FileMetadateInterface,
} from "../../../models/file-model";
import uuid from "uuid";
import crypto from "crypto";
import ForbiddenError from "../../../utils/ForbiddenError";
import env from "../../../enviroment/env";
import { getStorageActions } from "../actions/helper-actions";
import getFileSize from "./getFileSize";
import imageChecker from "../../../utils/imageChecker";
import videoChecker from "../../../utils/videoChecker";
import createVideoThumbnail from "./createVideoThumbnail";
import createThumbnail from "./createImageThumbnail";
type dataType = {
file: Stream;
filename: {
filename: string;
};
formData: Map<any, any>;
// TODO: We should stop using moongoose directly here,
// Also in our fileDB make sure we are actually using File instead
// Of just modifying data directly so we get validation
const storageActions = getStorageActions();
type FileInfo = {
file: FileInterface;
parent: string;
};
const getBusboyData = (busboy: any) => {
return new Promise<dataType>((resolve, reject) => {
const formData = new Map();
const processData = (busboy: any, user: UserInterface) => {
const eventEmitter = new EventEmitter();
try {
let parent = "";
let size = 0;
const handleFinish = async (
filename: string,
metadata: FileMetadateInterface
) => {
const date = new Date();
let length = 0;
if (env.dbType === "fs" && metadata.filePath) {
length = (await getFileSize(metadata.filePath)) as number;
} else {
// TODO: Fix this we should be using the encrypted file size
length = metadata.size;
}
const currentFile = new File({
filename,
uploadDate: date.toISOString(),
length,
metadata,
});
await currentFile.save();
const imageCheck = imageChecker(currentFile.filename);
const videoCheck = videoChecker(currentFile.filename);
if (videoCheck) {
const updatedFile = await createVideoThumbnail(
currentFile,
filename,
user
);
return updatedFile;
} else if (currentFile.length < 15728640 && imageCheck) {
const updatedFile = await createThumbnail(currentFile, filename, user);
return updatedFile;
} else {
return currentFile;
}
};
const uploadFile = (filename: string, fileStream: Stream) => {
return new Promise<{ filename: string; metadata: FileMetadateInterface }>(
(resolve, reject) => {
const randomFilenameID = uuid.v4();
const password = user.getEncryptionKey();
if (!password) throw new ForbiddenError("Invalid Encryption Key");
const initVect = crypto.randomBytes(16);
const CIPHER_KEY = crypto
.createHash("sha256")
.update(password)
.digest();
const cipher = crypto.createCipheriv("aes256", CIPHER_KEY, initVect);
const metadata = {
owner: user._id.toString(),
parent: "/",
parentList: ["/"].toString(),
hasThumbnail: false,
thumbnailID: "",
isVideo: false,
size,
IV: initVect,
processingFile: true,
} as FileMetadateInterface;
if (env.dbType === "fs") {
metadata.filePath = env.fsDirectory + randomFilenameID;
} else {
metadata.s3ID = randomFilenameID;
}
const { writeStream, emitter } = storageActions.createWriteStream(
metadata,
fileStream.pipe(cipher),
randomFilenameID
);
writeStream.on("error", (e: Error) => {
reject(e);
});
cipher.on("error", (e: Error) => {
reject(e);
});
fileStream.on("error", (e: Error) => {
reject(e);
});
cipher.pipe(writeStream);
if (emitter) {
emitter.on("finish", () => {
resolve({ filename, metadata });
});
} else {
writeStream.on("finish", () => {
resolve({ filename, metadata });
});
}
}
);
};
const processFile = async (filename: string, fileStream: Stream) => {
try {
const { filename: newFilename, metadata } = await uploadFile(
filename,
fileStream
);
const file = await handleFinish(newFilename, metadata);
eventEmitter.emit("finish", {
file,
parent,
});
} catch (e) {
eventEmitter.emit("error", e);
}
};
busboy.on("field", (field: any, val: any) => {
if (typeof val !== "string" || val !== "undefined") {
formData.set(field, val);
if (field === "parent") {
parent = val;
} else if (field === "size") {
size = +val;
}
});
@@ -23,18 +173,34 @@ const getBusboyData = (busboy: any) => {
(
_: string,
file: Stream,
filename: {
filedata: {
filename: string;
}
) => {
resolve({
file,
filename,
formData,
});
processFile(filedata.filename, file);
}
);
busboy.on("error", (e: Error) => {
eventEmitter.emit("error", e);
});
} catch (e) {
eventEmitter.emit("error", e);
}
return eventEmitter;
};
const uploadFileToStorage = (busboy: any, user: UserInterface) => {
return new Promise<FileInfo>((resolve, reject) => {
const eventEmitter = processData(busboy, user);
eventEmitter.on("finish", (data) => {
resolve(data);
});
eventEmitter.on("error", (e) => {
reject(e);
});
});
};
export default getBusboyData;
export default uploadFileToStorage;
backend/services/chunk-service/utils/getFolderUploadBusboyData.ts
+2 -2
View File
@@ -16,6 +16,7 @@ import videoChecker from "../../../utils/videoChecker";
import createVideoThumbnail from "./createVideoThumbnail";
import createThumbnail from "./createImageThumbnail";
import { EventEmitter } from "events";
import { getStorageActions } from "../actions/helper-actions";
type FileDataType = {
name: string;
@@ -27,8 +28,7 @@ type FileDataType = {
uploadedFileId: string;
};
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const storageActions = getStorageActions();
type dataType = Record<string, FileDataType>;
backend/services/chunk-service/utils/tempCreateVideoThumbnail.ts
+3 -3
View File
@@ -13,9 +13,9 @@ import ffmpeg from "fluent-ffmpeg";
import { S3Actions } from "../actions/S3-actions";
import { FilesystemActions } from "../actions/file-system-actions";
import { createGenericParams } from "./storageHelper";
import { getStorageActions } from "../actions/helper-actions";
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const storageActions = getStorageActions();
const tempCreateVideoThumbnail = (
file: FileInterface,
@@ -43,7 +43,7 @@ const tempCreateVideoThumbnail = (
const readStream = storageActions.createReadStream(readStreamParams);
const writeStream = storageActions.createWriteStream(
const { writeStream, emitter } = storageActions.createWriteStream(
readStreamParams,
readStream,
thumbnailFilename
backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts
+2 -2
View File
@@ -13,9 +13,9 @@ import ffmpeg from "fluent-ffmpeg";
import { S3Actions } from "../actions/S3-actions";
import { FilesystemActions } from "../actions/file-system-actions";
import { createGenericParams } from "./storageHelper";
import { getStorageActions } from "../actions/helper-actions";
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const storageActions = getStorageActions();
const tempCreateVideoThumbnailFS = (
file: FileInterface,