oxmc/myDrive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

started testing S3, most things for it are now working. Also fixed media list issue, added more migration scrips for old versions of mydrive

Browse Source
This commit is contained in:
subnub
2024-10-13 17:29:41 -04:00
parent 8d915d4477
commit 0ec24235cc
16 changed files with 288 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/controllers/file-controller.ts
+15 -18
View File
@@ -141,7 +141,8 @@ class FileController {
const handleError = () => {
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error uploading file");
res.writeHead(500, { Connection: "close" });
res.end();
}
};
@@ -215,7 +216,7 @@ class FileController {
req.pipe(busboy);
const { cipher, fileWriteStream, metadata, filename } =
const { cipher, fileWriteStream, emitter, metadata, filename } =
await this.chunkService.uploadFile(user, busboy, req);
cipher.on("error", (e: Error) => {
@@ -228,9 +229,17 @@ class FileController {
handleError();
});
cipher.pipe(fileWriteStream).on("finish", async () => {
handleFinish(filename, metadata);
});
if (emitter) {
emitter.on("finish", async () => {
await handleFinish(filename, metadata);
});
} else {
fileWriteStream.on("finish", async () => {
await handleFinish(filename, metadata);
});
}
cipher.pipe(fileWriteStream);
} catch (e: unknown) {
if (!responseSent) {
res.writeHead(500, { Connection: "close" });
@@ -616,18 +625,10 @@ class FileController {
readStream.on("error", (e: Error) => {
console.log("read stream error", e);
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error downloading file");
}
});
decipher.on("error", (e: Error) => {
console.log("decipher stream error", e);
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error downloading file");
}
console.log("decipher stream errozr", e);
});
res.set("Content-Type", "binary/octet-stream");
@@ -642,10 +643,6 @@ class FileController {
if (e instanceof Error) {
console.log("\nDownload File Error File Route:", e.message);
}
if (!responseSent) {
responseSent = true;
res.status(500).send("Server error downloading file");
}
}
};
backend/services/chunk-service/actions/S3-actions.ts
+14 -11
View File
@@ -1,8 +1,9 @@
import s3 from "../../../db/connections/s3";
import env from "../../../enviroment/env";
import { GenericParams, IStorageActions } from "../store-types";
import internal from "stream";
import internal, { EventEmitter } from "stream";
import { PassThrough } from "stream";
import stream from "stream";
class S3Actions implements IStorageActions {
getAuth() {
@@ -65,23 +66,25 @@ class S3Actions implements IStorageActions {
}
createWriteStream = (
params: GenericParams,
stream: NodeJS.ReadableStream,
readStream: NodeJS.ReadableStream,
randomID: string
) => {
const pass = new PassThrough();
const passThrough = new stream.PassThrough();
const emitter = new EventEmitter();
const { Key } = params;
if (!Key) throw new Error("S3 not configured");
const { s3Storage, bucket } = this.getAuth();
s3Storage.upload({ Bucket: bucket, Body: stream, Key: randomID }, (err) => {
if (err) {
console.log("S3 upload err", err);
pass.emit("error", err);
s3Storage.upload(
{ Bucket: bucket, Body: passThrough, Key: randomID },
(err) => {
if (err) {
console.log("S3 upload err", err);
}
emitter.emit("finish");
}
});
);
return pass;
return { writeStream: passThrough, emitter };
};
}
backend/services/chunk-service/actions/file-system-actions.ts
+4 -1
View File
@@ -61,7 +61,10 @@ class FilesystemActions implements IStorageActions {
randomID: string
) => {
const path = `${env.fsDirectory}${randomID}`;
return fs.createWriteStream(path);
return {
writeStream: fs.createWriteStream(path),
emitter: null,
};
};
}
backend/services/chunk-service/chunk-service.ts
+8 -3
View File
@@ -19,6 +19,7 @@ import { createGenericParams } from "./utils/storageHelper";
import { Readable } from "stream";
import ThumbnailDB from "../../db/mongoDB/thumbnailDB";
import UserDB from "../../db/mongoDB/userDB";
import fixEndChunkLength from "./utils/fixEndChunkLength";
const fileDB = new FileDB();
const folderDB = new FolderDB();
@@ -111,7 +112,7 @@ class StorageService {
metadata.s3ID = randomFilenameID;
}
const fileWriteStream = storageActions.createWriteStream(
const { writeStream, emitter } = storageActions.createWriteStream(
metadata,
file.pipe(cipher),
randomFilenameID
@@ -119,7 +120,8 @@ class StorageService {
return {
cipher,
fileWriteStream,
fileWriteStream: writeStream,
emitter,
metadata,
filename,
};
@@ -245,7 +247,10 @@ class StorageService {
let currentIV = IV;
let fixedStart = 0;
let fixedEnd = currentFile.length;
let fixedEnd =
currentFile.length % 16 === 0
? currentFile.length
: fixEndChunkLength(currentFile.length);
if (start === 0 && end === 1) {
fixedStart = 0;
backend/services/chunk-service/utils/createImageThumbnail.ts
+27 -6
View File
@@ -4,7 +4,7 @@ import Thumbnail from "../../../models/thumbnail-model";
import sharp from "sharp";
import { FileInterface } from "../../../models/file-model";
import { UserInterface } from "../../../models/user-model";
import fs from "fs";
import fs, { write } from "fs";
import uuid from "uuid";
import env from "../../../enviroment/env";
import { ObjectId } from "mongodb";
@@ -18,12 +18,20 @@ const conn = mongoose.connection;
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const waitTimer = () => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(null);
}, 5000);
});
};
const createThumbnail = (
file: FileInterface,
filename: string,
user: UserInterface
) => {
return new Promise<FileInterface>((resolve, reject) => {
return new Promise<FileInterface>(async (resolve, reject) => {
try {
const password = user.getEncryptionKey();
@@ -33,6 +41,12 @@ const createThumbnail = (
const thumbnailIV = crypto.randomBytes(16);
console.log("file metadata", file.metadata);
// if (env.dbType === "s3") {
// await waitTimer();
// }
const params = createGenericParams({
filePath: file.metadata.filePath,
Key: file.metadata.s3ID,
@@ -40,7 +54,7 @@ const createThumbnail = (
const readStream = storageActions.createReadStream(params);
const writeStream = storageActions.createWriteStream(
const { writeStream, emitter } = storageActions.createWriteStream(
params,
readStream,
thumbnailFilename
@@ -61,6 +75,7 @@ const createThumbnail = (
owner: user._id,
IV: thumbnailIV,
path: env.fsDirectory + thumbnailFilename,
s3ID: thumbnailFilename,
});
await thumbnailModel.save();
@@ -119,9 +134,15 @@ const createThumbnail = (
.pipe(thumbnailCipher)
.pipe(writeStream);
writeStream.on("finish", async () => {
await handleFinish();
});
if (emitter) {
emitter.on("finish", async () => {
await handleFinish();
});
} else {
writeStream.on("finish", async () => {
await handleFinish();
});
}
} catch (e) {
console.log("File service upload thumbnail error", e);
resolve(file);
backend/services/chunk-service/utils/createVideoThumbnail.ts
+4 -4
View File
@@ -38,7 +38,7 @@ const createVideoThumbnail = (
const readStream = storageActions.createReadStream(readStreamParams);
const writeStream = storageActions.createWriteStream(
const { writeStream } = storageActions.createWriteStream(
readStreamParams,
readStream,
thumbnailFilename
@@ -80,6 +80,7 @@ const createVideoThumbnail = (
owner: user._id,
IV: thumbnailIV,
path: env.fsDirectory + thumbnailFilename,
s3ID: thumbnailFilename,
});
await thumbnailModel.save();
@@ -106,8 +107,6 @@ const createVideoThumbnail = (
if (!updatedFile) return reject();
console.log("updated file", updatedFile, resolve);
resolve(updatedFile?.toObject());
} catch (e) {
console.log("thumbnail error", e);
@@ -129,7 +128,7 @@ const createVideoThumbnail = (
.on("end", async () => {
await handleFinish();
})
.on("error", async (err, _, stderr) => {
.on("error", async (err, sdf, stderr) => {
console.log("thumbnail error attempting temp directory fix");
if (env.tempDirectory) {
const updatedFile = await tempCreateVideoThumbnailFS(
@@ -141,6 +140,7 @@ const createVideoThumbnail = (
} else {
resolve(file);
}
// resolve(file);
})
.pipe(thumbnailCipher)
.pipe(writeStream, { end: true });
backend/services/chunk-service/utils/tempCreateVideoThumbnailFS.ts
+109 -94
View File
@@ -10,6 +10,12 @@ import env from "../../../enviroment/env";
import { ObjectId } from "mongodb";
import File from "../../../models/file-model";
import ffmpeg from "fluent-ffmpeg";
import { S3Actions } from "../actions/S3-actions";
import { FilesystemActions } from "../actions/file-system-actions";
import { createGenericParams } from "./storageHelper";
const storageActions =
env.dbType === "s3" ? new S3Actions() : new FilesystemActions();
const tempCreateVideoThumbnailFS = (
file: FileInterface,
@@ -23,13 +29,20 @@ const tempCreateVideoThumbnailFS = (
const thumbnailFilename = uuid.v4();
const readStream = fs.createReadStream(file.metadata.filePath!);
console.log("env", env);
const writeStream = fs.createWriteStream(
env.fsDirectory + thumbnailFilename
const readStreamParams = createGenericParams({
filePath: file.metadata.filePath,
Key: file.metadata.s3ID,
});
const readStream = storageActions.createReadStream(readStreamParams);
const { writeStream } = storageActions.createWriteStream(
readStreamParams,
readStream,
thumbnailFilename
);
const tempDirectory = env.fsDirectory + "temp/" + thumbnailFilename;
const tempDirectory = env.tempDirectory + thumbnailFilename;
const tempWriteStream = fs.createWriteStream(tempDirectory);
const decipher = crypto.createDecipheriv(
"aes256",
@@ -49,105 +62,107 @@ const tempCreateVideoThumbnailFS = (
decryptedReadStream.pipe(tempWriteStream, { end: true });
ffmpeg(tempDirectory, {
timeout: 60,
})
.seek(1)
.format("image2pipe")
.outputOptions([
"-f image2pipe",
"-vframes 1",
"-vf scale='if(gt(iw,ih),600,-1):if(gt(ih,iw),300,-1)'",
])
.on("start", (command) => {
/**
* log
*/
tempWriteStream.on("finish", () => {
ffmpeg(tempDirectory, {
timeout: 60,
})
.on("end", async () => {
console.log("end");
const thumbnailModel = new Thumbnail({
name: filename,
owner: user._id,
IV: thumbnailIV,
path: env.fsDirectory + thumbnailFilename,
});
.seek(1)
.format("image2pipe")
.outputOptions([
"-f image2pipe",
"-vframes 1",
"-vf scale='if(gt(iw,ih),600,-1):if(gt(ih,iw),300,-1)'",
])
.on("start", (command) => {
/**
* log
*/
})
.on("end", async () => {
const thumbnailModel = new Thumbnail({
name: filename,
owner: user._id,
IV: thumbnailIV,
path: env.fsDirectory + thumbnailFilename,
s3ID: thumbnailFilename,
});
await thumbnailModel.save();
if (!file._id) {
return reject();
}
const updateFileResponse = await File.updateOne(
{ _id: new ObjectId(file._id), "metadata.owner": user._id },
{
$set: {
"metadata.hasThumbnail": true,
"metadata.thumbnailID": thumbnailModel._id,
},
await thumbnailModel.save();
if (!file._id) {
return reject();
}
const updateFileResponse = await File.updateOne(
{ _id: new ObjectId(file._id), "metadata.owner": user._id },
{
$set: {
"metadata.hasThumbnail": true,
"metadata.thumbnailID": thumbnailModel._id,
},
}
);
if (updateFileResponse.modifiedCount === 0) {
return reject();
}
);
if (updateFileResponse.modifiedCount === 0) {
return reject();
}
const updatedFile = await File.findById({
_id: new ObjectId(file._id),
"metadata.owner": user._id,
});
const updatedFile = await File.findById({
_id: new ObjectId(file._id),
"metadata.owner": user._id,
});
if (!updatedFile) return reject();
if (!updatedFile) return reject();
fs.unlink(tempDirectory, (err) => {
resolve(updatedFile?.toObject());
});
})
.on("error", (err, _, stderr) => {
console.log("error", err, stderr);
resolve(file);
/**
* log
*/
})
.pipe(thumbnailCipher)
.pipe(writeStream, { end: true });
fs.unlink(tempDirectory, (err) => {
resolve(updatedFile?.toObject());
});
})
.on("error", (err, _, stderr) => {
console.log("error", err, stderr);
resolve(file);
/**
* log
*/
})
.pipe(thumbnailCipher)
.pipe(writeStream, { end: true });
// ffmpeg()
// .input(readStream.pipe(decipher))
// .seekInput("00:00:01.00")
// .outputFormat("image2")
// .pipe(writeStream, { end: true })
// .on("end", async () => {
// const thumbnailModel = new Thumbnail({
// name: filename,
// owner: user._id,
// IV: thumbnailIV,
// path: env.fsDirectory + thumbnailFilename,
// });
// ffmpeg()
// .input(readStream.pipe(decipher))
// .seekInput("00:00:01.00")
// .outputFormat("image2")
// .pipe(writeStream, { end: true })
// .on("end", async () => {
// const thumbnailModel = new Thumbnail({
// name: filename,
// owner: user._id,
// IV: thumbnailIV,
// path: env.fsDirectory + thumbnailFilename,
// });
// await thumbnailModel.save();
// resolve(file);
// })
// .on("error", (e) => {
// console.log("error", e);
// reject(e);
// });
// await thumbnailModel.save();
// resolve(file);
// })
// .on("error", (e) => {
// console.log("error", e);
// reject(e);
// });
// ffmpeg()
// .input(readStream.pipe(decipher))
// .size("300x?")
// .outputOptions("-frames:v 1")
// .on("end", async () => {
// const thumbnailModel = new Thumbnail({
// name: filename,
// owner: user._id,
// IV: thumbnailIV,
// path: env.fsDirectory + thumbnailFilename,
// });
// ffmpeg()
// .input(readStream.pipe(decipher))
// .size("300x?")
// .outputOptions("-frames:v 1")
// .on("end", async () => {
// const thumbnailModel = new Thumbnail({
// name: filename,
// owner: user._id,
// IV: thumbnailIV,
// path: env.fsDirectory + thumbnailFilename,
// });
// await thumbnailModel.save();
// resolve(file);
// })
// .output(writeStream);
// await thumbnailModel.save();
// resolve(file);
// })
// .output(writeStream);
});
});
};
backend/utils/createQuery.ts
+2 -1
View File
@@ -35,7 +35,7 @@ export const createFileQuery = ({
if (search && search !== "") {
query["filename"] = new RegExp(search, "i");
} else {
} else if (!mediaMode) {
query["metadata.parent"] = parent;
}
@@ -58,6 +58,7 @@ export const createFileQuery = ({
if (mediaMode) {
query["metadata.hasThumbnail"] = true;
}
return query;
};
serverUtils/migrateMyDrive4.js
+47
View File
@@ -0,0 +1,47 @@
const getEnvVariables = require("../dist/enviroment/get-env-variables");
getEnvVariables();
const mongoose = require("./mongoServerUtil");
const conn = mongoose.connection;
const File = require("../dist/models/file-model");
const waitForDatabase = () => {
return new Promise((resolve, reject) => {
if (conn.readyState !== 1) {
conn.once("open", () => {
resolve();
});
} else {
resolve();
}
});
};
const updateDocs = async () => {
console.log("\nWaiting for database...");
await waitForDatabase();
console.log("Connected to database\n");
console.log("Getting file list...");
const files = await File.find({});
console.log("Found", files.length, "files");
for (let i = 0; i < files.length; i++) {
const currentFile = files[i];
await File.updateOne(
{ _id: currentFile._id },
{
$set: {
"metadata.owner": currentFile.metadata.owner.toString(),
"metadata.thumbnailID": currentFile.metadata.thumbnailID.toString(),
},
}
);
}
console.log("Done");
process.exit();
};
updateDocs();
serverUtils/mongoServerUtil.js
+2 -8
View File
@@ -1,11 +1,5 @@
const mongoose = require("mongoose");
mongoose.connect(process.env.MONGODB_URL, {
useNewUrlParser: true,
useCreateIndex: true,
useUnifiedTopology: true,
socketTimeoutMS: 30000000,
keepAlive: true,
})
mongoose.connect(process.env.MONGODB_URL, {});
module.exports = mongoose;
module.exports = mongoose;
src/components/Folders/Folders.tsx
+1 -1
View File
@@ -67,7 +67,7 @@ const Folders = memo(() => {
return (
<div className="mt-8">
<div className="flex flex-row mb-[20px] justify-between text-[#212b36] items-center">
<div className="flex flex-row mb-5 justify-between items-center">
<h2
className={classNames(
"m-0 text-xl font-medium",
src/components/Header/Header.tsx
+30 -10
View File
@@ -2,9 +2,11 @@ import { useNavigate } from "react-router-dom";
import SearchBar from "../SearchBar/SearchBar";
import MenuIcon from "../../icons/MenuIcon";
import { useAppDispatch, useAppSelector } from "../../hooks/store";
import { useCallback, useMemo } from "react";
import { toggleDrawer } from "../../reducers/leftSection";
import { useCallback } from "react";
import { closeDrawer, toggleDrawer } from "../../reducers/leftSection";
import { useUtils } from "../../hooks/utils";
import ChevronOutline from "../../icons/ChevronOutline";
import SettingsIconSolid from "../../icons/SettingsIconSolid";
const Header = () => {
const drawerOpen = useAppSelector((state) => state.leftSection.drawOpen);
@@ -19,25 +21,43 @@ const Header = () => {
},
[toggleDrawer]
);
const closeDrawerClick = useCallback(
(e: any) => {
e.stopPropagation();
dispatch(closeDrawer());
},
[closeDrawer]
);
return (
<header id="header">
<div className="px-6 flex justify-between min-h-[68px] items-center py-[15px]">
<div className="px-6 flex justify-between min-h-16 items-center py-3.5">
<div className="items-center w-[260px] hidden desktopMode:flex">
<a
className="inline-flex items-center justify-center cursor-pointer"
onClick={() => navigate("/")}
>
<img className="w-[35px]" src="/images/icon.png" alt="logo" />
<img className="w-9" src="/images/icon.png" alt="logo" />
</a>
</div>
{!isSettings && (
<div className="items-center flex desktopMode:hidden mr-4">
<a className="inline-flex items-center justify-center cursor-pointer">
<MenuIcon
id="menu-icon"
onClick={toggleDrawerClick}
className="text-[#3c85ee] w-[35px]"
/>
{!drawerOpen && (
<MenuIcon
id="menu-icon"
onClick={toggleDrawerClick}
className="text-primary w-9"
/>
)}
{drawerOpen && (
<ChevronOutline
id="menu-icon"
onClick={closeDrawerClick}
className="text-primary w-9 rotate-90"
/>
)}
</a>
</div>
)}
@@ -49,7 +69,7 @@ const Header = () => {
onClick={() => navigate("/settings")}
className="cursor-pointer"
>
<img src="/assets/settings.svg" alt="settings" />
<SettingsIconSolid className="w-7 h-7 text-gray-primary" />
</a>
</div>
</div>
src/components/HocComponent/index.jsx
-11
View File
@@ -1,11 +0,0 @@
import { useLocation, useNavigate } from "react-router-dom";
const withNavigate = (Component) => {
return (props) => {
const navigate = useNavigate();
const location = useLocation();
return <Component {...props} navigate={navigate} location={location} />;
};
};
export default withNavigate;
src/components/HomepageSpinner/HomepageSpinner.jsx
-25
View File
@@ -1,25 +0,0 @@
import React from "react";
import {connect} from "react-redux";
class HomepageSpinnerContainer extends React.Component {
constructor(props) {
super(props)
}
render() {
return (
<div className="homepage__spinner__wrapper" style={this.props.loading ? {} : {display:"none"}}>
<div className="spinner spinner__no-margin">
</div>
</div>
)
}
}
const connectPropToStore = (state) => ({
loading: state.main.loading
})
export default connect(connectPropToStore)(HomepageSpinnerContainer);
src/components/QuickAccess/QuickAccess.tsx
+1 -1
View File
@@ -17,7 +17,7 @@ const QuickAccess = memo(() => {
className="overflow-hidden"
style={isHome ? { display: "block" } : { display: "none" }}
>
<div className="flex flex-row items-center justify-between mb-4">
<div className="flex flex-row items-center justify-between mb-5">
<h2 className=" text-[#212b36] text-xl font-medium">Quick Access</h2>
<ChevronOutline
onClick={() => setQuickAccessExpanded(!quickAccessExpanded)}
src/icons/SettingsIconSolid.tsx
+24
View File
@@ -0,0 +1,24 @@
type SettingsIconSolidType = React.SVGAttributes<SVGSVGElement>;
const SettingsIconSolid: React.FC<SettingsIconSolidType> = (props) => {
return (
<svg
width="27"
height="28"
viewBox="0 0 27 28"
fill="none"
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<path
id="Shape"
fill-rule="evenodd"
clip-rule="evenodd"
d="M23.9831 15.0558L26.5315 17.0683C27.0081 17.4528 27.1391 18.1291 26.8411 18.6667L24.1794 23.3217C23.8787 23.8621 23.2292 24.0957 22.6579 23.8688L19.6427 22.645C19.0776 23.0565 18.474 23.4113 17.8405 23.7043L17.3785 26.9359C17.2922 27.5501 16.7692 28.0049 16.1551 27.9999H10.8409C10.2246 28.0024 9.70233 27.5424 9.62099 26.9254L9.15891 23.7054C8.52317 23.4148 7.91818 23.0599 7.35326 22.6461L4.34961 23.8664C3.77234 24.0877 3.1216 23.8509 2.81661 23.3088L0.166459 18.6795C-0.138039 18.141 -0.00969806 17.4589 0.469115 17.0706L3.0199 15.0581C2.94013 14.3558 2.94013 13.6465 3.0199 12.9442L0.471444 10.9317C-0.0138678 10.5443 -0.142479 9.85504 0.169925 9.31583L2.82354 4.6795C3.12393 4.13864 3.7737 3.90491 4.345 4.13236L7.35678 5.35505C7.92181 4.94337 8.52544 4.58856 9.15897 4.29571L9.62105 1.06408C9.70661 0.451033 10.2279 -0.00363585 10.841 8.28791e-05H16.1551C16.7733 -0.00500303 17.2989 0.454807 17.382 1.07343L17.8429 4.29576C18.4784 4.58582 19.083 4.94036 19.6474 5.3539L22.6511 4.13356C23.2284 3.9129 23.8787 4.14947 24.1841 4.6912L26.8354 9.31933C27.1396 9.85767 27.0119 10.5394 26.5339 10.9282L23.9831 12.9418C24.0621 13.6442 24.0621 14.3534 23.9831 15.0558ZM8 13.95C8 17.0214 10.4625 19.5111 13.5 19.5111C16.536 19.5075 18.9963 17.0198 19 13.95C19 10.8787 16.5376 8.38893 13.5 8.38893C10.4624 8.38893 8 10.8787 8 13.95Z"
fill="currentColor"
/>
</svg>
);
};
export default SettingsIconSolid;