From 170c93c4860296cc24a05c44ad62c9b986b495bc Mon Sep 17 00:00:00 2001 From: kyle hoell Date: Mon, 30 Nov 2020 15:41:03 -0500 Subject: [PATCH] fixed uploading/downloading s3 metadata. Added failsafe to getting user detailed for storage accounts if they cause an issue, so users can still remove them. Fixed queries when you had s3 data but then disabled it without removing metadata --- backend/db/utils/fileUtils/index.ts | 7 +-- backend/db/utils/folderUtils/index.ts | 8 ++++ backend/express-routers/userPersonal.ts | 2 +- backend/models/file.ts | 4 +- backend/models/user.ts | 16 +++++-- backend/services/UserPersonalService/index.ts | 14 +++--- backend/services/UserService/index.ts | 46 +++++++++++++------ backend/utils/createQuery.ts | 7 +-- src/components/SettingsPage/index.js | 25 ++++++---- 9 files changed, 84 insertions(+), 45 deletions(-) diff --git a/backend/db/utils/fileUtils/index.ts b/backend/db/utils/fileUtils/index.ts index 43085fa..e553651 100644 --- a/backend/db/utils/fileUtils/index.ts +++ b/backend/db/utils/fileUtils/index.ts @@ -80,9 +80,10 @@ class DbUtil { // if (s3Enabled) { // query = {...query, "metadata.personalFile": true} - // } else if (!s3Enabled){ - // query = {...query, "metadata.personalFile": null} - // } + // } else + if (!s3Enabled){ + query = {...query, "metadata.personalFile": null} + } const fileList = await conn.db.collection("fs.files") .find(query) diff --git a/backend/db/utils/folderUtils/index.ts b/backend/db/utils/folderUtils/index.ts index dc62b70..581972c 100644 --- a/backend/db/utils/folderUtils/index.ts +++ b/backend/db/utils/folderUtils/index.ts @@ -33,6 +33,10 @@ class DbUtil { // query = {...query, "personalFolder": null} // } + if (!s3Enabled) { + query = {...query, "personalFolder": null} + } + if (type) { if (type === "mongo") { query = {...query, "personalFolder": null} @@ -48,8 +52,12 @@ class DbUtil { // if (storageType === "s3") { // query = {...query, "personalFolder": true} + // } else { + // query = {...query, "personalFolder": null} // } + console.log("folder query", query); + const folderList = await Folder.find(query) .sort(sortBy) as FolderInterface[]; diff --git a/backend/express-routers/userPersonal.ts b/backend/express-routers/userPersonal.ts index f66d6b3..9d979a1 100644 --- a/backend/express-routers/userPersonal.ts +++ b/backend/express-routers/userPersonal.ts @@ -7,7 +7,7 @@ const userPersonalController = new UserPersonalController(); const router = Router(); -router.get("/user-service/download-personal-file-list/:tempToken", auth, userPersonalController.downloadPersonalFileList); +router.get("/user-service/download-personal-file-list", auth, userPersonalController.downloadPersonalFileList); router.post("/user-service/upload-personal-file-list", auth, userPersonalController.uploadPersonalFileList); diff --git a/backend/models/file.ts b/backend/models/file.ts index 060e4ad..17d3ee9 100644 --- a/backend/models/file.ts +++ b/backend/models/file.ts @@ -1,5 +1,5 @@ import mongoose, {Document} from "mongoose"; -import { Binary } from "mongodb"; +import { Binary, ObjectID } from "mongodb"; const fileSchema = new mongoose.Schema({ @@ -68,7 +68,7 @@ export interface FileInterface extends Document { filename: string, lastErrorObject: {updatedExisting: any} metadata: { - owner: string, + owner: string | ObjectID, parent: string, parentList: string, hasThumbnail: boolean, diff --git a/backend/models/user.ts b/backend/models/user.ts index 5fbca33..b58b432 100644 --- a/backend/models/user.ts +++ b/backend/models/user.ts @@ -126,14 +126,17 @@ const userSchema = new mongoose.Schema({ personalStorageCanceledDate: Number, storageData: { storageSize: Number, - storageLimit: Number + storageLimit: Number, + failed: Boolean, }, storageDataPersonal: { storageSize: Number, + failed: Boolean, }, storageDataGoogle: { storageSize: Number, - storageLimit: Number + storageLimit: Number, + failed: Boolean, }, activeSubscription: Boolean, planID: String, @@ -177,14 +180,17 @@ export interface UserInterface extends Document { }, storageData?: { storageSize?: number, - storageLimit?: number + storageLimit?: number, + failed?: boolean, }, storageDataPersonal?: { - storageSize?: number + storageSize?: number, + failed?: boolean, }, storageDataGoogle?: { storageSize?: number, - storageLimit?: number + storageLimit?: number, + failed?: boolean, }, activeSubscription?: boolean, passwordLastModified?: number, diff --git a/backend/services/UserPersonalService/index.ts b/backend/services/UserPersonalService/index.ts index bbcf4a5..9b6b6f3 100644 --- a/backend/services/UserPersonalService/index.ts +++ b/backend/services/UserPersonalService/index.ts @@ -48,7 +48,7 @@ class UserPeronsalService { downloadPersonalFileList = async(user: userAccessType) => { - const fileList = await File.find({"metadata.owner": user._id, "metadata.personalFile": true}); + const fileList = await File.find({"metadata.owner": new ObjectID(user._id), "metadata.personalFile": true}); const folderList = await Folder.find({"owner": user._id.toString(), 'personalFolder': true}); @@ -80,9 +80,9 @@ class UserPeronsalService { for (let currentObj of personalFileList) { - await File.deleteMany({_id: currentObj._id, 'metadata.owner': user._id}); + await File.deleteMany({_id: new ObjectID(currentObj._id), 'metadata.owner': new ObjectID(user._id)}); - currentObj.metadata.owner = user._id; + currentObj.metadata.owner = new ObjectID(user._id); currentObj._id = new ObjectID(currentObj._id) const oldIV: any = currentObj.metadata.IV; const IV: any = Buffer.from(oldIV, 'base64'); @@ -98,7 +98,7 @@ class UserPeronsalService { for (let currentObj of personalFolderList) { - await Folder.deleteMany({_id: currentObj._id, owner: user._id.toString()}); + await Folder.deleteMany({_id: new ObjectID(currentObj._id), owner: user._id.toString()}); currentObj._id = new ObjectID(currentObj._id) currentObj.owner = user._id.toString(); @@ -111,7 +111,7 @@ class UserPeronsalService { for (let currentObj of personalThumbnailList) { - await Thumbnail.deleteMany({_id: currentObj._id, owner: user._id.toString()}); + await Thumbnail.deleteMany({_id: new ObjectID(currentObj._id), owner: user._id.toString()}); currentObj._id = new ObjectID(currentObj._id); currentObj.owner = user._id.toString(); @@ -131,15 +131,17 @@ class UserPeronsalService { removeS3Metadata = async(user: userAccessType) => { - const fileList = await File.find({"metadata.owner": user._id, + const fileList = await File.find({"metadata.owner": new ObjectID(user._id), "metadata.personalFile": true}) for (let currentFile of fileList) { + await File.deleteOne({_id: new ObjectID(currentFile._id)}); await File.deleteOne({_id: currentFile._id}); if (currentFile.metadata.hasThumbnail) { + await Thumbnail.deleteOne({_id: new ObjectID(currentFile.metadata.thumbnailID)}) await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID}) } } diff --git a/backend/services/UserService/index.ts b/backend/services/UserService/index.ts index 3bacd4b..516fc47 100644 --- a/backend/services/UserService/index.ts +++ b/backend/services/UserService/index.ts @@ -217,28 +217,44 @@ class UserService { if (!user) throw new NotFoundError("Cannot find user"); if (user.s3Enabled) { - const {bucket} = await user.decryptS3Data() - user.s3Data!.bucket = bucket; + + try { + const {bucket} = await user.decryptS3Data() + user.s3Data!.bucket = bucket; + } catch (e) { + console.log("getting s3 storage data error"); + user.storageDataPersonal = {storageSize: 0, failed: true} + } + } if (user.googleDriveEnabled) { + + try { + + const {clientID} = await user.decryptDriveIDandKey() - const {clientID} = await user.decryptDriveIDandKey() - - const oauth2Client = await getGoogleAuth(user); - const drive = google.drive({version:"v3", auth: oauth2Client}); - - const googleData = await drive.about.get({ - fields: "storageQuota" - }) - - user.storageDataGoogle = {storageLimit: +googleData.data.storageQuota!.limit!, storageSize: +googleData.data.storageQuota!.usage!} - user.googleDriveData!.id = clientID; + const oauth2Client = await getGoogleAuth(user); + const drive = google.drive({version:"v3", auth: oauth2Client}); + + const googleData = await drive.about.get({ + fields: "storageQuota" + }) + + user.storageDataGoogle = {storageLimit: +googleData.data.storageQuota!.limit!, storageSize: +googleData.data.storageQuota!.usage!} + user.googleDriveData!.id = clientID; + + } catch (e) { + + user.storageDataGoogle = {storageLimit: 0, storageSize: 0, failed: true} + console.log("get google drive storage data error", e.message); + } + } if (!user.storageData || (!user.storageData.storageSize && !user.storageData.storageLimit)) user.storageData = {storageLimit: 0, storageSize: 0} - if (!user.storageDataPersonal || (!user.storageDataPersonal.storageSize)) user.storageDataPersonal = {storageSize: 0} - if (!user.storageDataGoogle || (!user.storageDataGoogle.storageLimit && !user.storageDataGoogle.storageSize)) user.storageDataGoogle = {storageLimit: 0, storageSize: 0} + if (!user.storageDataPersonal || (!user.storageDataPersonal.storageSize && !user.storageDataPersonal.failed)) user.storageDataPersonal = {storageSize: 0} + if (!user.storageDataGoogle || (!user.storageDataGoogle.storageLimit && !user.storageDataGoogle.storageSize && !user.storageDataGoogle.failed)) user.storageDataGoogle = {storageLimit: 0, storageSize: 0} delete user.privateKey; delete user.publicKey; diff --git a/backend/utils/createQuery.ts b/backend/utils/createQuery.ts index 1c96019..0592842 100644 --- a/backend/utils/createQuery.ts +++ b/backend/utils/createQuery.ts @@ -55,9 +55,10 @@ const createQuery = (owner: string, parent: string, sortBy: string, startAt: num // if (s3Enabled) { // query = {...query, "metadata.personalFile": true} - // } else if (!s3Enabled) { - // query = {...query, "metadata.personalFile": null} - // } + // } else + if (!s3Enabled) { + query = {...query, "metadata.personalFile": null} + } // if (storageType === "s3") { // query = {...query, "metadata.personalFile": true} diff --git a/src/components/SettingsPage/index.js b/src/components/SettingsPage/index.js index a786de0..e4447dd 100644 --- a/src/components/SettingsPage/index.js +++ b/src/components/SettingsPage/index.js @@ -70,6 +70,8 @@ class SettingsPageContainer extends React.Component { env.emailAddress = response.data.email this.props.dispatch(setParent(uuid.v4())) + console.log("user detailed", response.data) + this.setState(() => { return { ...this.state, @@ -78,7 +80,7 @@ class SettingsPageContainer extends React.Component { } }, () => { - this.getInvoiceData(); + // this.getInvoiceData(); this.props.dispatch(setLoading(false)) }) @@ -640,11 +642,9 @@ class SettingsPageContainer extends React.Component { e.preventDefault() - axios.get('/file-service/download/get-token').then((response) => { + axios.post('/user-service/get-token').then((response) => { - const tempToken = response.data.tempToken; - - const finalUrl = `/user-service/download-personal-file-list/${tempToken}`; + const finalUrl = `/user-service/download-personal-file-list`; const link = document.createElement('a'); document.body.appendChild(link); @@ -831,6 +831,11 @@ class SettingsPageContainer extends React.Component { } render() { + + if (this.props.loading) { + return + } + return (
@@ -841,8 +846,6 @@ class SettingsPageContainer extends React.Component {
- -
@@ -1116,8 +1119,9 @@ class SettingsPageContainer extends React.Component { style={{backgroundColor:"#3c85ee", width: !this.state.loaded ? "0%" : this.state.userDetails.googleDriveEnabled ? `${this.getStoragePercentageGoogle()}%` : '0%'}} >
- {!this.state.loaded ? "Loading..." : this.state.userDetails.googleDriveEnabled ? `${bytes(this.state.userDetails.storageDataGoogle.storageSize)} of - ${bytes(this.state.userDetails.storageDataGoogle.storageLimit)} used` : ""} + {this.state.userDetails.googleDriveEnabled ? !this.state.userDetails.storageDataGoogle.failed ? `${bytes(this.state.userDetails.storageDataGoogle.storageSize)} of + ${bytes(this.state.userDetails.storageDataGoogle.storageLimit)} used` : "Failed" : ""} +
@@ -1140,7 +1144,8 @@ class SettingsPageContainer extends React.Component { style={{backgroundColor:"#3c85ee", width:"100%"}} >
- {!this.state.loaded ? "Loading..." : this.state.userDetails.s3Enabled ? `${bytes(this.state.userDetails.storageDataPersonal.storageSize)} used` : ""} + {this.state.userDetails.s3Enabled ? !this.state.userDetails.storageDataPersonal.failed ? + `${bytes(this.state.userDetails.storageDataPersonal.storageSize)} used` : "Failed" : ""}