oxmc/myDrive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed uploading/downloading s3 metadata. Added failsafe to getting user detailed for storage accounts if they cause an issue, so users can still remove them. Fixed queries when you had s3 data but then disabled it without removing metadata

Browse Source
This commit is contained in:
kyle hoell
2020-11-30 15:41:03 -05:00
parent 9ef9f0d742
commit 170c93c486
9 changed files with 84 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/db/utils/fileUtils/index.ts
+4 -3
View File
@@ -80,9 +80,10 @@ class DbUtil {
// if (s3Enabled) {
// query = {...query, "metadata.personalFile": true}
// } else if (!s3Enabled){
// query = {...query, "metadata.personalFile": null}
// }
// } else
if (!s3Enabled){
query = {...query, "metadata.personalFile": null}
}
const fileList = await conn.db.collection("fs.files")
.find(query)
backend/db/utils/folderUtils/index.ts
+8
View File
@@ -33,6 +33,10 @@ class DbUtil {
// query = {...query, "personalFolder": null}
// }
if (!s3Enabled) {
query = {...query, "personalFolder": null}
}
if (type) {
if (type === "mongo") {
query = {...query, "personalFolder": null}
@@ -48,8 +52,12 @@ class DbUtil {
// if (storageType === "s3") {
// query = {...query, "personalFolder": true}
// } else {
// query = {...query, "personalFolder": null}
// }
console.log("folder query", query);
const folderList = await Folder.find(query)
.sort(sortBy) as FolderInterface[];
backend/express-routers/userPersonal.ts
+1 -1
View File
@@ -7,7 +7,7 @@ const userPersonalController = new UserPersonalController();
const router = Router();
router.get("/user-service/download-personal-file-list/:tempToken", auth, userPersonalController.downloadPersonalFileList);
router.get("/user-service/download-personal-file-list", auth, userPersonalController.downloadPersonalFileList);
router.post("/user-service/upload-personal-file-list", auth, userPersonalController.uploadPersonalFileList);
backend/models/file.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import mongoose, {Document} from "mongoose";
import { Binary } from "mongodb";
import { Binary, ObjectID } from "mongodb";
const fileSchema = new mongoose.Schema({
@@ -68,7 +68,7 @@ export interface FileInterface extends Document {
filename: string,
lastErrorObject: {updatedExisting: any}
metadata: {
owner: string,
owner: string | ObjectID,
parent: string,
parentList: string,
hasThumbnail: boolean,
backend/models/user.ts
+11 -5
View File
@@ -126,14 +126,17 @@ const userSchema = new mongoose.Schema({
personalStorageCanceledDate: Number,
storageData: {
storageSize: Number,
storageLimit: Number
storageLimit: Number,
failed: Boolean,
},
storageDataPersonal: {
storageSize: Number,
failed: Boolean,
},
storageDataGoogle: {
storageSize: Number,
storageLimit: Number
storageLimit: Number,
failed: Boolean,
},
activeSubscription: Boolean,
planID: String,
@@ -177,14 +180,17 @@ export interface UserInterface extends Document {
},
storageData?: {
storageSize?: number,
storageLimit?: number
storageLimit?: number,
failed?: boolean,
},
storageDataPersonal?: {
storageSize?: number
storageSize?: number,
failed?: boolean,
},
storageDataGoogle?: {
storageSize?: number,
storageLimit?: number
storageLimit?: number,
failed?: boolean,
},
activeSubscription?: boolean,
passwordLastModified?: number,
backend/services/UserPersonalService/index.ts
+8 -6
View File
@@ -48,7 +48,7 @@ class UserPeronsalService {
downloadPersonalFileList = async(user: userAccessType) => {
const fileList = await File.find({"metadata.owner": user._id, "metadata.personalFile": true});
const fileList = await File.find({"metadata.owner": new ObjectID(user._id), "metadata.personalFile": true});
const folderList = await Folder.find({"owner": user._id.toString(), 'personalFolder': true});
@@ -80,9 +80,9 @@ class UserPeronsalService {
for (let currentObj of personalFileList) {
await File.deleteMany({_id: currentObj._id, 'metadata.owner': user._id});
await File.deleteMany({_id: new ObjectID(currentObj._id), 'metadata.owner': new ObjectID(user._id)});
currentObj.metadata.owner = user._id;
currentObj.metadata.owner = new ObjectID(user._id);
currentObj._id = new ObjectID(currentObj._id)
const oldIV: any = currentObj.metadata.IV;
const IV: any = Buffer.from(oldIV, 'base64');
@@ -98,7 +98,7 @@ class UserPeronsalService {
for (let currentObj of personalFolderList) {
await Folder.deleteMany({_id: currentObj._id, owner: user._id.toString()});
await Folder.deleteMany({_id: new ObjectID(currentObj._id), owner: user._id.toString()});
currentObj._id = new ObjectID(currentObj._id)
currentObj.owner = user._id.toString();
@@ -111,7 +111,7 @@ class UserPeronsalService {
for (let currentObj of personalThumbnailList) {
await Thumbnail.deleteMany({_id: currentObj._id, owner: user._id.toString()});
await Thumbnail.deleteMany({_id: new ObjectID(currentObj._id), owner: user._id.toString()});
currentObj._id = new ObjectID(currentObj._id);
currentObj.owner = user._id.toString();
@@ -131,15 +131,17 @@ class UserPeronsalService {
removeS3Metadata = async(user: userAccessType) => {
const fileList = await File.find({"metadata.owner": user._id,
const fileList = await File.find({"metadata.owner": new ObjectID(user._id),
"metadata.personalFile": true})
for (let currentFile of fileList) {
await File.deleteOne({_id: new ObjectID(currentFile._id)});
await File.deleteOne({_id: currentFile._id});
if (currentFile.metadata.hasThumbnail) {
await Thumbnail.deleteOne({_id: new ObjectID(currentFile.metadata.thumbnailID)})
await Thumbnail.deleteOne({_id: currentFile.metadata.thumbnailID})
}
}
backend/services/UserService/index.ts
+31 -15
View File
@@ -217,28 +217,44 @@ class UserService {
if (!user) throw new NotFoundError("Cannot find user");
if (user.s3Enabled) {
const {bucket} = await user.decryptS3Data()
user.s3Data!.bucket = bucket;
try {
const {bucket} = await user.decryptS3Data()
user.s3Data!.bucket = bucket;
} catch (e) {
console.log("getting s3 storage data error");
user.storageDataPersonal = {storageSize: 0, failed: true}
}
}
if (user.googleDriveEnabled) {
try {
const {clientID} = await user.decryptDriveIDandKey()
const {clientID} = await user.decryptDriveIDandKey()
const oauth2Client = await getGoogleAuth(user);
const drive = google.drive({version:"v3", auth: oauth2Client});
const googleData = await drive.about.get({
fields: "storageQuota"
})
user.storageDataGoogle = {storageLimit: +googleData.data.storageQuota!.limit!, storageSize: +googleData.data.storageQuota!.usage!}
user.googleDriveData!.id = clientID;
const oauth2Client = await getGoogleAuth(user);
const drive = google.drive({version:"v3", auth: oauth2Client});
const googleData = await drive.about.get({
fields: "storageQuota"
})
user.storageDataGoogle = {storageLimit: +googleData.data.storageQuota!.limit!, storageSize: +googleData.data.storageQuota!.usage!}
user.googleDriveData!.id = clientID;
} catch (e) {
user.storageDataGoogle = {storageLimit: 0, storageSize: 0, failed: true}
console.log("get google drive storage data error", e.message);
}
}
if (!user.storageData || (!user.storageData.storageSize && !user.storageData.storageLimit)) user.storageData = {storageLimit: 0, storageSize: 0}
if (!user.storageDataPersonal || (!user.storageDataPersonal.storageSize)) user.storageDataPersonal = {storageSize: 0}
if (!user.storageDataGoogle || (!user.storageDataGoogle.storageLimit && !user.storageDataGoogle.storageSize)) user.storageDataGoogle = {storageLimit: 0, storageSize: 0}
if (!user.storageDataPersonal || (!user.storageDataPersonal.storageSize && !user.storageDataPersonal.failed)) user.storageDataPersonal = {storageSize: 0}
if (!user.storageDataGoogle || (!user.storageDataGoogle.storageLimit && !user.storageDataGoogle.storageSize && !user.storageDataGoogle.failed)) user.storageDataGoogle = {storageLimit: 0, storageSize: 0}
delete user.privateKey;
delete user.publicKey;
backend/utils/createQuery.ts
+4 -3
View File
@@ -55,9 +55,10 @@ const createQuery = (owner: string, parent: string, sortBy: string, startAt: num
// if (s3Enabled) {
// query = {...query, "metadata.personalFile": true}
// } else if (!s3Enabled) {
// query = {...query, "metadata.personalFile": null}
// }
// } else
if (!s3Enabled) {
query = {...query, "metadata.personalFile": null}
}
// if (storageType === "s3") {
// query = {...query, "metadata.personalFile": true}
src/components/SettingsPage/index.js
+15 -10
View File
@@ -70,6 +70,8 @@ class SettingsPageContainer extends React.Component {
env.emailAddress = response.data.email
this.props.dispatch(setParent(uuid.v4()))
console.log("user detailed", response.data)
this.setState(() => {
return {
...this.state,
@@ -78,7 +80,7 @@ class SettingsPageContainer extends React.Component {
}
}, () => {
this.getInvoiceData();
// this.getInvoiceData();
this.props.dispatch(setLoading(false))
})
@@ -640,11 +642,9 @@ class SettingsPageContainer extends React.Component {
e.preventDefault()
axios.get('/file-service/download/get-token').then((response) => {
axios.post('/user-service/get-token').then((response) => {
const tempToken = response.data.tempToken;
const finalUrl = `/user-service/download-personal-file-list/${tempToken}`;
const finalUrl = `/user-service/download-personal-file-list`;
const link = document.createElement('a');
document.body.appendChild(link);
@@ -831,6 +831,11 @@ class SettingsPageContainer extends React.Component {
}
render() {
if (this.props.loading) {
return <HomepageSpinner />
}
return (
<div>
@@ -841,8 +846,6 @@ class SettingsPageContainer extends React.Component {
<Header goHome={this.goHome}/>
<HomepageSpinner />
<div class="main__wrapper--container settings__container" style={this.props.loading ? {display: "none"} : {}}>
<div class="menu__block" style={this.state.sideBarOpen ? {left: "0px"} : {left:"-250px"}}>
<div class="file__settings">
@@ -1116,8 +1119,9 @@ class SettingsPageContainer extends React.Component {
style={{backgroundColor:"#3c85ee", width: !this.state.loaded ? "0%" : this.state.userDetails.googleDriveEnabled ? `${this.getStoragePercentageGoogle()}%` : '0%'}}
></div>
</div>
<span>{!this.state.loaded ? "Loading..." : this.state.userDetails.googleDriveEnabled ? `${bytes(this.state.userDetails.storageDataGoogle.storageSize)} of
${bytes(this.state.userDetails.storageDataGoogle.storageLimit)} used` : ""}</span>
<span>{this.state.userDetails.googleDriveEnabled ? !this.state.userDetails.storageDataGoogle.failed ? `${bytes(this.state.userDetails.storageDataGoogle.storageSize)} of
${bytes(this.state.userDetails.storageDataGoogle.storageLimit)} used` : "Failed" : ""}
</span>
</div>
</div>
<div class="value__updater">
@@ -1140,7 +1144,8 @@ class SettingsPageContainer extends React.Component {
style={{backgroundColor:"#3c85ee", width:"100%"}}
></div>
</div>
<span>{!this.state.loaded ? "Loading..." : this.state.userDetails.s3Enabled ? `${bytes(this.state.userDetails.storageDataPersonal.storageSize)} used` : ""}</span>
<span>{this.state.userDetails.s3Enabled ? !this.state.userDetails.storageDataPersonal.failed ?
`${bytes(this.state.userDetails.storageDataPersonal.storageSize)} used` : "Failed" : ""}</span>
</div>
</div>
<div class="value__updater">