Files
free-file-sync-mirror/FreeFileSync/Source/base/synchronization.cpp
2025-12-10 14:38:26 -08:00

2995 lines
141 KiB
C++

// *****************************************************************************
// * This file is part of the FreeFileSync project. It is distributed under *
// * GNU General Public License: https://www.gnu.org/licenses/gpl-3.0 *
// * Copyright (C) Zenju (zenju AT freefilesync DOT org) - All Rights Reserved *
// *****************************************************************************
#include "synchronization.h"
#include <tuple>
#include <zen/process_priority.h>
#include "algorithm.h"
#include "db_file.h"
#include "status_handler_impl.h"
#include "versioning.h"
#include "binary.h"
#include "../afs/concrete.h"
#include "../afs/native.h"
#include <unistd.h> //fsync
#include <fcntl.h> //open
using namespace zen;
using namespace fff;
namespace
{
const size_t CONFLICTS_PREVIEW_MAX = 25; //=> consider memory consumption, log file size, email size!
}
SyncStatistics::SyncStatistics(const FolderComparison& folderCmp)
{
for (const BaseFolderPair& baseFolder : asRange(folderCmp))
recurse(baseFolder);
}
SyncStatistics::SyncStatistics(const ContainerObject& conObj)
{
recurse(conObj);
}
SyncStatistics::SyncStatistics(const FilePair& file)
{
processFile(file);
++rowsTotal_;
}
inline
void SyncStatistics::recurse(const ContainerObject& conObj)
{
for (const FilePair& file : conObj.files())
processFile(file);
for (const SymlinkPair& symlink : conObj.symlinks())
processLink(symlink);
for (const FolderPair& folder : conObj.subfolders())
processFolder(folder);
rowsTotal_ += conObj.subfolders().size();
rowsTotal_ += conObj.files ().size();
rowsTotal_ += conObj.symlinks ().size();
}
inline
void SyncStatistics::logConflict(const FileSystemObject& fsObj)
{
if (conflictsPreview_.size() < CONFLICTS_PREVIEW_MAX)
{
const Zstring& relPathL = fsObj.getRelativePath<SelectSide::left >();
const Zstring& relPathR = fsObj.getRelativePath<SelectSide::right>();
conflictsPreview_.push_back((getUnicodeNormalForm(relPathL) == getUnicodeNormalForm(relPathR) ?
utfTo<std::wstring>(relPathL) :
utfTo<std::wstring>(relPathL + Zstr('\n') + relPathR)) + L": " + fsObj.getSyncOpConflict());
}
}
inline
void SyncStatistics::processFile(const FilePair& file)
{
switch (file.getSyncOperation()) //evaluate comparison result and sync direction
{
case SO_CREATE_LEFT:
++createLeft_;
bytesToProcess_ += static_cast<int64_t>(file.getFileSize<SelectSide::right>());
break;
case SO_CREATE_RIGHT:
++createRight_;
bytesToProcess_ += static_cast<int64_t>(file.getFileSize<SelectSide::left>());
break;
case SO_DELETE_LEFT:
++deleteLeft_;
break;
case SO_DELETE_RIGHT:
++deleteRight_;
break;
case SO_MOVE_LEFT_TO:
++updateLeft_;
break;
case SO_MOVE_RIGHT_TO:
++updateRight_;
break;
case SO_MOVE_LEFT_FROM: //ignore; already counted
case SO_MOVE_RIGHT_FROM: //=> harmonize with FileView::applyActionFilter()
break;
case SO_OVERWRITE_LEFT:
++updateLeft_;
bytesToProcess_ += static_cast<int64_t>(file.getFileSize<SelectSide::right>());
break;
case SO_OVERWRITE_RIGHT:
++updateRight_;
bytesToProcess_ += static_cast<int64_t>(file.getFileSize<SelectSide::left>());
break;
case SO_UNRESOLVED_CONFLICT:
++conflictCount_;
logConflict(file);
break;
case SO_RENAME_LEFT:
++updateLeft_;
break;
case SO_RENAME_RIGHT:
++updateRight_;
break;
case SO_DO_NOTHING:
case SO_EQUAL:
break;
}
}
inline
void SyncStatistics::processLink(const SymlinkPair& symlink)
{
switch (symlink.getSyncOperation()) //evaluate comparison result and sync direction
{
case SO_CREATE_LEFT:
++createLeft_;
break;
case SO_CREATE_RIGHT:
++createRight_;
break;
case SO_DELETE_LEFT:
++deleteLeft_;
break;
case SO_DELETE_RIGHT:
++deleteRight_;
break;
case SO_OVERWRITE_LEFT:
case SO_RENAME_LEFT:
++updateLeft_;
break;
case SO_OVERWRITE_RIGHT:
case SO_RENAME_RIGHT:
++updateRight_;
break;
case SO_UNRESOLVED_CONFLICT:
++conflictCount_;
logConflict(symlink);
break;
case SO_MOVE_LEFT_FROM:
case SO_MOVE_RIGHT_FROM:
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
assert(false);
break;
case SO_DO_NOTHING:
case SO_EQUAL:
break;
}
}
inline
void SyncStatistics::processFolder(const FolderPair& folder)
{
switch (folder.getSyncOperation()) //evaluate comparison result and sync direction
{
case SO_CREATE_LEFT:
++createLeft_;
break;
case SO_CREATE_RIGHT:
++createRight_;
break;
case SO_DELETE_LEFT: //if deletion variant == versioning with user-defined directory existing on other volume, this results in a full copy + delete operation!
++deleteLeft_; //however we cannot (reliably) anticipate this situation, fortunately statistics can be adapted during sync!
break;
case SO_DELETE_RIGHT:
++deleteRight_;
break;
case SO_UNRESOLVED_CONFLICT:
++conflictCount_;
logConflict(folder);
break;
case SO_RENAME_LEFT:
++updateLeft_;
break;
case SO_RENAME_RIGHT:
++updateRight_;
break;
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
case SO_MOVE_LEFT_FROM:
case SO_MOVE_RIGHT_FROM:
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
assert(false);
[[fallthrough]];
case SO_DO_NOTHING:
case SO_EQUAL:
break;
}
recurse(folder); //since we model logical stats, we recurse, even if deletion variant is "recycler" or "versioning + same volume", which is a single physical operation!
}
/* DeletionVariant::permanent: deletion frees space
DeletionVariant::recycler: won't free space until recycler is full, but then frees space
DeletionVariant::versioning: depends on whether versioning folder is on a different volume
-> if deleted item is a followed symlink, no space is freed
-> created/updated/deleted item may be on a different volume than base directory: consider symlinks, junctions!
=> generally assume deletion frees space; may avoid false-positive disk space warnings for recycler and versioning */
class MinimumDiskSpaceNeeded
{
public:
static std::pair<int64_t, int64_t> calculate(const BaseFolderPair& baseFolder)
{
MinimumDiskSpaceNeeded inst;
inst.recurse(baseFolder);
return {inst.spaceNeededLeft_, inst.spaceNeededRight_};
}
private:
void recurse(const ContainerObject& conObj)
{
//process files
for (const FilePair& file : conObj.files())
switch (file.getSyncOperation()) //evaluate comparison result and sync direction
{
case SO_CREATE_LEFT:
spaceNeededLeft_ += static_cast<int64_t>(file.getFileSize<SelectSide::right>());
break;
case SO_CREATE_RIGHT:
spaceNeededRight_ += static_cast<int64_t>(file.getFileSize<SelectSide::left>());
break;
case SO_DELETE_LEFT:
if (!file.isFollowedSymlink<SelectSide::left>())
spaceNeededLeft_ -= static_cast<int64_t>(file.getFileSize<SelectSide::left>());
break;
case SO_DELETE_RIGHT:
if (!file.isFollowedSymlink<SelectSide::right>())
spaceNeededRight_ -= static_cast<int64_t>(file.getFileSize<SelectSide::right>());
break;
case SO_OVERWRITE_LEFT:
if (!file.isFollowedSymlink<SelectSide::left>())
spaceNeededLeft_ -= static_cast<int64_t>(file.getFileSize<SelectSide::left>());
spaceNeededLeft_ += static_cast<int64_t>(file.getFileSize<SelectSide::right>());
break;
case SO_OVERWRITE_RIGHT:
if (!file.isFollowedSymlink<SelectSide::right>())
spaceNeededRight_ -= static_cast<int64_t>(file.getFileSize<SelectSide::right>());
spaceNeededRight_ += static_cast<int64_t>(file.getFileSize<SelectSide::left>());
break;
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
case SO_MOVE_LEFT_FROM:
case SO_MOVE_RIGHT_FROM:
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
break;
}
//symbolic links
//[...]
//recurse into sub-dirs
for (const FolderPair& folder : conObj.subfolders())
switch (folder.getSyncOperation())
{
case SO_DELETE_LEFT:
if (!folder.isFollowedSymlink<SelectSide::left>())
recurse(folder); //not 100% correct: in fact more that what our model contains may be deleted (consider file filter!)
break;
case SO_DELETE_RIGHT:
if (!folder.isFollowedSymlink<SelectSide::right>())
recurse(folder);
break;
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
case SO_MOVE_LEFT_FROM:
case SO_MOVE_RIGHT_FROM:
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
assert(false);
[[fallthrough]];
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
recurse(folder); //not 100% correct: what if left or right folder is symlink!? => file operations may happen on different volume!
break;
}
}
int64_t spaceNeededLeft_ = 0;
int64_t spaceNeededRight_ = 0;
};
//-----------------------------------------------------------------------------------------------------------
std::vector<FolderPairSyncCfg> fff::extractSyncCfg(const MainConfiguration& mainCfg)
{
//merge first and additional pairs
std::vector<LocalPairConfig> localCfgs = {mainCfg.firstPair};
append(localCfgs, mainCfg.additionalPairs);
std::vector<FolderPairSyncCfg> output;
for (const LocalPairConfig& lpc : localCfgs)
{
//const CompConfig cmpCfg = lpc.localCmpCfg ? *lpc.localCmpCfg : mainCfg.cmpCfg;
const SyncConfig syncCfg = lpc.localSyncCfg ? *lpc.localSyncCfg : mainCfg.syncCfg;
output.push_back(
{
getSyncVariant(syncCfg.directionCfg),
!!std::get_if<DirectionByChange>(&syncCfg.directionCfg.dirs),
syncCfg.deletionVariant,
syncCfg.versioningFolderPhrase,
syncCfg.versioningStyle,
syncCfg.versionMaxAgeDays,
syncCfg.versionCountMin,
syncCfg.versionCountMax
});
}
return output;
}
//------------------------------------------------------------------------------------------------------------
namespace
{
//test if user accidentally selected the wrong folders to sync
bool significantDifferenceDetected(const SyncStatistics& folderPairStat)
{
//initial file copying shall not be detected as major difference
if ((folderPairStat.createCount<SelectSide::left >() == 0 ||
folderPairStat.createCount<SelectSide::right>() == 0) &&
folderPairStat.updateCount () == 0 &&
folderPairStat.deleteCount () == 0 &&
folderPairStat.conflictCount() == 0)
return false;
const int nonMatchingRows = folderPairStat.createCount() +
folderPairStat.deleteCount();
//folderPairStat.updateCount() + -> not relevant when testing for "wrong folder selected"
//folderPairStat.conflictCount();
return nonMatchingRows >= 10 && nonMatchingRows > 0.5 * folderPairStat.rowCount();
}
//---------------------------------------------------------------------------------------------
template <SelectSide side>
bool plannedWriteAccess(const FileSystemObject& fsObj)
{
switch (getEffectiveSyncDir(fsObj.getSyncOperation()))
{
case SyncDirection::none: return false;
case SyncDirection::left: return side == SelectSide::left;
case SyncDirection::right: return side == SelectSide::right;
}
throw std::logic_error(std::string(__FILE__) + '[' + numberTo<std::string>(__LINE__) + "] Contract violation!");
}
inline
AbstractPath getAbstractPath(const FileSystemObject& fsObj, SelectSide side)
{
return side == SelectSide::left ? fsObj.getAbstractPath<SelectSide::left>() : fsObj.getAbstractPath<SelectSide::right>();
}
struct PathRaceItem
{
const FileSystemObject* fsObj;
SelectSide side;
std::strong_ordering operator<=>(const PathRaceItem&) const = default;
};
std::weak_ordering comparePathNoCase(const PathRaceItem& lhs, const PathRaceItem& rhs)
{
const AbstractPath& itemPathL = getAbstractPath(*lhs.fsObj, lhs.side);
const AbstractPath& itemPathR = getAbstractPath(*rhs.fsObj, rhs.side);
if (const std::weak_ordering cmp = itemPathL.afsDevice <=> itemPathR.afsDevice;
cmp != std::weak_ordering::equivalent)
return cmp;
return compareNoCase(itemPathL.afsPath.value, //no hashing: want natural sort order!
itemPathR.afsPath.value);
}
std::wstring formatRaceItem(const PathRaceItem& item)
{
const SyncDirection syncDir = getEffectiveSyncDir(item.fsObj->getSyncOperation());
const bool writeAcess = (syncDir == SyncDirection::left && item.side == SelectSide::left) ||
(syncDir == SyncDirection::right && item.side == SelectSide::right);
return AFS::getDisplayPath(item.side == SelectSide::left ?
item.fsObj->base().getAbstractPath<SelectSide::left>() :
item.fsObj->base().getAbstractPath<SelectSide::right>()) +
(writeAcess ? L" 💾 " : L" 👓 ") +
utfTo<std::wstring>(item.side == SelectSide::left ?
item.fsObj->getRelativePath<SelectSide::left>() :
item.fsObj->getRelativePath<SelectSide::right>());
//e.g. D:\folder 💾 subfolder\file.txt
// D:\folder\subfolder 👓 file.txt
}
struct ChildPathRef
{
const FileSystemObject* fsObj = nullptr;
uint64_t afsPathHash = 0; //of *case-normalized* AfsPath
};
template <SelectSide side>
class GetChildItemsHashed
{
public:
static std::vector<ChildPathRef> execute(const ContainerObject& folder)
{
FNV1aHash<uint64_t> pathHash;
for (const Zstring& itemName : splitCpy<Zstring>(folder.getAbstractPath<side>().afsPath.value, FILE_NAME_SEPARATOR, SplitOnEmpty::skip))
hashAdd(pathHash, itemName); //not really needed ATM, but it's cleaner to hash *full* afsPath
GetChildItemsHashed inst;
inst.recurse(folder, pathHash.get());
return std::move(inst.childPathRefs_);
}
private:
GetChildItemsHashed (const GetChildItemsHashed&) = delete;
GetChildItemsHashed& operator=(const GetChildItemsHashed&) = delete;
GetChildItemsHashed() {}
void recurse(const ContainerObject& conObj, uint64_t parentPathHash)
{
for (const FilePair& file : conObj.files())
childPathRefs_.push_back({&file, getPathHash(file, parentPathHash)});
//S1 -> T (update) is not a conflict (anymore) if S1, S2 contain different files
//S2 -> T (update) https://freefilesync.org/forum/viewtopic.php?t=9365#p36466
for (const SymlinkPair& symlink : conObj.symlinks())
childPathRefs_.push_back({&symlink, getPathHash(symlink, parentPathHash)});
for (const FolderPair& subFolder : conObj.subfolders())
{
const uint64_t folderPathHash = getPathHash(subFolder, parentPathHash);
childPathRefs_.push_back({&subFolder, folderPathHash});
recurse(subFolder, folderPathHash);
}
}
static void hashAdd(FNV1aHash<uint64_t>& hash, const Zstring& itemName)
{
if (isAsciiString(itemName)) //fast path: no need for extra memory allocation!
for (const Zchar c : itemName)
hash.add(asciiToUpper(c));
else
for (const Zchar c : getUpperCase(itemName))
hash.add(c);
}
static uint64_t getPathHash(const FileSystemObject& fsObj, uint64_t parentPathHash)
{
FNV1aHash<uint64_t> hash(parentPathHash);
hashAdd(hash, fsObj.getItemName<side>());
return hash.get();
}
std::vector<ChildPathRef> childPathRefs_;
};
template <SelectSide sideL, SelectSide sideR>
std::weak_ordering comparePathNoCase(const ChildPathRef& lhs, const ChildPathRef& rhs)
{
//assert(lhs.fsObj->getAbstractPath<sideL>().afsDevice == -> too slow, even for debug build
// rhs.fsObj->getAbstractPath<sideR>().afsDevice);
if (const std::weak_ordering cmp = lhs.afsPathHash <=> rhs.afsPathHash;
cmp != std::weak_ordering::equivalent)
return cmp; //fast path!
return compareNoCase(lhs.fsObj->getAbstractPath<sideL>().afsPath.value, //fsObj may come from *different* BaseFolderPair
rhs.fsObj->getAbstractPath<sideR>().afsPath.value); // => don't compare getRelativePath()!
}
template <SelectSide side>
void sortAndRemoveDuplicates(std::vector<ChildPathRef>& pathRefs)
{
std::sort(pathRefs.begin(), pathRefs.end(), [](const ChildPathRef& lhs, const ChildPathRef& rhs)
{
if (const std::weak_ordering cmp = comparePathNoCase<side, side>(lhs, rhs);
cmp != std::weak_ordering::equivalent)
return cmp < 0;
return //multiple (case-insensitive) relPaths? => order write-access before read-access, so that std::unique leaves a write if existing!
plannedWriteAccess<side>(*lhs.fsObj) >
plannedWriteAccess<side>(*rhs.fsObj);
});
pathRefs.erase(std::unique(pathRefs.begin(), pathRefs.end(),
[](const ChildPathRef& lhs, const ChildPathRef& rhs) { return comparePathNoCase<side, side>(lhs, rhs) == std::weak_ordering::equivalent; }),
pathRefs.end());
//let's not use removeDuplicates(): we rely too much on implementation details!
}
//check if some files/folders are included more than once and form a race condition (:= multiple accesses of which at least one is a write)
// - checking filter for subfolder exclusion is not good enough: one folder may have a *.txt include-filter, the other a *.lng include filter => still no dependencies
// - user may have manually excluded the conflicting items or changed the filter settings without running a re-compare
template <SelectSide sideP, SelectSide sideC>
void checkPathRaceCondition(const BaseFolderPair& baseFolderP, const BaseFolderPair& baseFolderC, std::vector<PathRaceItem>& pathRaceItems)
{
const AbstractPath basePathP = baseFolderP.getAbstractPath<sideP>(); //parent/child notion is tentative at this point
const AbstractPath basePathC = baseFolderC.getAbstractPath<sideC>(); //=> will be swapped if necessary
assert(!AFS::isNullPath(basePathP) && !AFS::isNullPath(basePathC));
if (basePathP.afsDevice == basePathC.afsDevice)
{
if (basePathP.afsPath.value.size() > basePathC.afsPath.value.size())
return checkPathRaceCondition<sideC, sideP>(baseFolderC, baseFolderP, pathRaceItems);
const std::vector<Zstring> relPathP = splitCpy(basePathP.afsPath.value, FILE_NAME_SEPARATOR, SplitOnEmpty::skip);
const std::vector<Zstring> relPathC = splitCpy(basePathC.afsPath.value, FILE_NAME_SEPARATOR, SplitOnEmpty::skip);
if (relPathP.size() <= relPathC.size() &&
/**/std::equal(relPathP.begin(), relPathP.end(), relPathC.begin(), [](const Zstring& lhs, const Zstring& rhs) { return equalNoCase(lhs, rhs); }))
{
//=> at this point parent/child folders are confirmed
//now find child folder match inside baseFolderP
//e.g. C:\folder <-> C:\folder\sub => find "sub" inside C:\folder
std::vector<const ContainerObject*> childFolderP{&baseFolderP};
std::for_each(relPathC.begin() + relPathP.size(), relPathC.end(), [&](const Zstring& itemName)
{
std::vector<const ContainerObject*> childFolderP2;
for (const ContainerObject* childFolder : childFolderP)
for (const FolderPair& folder : childFolder->subfolders())
if (equalNoCase(folder.getItemName<sideP>(), itemName))
childFolderP2.push_back(&folder);
//no "break": yes, weird, but there could be more than one (for case-sensitive file system)
childFolderP = std::move(childFolderP2);
});
std::vector<ChildPathRef> pathRefsP;
for (const ContainerObject* childFolder : childFolderP)
append(pathRefsP, GetChildItemsHashed<sideP>::execute(*childFolder));
std::vector<ChildPathRef> pathRefsC = GetChildItemsHashed<sideC>::execute(baseFolderC);
//---------------------------------------------------------------------------------------------------
//case-sensitive comparison because items were scanned by FFS (=> no messy user input)?
//not good enough! E.g. not-yet-existing files are set to be created with different case!
// + (weird) a file and a folder are set to be created with same name
// => (throw hands in the air) fine, check path only and don't consider case
sortAndRemoveDuplicates<sideP>(pathRefsP);
sortAndRemoveDuplicates<sideC>(pathRefsC);
mergeTraversal(pathRefsP.begin(), pathRefsP.end(),
pathRefsC.begin(), pathRefsC.end(),
[](const ChildPathRef&) {} /*left only*/,
[&](const ChildPathRef& lhs, const ChildPathRef& rhs)
{
if (plannedWriteAccess<sideP>(*lhs.fsObj) ||
plannedWriteAccess<sideC>(*rhs.fsObj))
{
pathRaceItems.push_back({lhs.fsObj, sideP});
pathRaceItems.push_back({rhs.fsObj, sideC});
}
},
[](const ChildPathRef&) {} /*right only*/, comparePathNoCase<sideP, sideC>);
}
}
}
//#################################################################################################################
//--------------------- data verification -------------------------
void flushFileBuffers(const Zstring& nativeFilePath) //throw FileError
{
const int fdFile = ::open(nativeFilePath.c_str(), O_WRONLY | O_APPEND | O_CLOEXEC);
if (fdFile == -1)
THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot open file %x."), L"%x", fmtPath(nativeFilePath)), "open");
ZEN_ON_SCOPE_EXIT(::close(fdFile));
if (::fsync(fdFile) != 0)
THROW_LAST_FILE_ERROR(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(nativeFilePath)), "fsync");
}
void verifyFiles(const AbstractPath& sourcePath, const AbstractPath& targetPath, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X
{
try
{
//do like "copy /v": 1. flush target file buffers, 2. read again as usual (using OS buffers)
// => it seems OS buffers are not invalidated by this: snake oil???
if (const Zstring& targetPathNative = getNativeItemPath(targetPath);
!targetPathNative.empty())
flushFileBuffers(targetPathNative); //throw FileError
if (!filesHaveSameContent(sourcePath, targetPath, notifyUnbufferedIO)) //throw FileError, X
throw FileError(replaceCpy(replaceCpy(_("%x and %y have different content."),
L"%x", L'\n' + fmtPath(AFS::getDisplayPath(sourcePath))),
L"%y", L'\n' + fmtPath(AFS::getDisplayPath(targetPath))));
}
catch (const FileError& e) //add some context to error message
{
throw FileError(_("Data verification error:"), e.toString());
}
}
//#################################################################################################################
//#################################################################################################################
/* ________________________________________________________________
| |
| Multithreaded File Copy: Parallel API for expensive file I/O |
|______________________________________________________________| */
namespace parallel
{
inline
AFS::ItemType getItemType(const AbstractPath& itemPath, std::mutex& singleThread) //throw FileError
{ return parallelScope([itemPath] { return AFS::getItemType(itemPath); /*throw FileError*/ }, singleThread); }
inline
bool itemExists(const AbstractPath& itemPath, std::mutex& singleThread) //throw FileError
{ return parallelScope([itemPath] { return AFS::itemExists(itemPath); /*throw FileError*/ }, singleThread); }
inline
void removeFileIfExists(const AbstractPath& filePath, std::mutex& singleThread) //throw FileError
{ parallelScope([filePath] { AFS::removeFileIfExists(filePath); /*throw FileError*/ }, singleThread); }
inline
void removeSymlinkIfExists(const AbstractPath& linkPath, std::mutex& singleThread) //throw FileError
{ parallelScope([linkPath] { AFS::removeSymlinkIfExists(linkPath); /*throw FileError*/ }, singleThread); }
inline
void moveAndRenameItem(const AbstractPath& pathFrom, const AbstractPath& pathTo, std::mutex& singleThread) //throw FileError, ErrorMoveUnsupported
{ parallelScope([pathFrom, pathTo] { AFS::moveAndRenameItem(pathFrom, pathTo); /*throw FileError, ErrorMoveUnsupported*/ }, singleThread); }
inline
AbstractPath getSymlinkResolvedPath(const AbstractPath& linkPath, std::mutex& singleThread) //throw FileError
{ return parallelScope([linkPath] { return AFS::getSymlinkResolvedPath(linkPath); /*throw FileError*/ }, singleThread); }
inline
void copySymlink(const AbstractPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions, std::mutex& singleThread) //throw FileError
{ parallelScope([sourcePath, targetPath, copyFilePermissions] { AFS::copySymlink(sourcePath, targetPath, copyFilePermissions); /*throw FileError*/ }, singleThread); }
inline
void copyNewFolder(const AbstractPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions, std::mutex& singleThread) //throw FileError
{ parallelScope([sourcePath, targetPath, copyFilePermissions] { return AFS::copyNewFolder(sourcePath, targetPath, copyFilePermissions); /*throw FileError*/ }, singleThread); }
inline
void removeFilePlain(const AbstractPath& filePath, std::mutex& singleThread) //throw FileError
{ parallelScope([filePath] { AFS::removeFilePlain(filePath); /*throw FileError*/ }, singleThread); }
//--------------------------------------------------------------
//ATTENTION CALLBACKS: they also run asynchronously *outside* the singleThread lock!
//--------------------------------------------------------------
inline
void removeFolderIfExistsRecursion(const AbstractPath& folderPath, //throw FileError
const std::function<void(const std::wstring& displayPath)>& onBeforeFileDeletion /*throw X*/, //
const std::function<void(const std::wstring& displayPath)>& onBeforeSymlinkDeletion /*throw X*/, //optional
const std::function<void(const std::wstring& displayPath)>& onBeforeFolderDeletion /*throw X*/, //
std::mutex& singleThread)
{
parallelScope([folderPath, onBeforeFileDeletion, onBeforeSymlinkDeletion, onBeforeFolderDeletion]
{ AFS::removeFolderIfExistsRecursion(folderPath, onBeforeFileDeletion, onBeforeSymlinkDeletion, onBeforeFolderDeletion); /*throw FileError*/ }, singleThread);
}
inline
AFS::FileCopyResult copyFileTransactional(const AbstractPath& sourcePath, const AFS::StreamAttributes& attrSource, //throw FileError, ErrorFileLocked, X
const AbstractPath& targetPath,
bool copyFilePermissions,
bool transactionalCopy,
const std::function<void()>& onDeleteTargetFile /*throw X*/,
const IoCallback& notifyUnbufferedIO /*throw X*/,
std::mutex& singleThread)
{
return parallelScope([=]
{
return AFS::copyFileTransactional(sourcePath, attrSource, targetPath, copyFilePermissions, transactionalCopy, onDeleteTargetFile, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X
}, singleThread);
}
inline //RecycleSession::moveToRecycleBin() is internally synchronized!
void moveToRecycleBinIfExists(AFS::RecycleSession& recyclerSession, const AbstractPath& itemPath, const Zstring& logicalRelPath, std::mutex& singleThread) //throw FileError, RecycleBinUnavailable
{ parallelScope([=, &recyclerSession] { return recyclerSession.moveToRecycleBinIfExists(itemPath, logicalRelPath); /*throw FileError, RecycleBinUnavailable*/ }, singleThread); }
inline //FileVersioner::revisionFile() is internally synchronized!
void revisionFile(FileVersioner& versioner, const FileDescriptor& fileDescr, const Zstring& relativePath, const IoCallback& notifyUnbufferedIO /*throw X*/, std::mutex& singleThread) //throw FileError, X
{ parallelScope([=, &versioner] { versioner.revisionFile(fileDescr, relativePath, notifyUnbufferedIO); /*throw FileError, X*/ }, singleThread); }
inline //FileVersioner::revisionSymlink() is internally synchronized!
void revisionSymlink(FileVersioner& versioner, const AbstractPath& linkPath, const Zstring& relativePath, std::mutex& singleThread) //throw FileError
{ parallelScope([=, &versioner] { versioner.revisionSymlink(linkPath, relativePath); /*throw FileError*/ }, singleThread); }
inline //FileVersioner::revisionFolder() is internally synchronized!
void revisionFolder(FileVersioner& versioner,
const AbstractPath& folderPath, const Zstring& relativePath,
const std::function<void(const std::wstring& displayPathFrom, const std::wstring& displayPathTo)>& onBeforeFileMove /*throw X*/,
const std::function<void(const std::wstring& displayPathFrom, const std::wstring& displayPathTo)>& onBeforeFolderMove /*throw X*/,
const IoCallback& notifyUnbufferedIO /*throw X*/,
std::mutex& singleThread) //throw FileError, X
{ parallelScope([=, &versioner] { versioner.revisionFolder(folderPath, relativePath, onBeforeFileMove, onBeforeFolderMove, notifyUnbufferedIO); /*throw FileError, X*/ }, singleThread); }
inline
void verifyFiles(const AbstractPath& sourcePath, const AbstractPath& targetPath, const IoCallback& notifyUnbufferedIO /*throw X*/, std::mutex& singleThread) //throw FileError, X
{ parallelScope([=] { ::verifyFiles(sourcePath, targetPath, notifyUnbufferedIO); /*throw FileError, X*/ }, singleThread); }
}
//#################################################################################################################
//#################################################################################################################
class DeletionHandler //abstract deletion variants: permanently, recycle bin, user-defined directory
{
public:
DeletionHandler(const AbstractPath& baseFolderPath,
bool& recyclerMissingReportOnce,
bool& warnRecyclerMissing,
DeletionVariant deletionVariant,
const AbstractPath& versioningFolderPath,
VersioningStyle versioningStyle,
time_t syncStartTime); //nothrow!
//clean-up temporary directory (recycle bin optimization)
void tryCleanup(PhaseCallback& cb /*throw X*/); //throw X
void removeFileWithCallback(const FileDescriptor& fileDescr, const Zstring& relPath, bool beforeOverwrite, AsyncItemStatReporter& statReporter, std::mutex& singleThread); //throw FileError, ThreadStopRequest
void removeLinkWithCallback(const AbstractPath& linkPath, const Zstring& relPath, bool beforeOverwrite, AsyncItemStatReporter& statReporter, std::mutex& singleThread); //
void removeDirWithCallback (const AbstractPath& dirPath, const Zstring& relPath, AsyncItemStatReporter& statReporter, std::mutex& singleThread); //
private:
DeletionHandler (const DeletionHandler&) = delete;
DeletionHandler& operator=(const DeletionHandler&) = delete;
void moveToRecycleBinIfExists(const AbstractPath& itemPath, const Zstring& relPath, std::mutex& singleThread) //throw FileError, RecycleBinUnavailable
{
assert(deletionVariant_ == DeletionVariant::recycler);
//might not be needed => create lazily:
if (!recyclerSession_ && !recyclerUnavailableExcept_)
try
{
recyclerSession_ = AFS::createRecyclerSession(baseFolderPath_); //throw FileError, RecycleBinUnavailable
//double-initialization caveat: do NOT run session initialization in parallel!
// => createRecyclerSession must *not* do file I/O!
}
catch (const RecycleBinUnavailable& e) { recyclerUnavailableExcept_ = e; }
if (recyclerUnavailableExcept_) //add context, or user might think we're removing baseFolderPath_!
throw RecycleBinUnavailable(replaceCpy(_("Unable to move %x to the recycle bin."), L"%x", fmtPath(AFS::getDisplayPath(itemPath))),
replaceCpy(recyclerUnavailableExcept_->toString(), L"\n\n", L'\n'));
/* "Unable to move "Z:\folder\file.txt" to the recycle bin.
The recycle bin is not available for "Z:\".
Ignore and delete permanently each time recycle bin is unavailable?" */
parallel::moveToRecycleBinIfExists(*recyclerSession_, itemPath, relPath, singleThread); //throw FileError, RecycleBinUnavailable
}
//might not be needed => create lazily:
FileVersioner& getOrCreateVersioner() //throw FileError
{
assert(deletionVariant_ == DeletionVariant::versioning);
if (!versioner_)
versioner_.emplace(versioningFolderPath_, versioningStyle_, syncStartTime_); //throw FileError
return *versioner_;
}
bool& recyclerMissingReportOnce_; //shared by threads! access under "singleThread" lock!
bool& warnRecyclerMissing_; //WarningDialogs::warnRecyclerMissing
const DeletionVariant deletionVariant_; //keep it invariant! e.g. consider getOrCreateVersioner() one-time construction!
const AbstractPath baseFolderPath_;
std::unique_ptr<AFS::RecycleSession> recyclerSession_; //it's one of these (or none if not yet initialized)
std::optional<RecycleBinUnavailable> recyclerUnavailableExcept_; //
//used only for DeletionVariant::versioning:
const AbstractPath versioningFolderPath_;
const VersioningStyle versioningStyle_;
const time_t syncStartTime_;
std::optional<FileVersioner> versioner_;
//buffer status texts:
const std::wstring txtDelFilePermanent_ = _("Deleting file %x");
const std::wstring txtDelFileRecycler_ = _("Moving file %x to the recycle bin");
const std::wstring txtDelFileVersioning_ = replaceCpy(_("Moving file %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_)));
const std::wstring txtDelSymlinkPermanent_ = _("Deleting symbolic link %x");
const std::wstring txtDelSymlinkRecycler_ = _("Moving symbolic link %x to the recycle bin");
const std::wstring txtDelSymlinkVersioning_ = replaceCpy(_("Moving symbolic link %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_)));
const std::wstring txtDelFolderPermanent_ = _("Deleting folder %x");
const std::wstring txtDelFolderRecycler_ = _("Moving folder %x to the recycle bin");
const std::wstring txtDelFolderVersioning_ = replaceCpy(_("Moving folder %x to %y"), L"%y", fmtPath(AFS::getDisplayPath(versioningFolderPath_)));
const std::wstring txtMovingFileXtoY_ = _("Moving file %x to %y");
const std::wstring txtMovingFolderXtoY_ = _("Moving folder %x to %y");
};
DeletionHandler::DeletionHandler(const AbstractPath& baseFolderPath,
bool& recyclerMissingReportOnce,
bool& warnRecyclerMissing,
DeletionVariant deletionVariant,
const AbstractPath& versioningFolderPath,
VersioningStyle versioningStyle,
time_t syncStartTime) :
recyclerMissingReportOnce_(recyclerMissingReportOnce),
warnRecyclerMissing_(warnRecyclerMissing),
deletionVariant_(deletionVariant),
baseFolderPath_(baseFolderPath),
versioningFolderPath_(versioningFolderPath),
versioningStyle_(versioningStyle),
syncStartTime_(syncStartTime) {}
void DeletionHandler::tryCleanup(PhaseCallback& cb /*throw X*/) //throw X
{
assert(runningOnMainThread());
switch (deletionVariant_)
{
case DeletionVariant::recycler:
if (recyclerSession_)
{
auto notifyDeletionStatus = [&](const std::wstring& displayPath)
{
if (!displayPath.empty())
cb.updateStatus(replaceCpy(txtDelFileRecycler_, L"%x", fmtPath(displayPath))); //throw X
else
cb.requestUiUpdate(); //throw X
};
//move content of temporary directory to recycle bin in one go
tryReportingError([&] { recyclerSession_->tryCleanup(notifyDeletionStatus); /*throw FileError*/}, cb); //throw X
}
break;
case DeletionVariant::permanent:
case DeletionVariant::versioning:
break;
}
}
void DeletionHandler::removeFileWithCallback(const FileDescriptor& fileDescr, const Zstring& relPath, bool beforeOverwrite,
AsyncItemStatReporter& statReporter, std::mutex& singleThread) //throw FileError, ThreadStopRequest
{
if (deletionVariant_ != DeletionVariant::permanent &&
endsWith(relPath, AFS::TEMP_FILE_ENDING)) //special rule: always delete .ffs_tmp files permanently!
{
if (!beforeOverwrite) reportInfo(replaceCpy(txtDelFilePermanent_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))), statReporter); //throw ThreadStopRequest
parallel::removeFileIfExists(fileDescr.path, singleThread); //throw FileError
}
else
/* don't use AsyncItemStatReporter if "beforeOverwrite":
- logInfo/updateStatus() is superfluous/confuses user, except: do show progress and allow cancel for versioning!
- no (logical) item count update desired
=> BUT: total byte count should still be adjusted if versioning requires a file copy instead of a move!
- if fail-safe file copy is active, then the next operation will be a simple "rename"
=> don't risk updateStatus() throwing ThreadStopRequest() leaving the target deleted rather than updated! */
switch (deletionVariant_)
{
case DeletionVariant::permanent:
if (!beforeOverwrite) reportInfo(replaceCpy(txtDelFilePermanent_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))), statReporter); //throw ThreadStopRequest
parallel::removeFileIfExists(fileDescr.path, singleThread); //throw FileError
break;
case DeletionVariant::recycler:
if (!beforeOverwrite) reportInfo(replaceCpy(txtDelFileRecycler_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))), statReporter); //throw ThreadStopRequest
try
{
moveToRecycleBinIfExists(fileDescr.path, relPath, singleThread); //throw FileError, RecycleBinUnavailable
}
catch (const RecycleBinUnavailable& e)
{
if (!recyclerMissingReportOnce_) //shared by threads! access under "singleThread" lock!
{
recyclerMissingReportOnce_ = true;
statReporter.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing_); //throw ThreadStopRequest
}
if (!beforeOverwrite) statReporter.logMessage(replaceCpy(txtDelFilePermanent_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path))) +
L" [" + _("Recycle bin unavailable") + L']', PhaseCallback::MsgType::warning); //throw ThreadStopRequest
parallel::removeFileIfExists(fileDescr.path, singleThread); //throw FileError
}
break;
case DeletionVariant::versioning:
{
std::wstring statusMsg = replaceCpy(txtDelFileVersioning_, L"%x", fmtPath(AFS::getDisplayPath(fileDescr.path)));
PercentStatReporter percentReporter(statusMsg, fileDescr.attr.fileSize, statReporter);
if (!beforeOverwrite) reportInfo(std::move(statusMsg), statReporter); //throw ThreadStopRequest
//else: 1. versioning is moving only: no (potentially throwing) status updates
// 2. versioning needs to copy: may throw ThreadStopRequest, but *no* status updates, unless copying takes so long that % needs to be displayed
//callback runs *outside* singleThread_ lock! => fine
IoCallback notifyUnbufferedIO = [&](int64_t bytesDelta)
{
percentReporter.updateDeltaAndStatus(bytesDelta); //throw ThreadStopRequest
interruptionPoint(); //throw ThreadStopRequest => not reliably covered by PercentStatReporter::updateDeltaAndStatus()!
};
parallel::revisionFile(getOrCreateVersioner(), fileDescr, relPath, notifyUnbufferedIO, singleThread); //throw FileError, ThreadStopRequest
}
break;
}
//even if the source item did not exist anymore, significant I/O work was done => report unconditionally
if (!beforeOverwrite) statReporter.reportDelta(1, 0);
}
void DeletionHandler::removeLinkWithCallback(const AbstractPath& linkPath, const Zstring& relPath, bool beforeOverwrite,
AsyncItemStatReporter& statReporter, std::mutex& singleThread) //throw FileError, throw ThreadStopRequest
{
/* don't use AsyncItemStatReporter if "beforeOverwrite":
- logInfo() is superfluous/confuses user
- no (logical) item count update desired
- don't risk updateStatus() throwing ThreadStopRequest() leaving the target deleted rather than updated! */
switch (deletionVariant_)
{
case DeletionVariant::permanent:
if (!beforeOverwrite) reportInfo(replaceCpy(txtDelSymlinkPermanent_, L"%x", fmtPath(AFS::getDisplayPath(linkPath))), statReporter); //throw ThreadStopRequest
parallel::removeSymlinkIfExists(linkPath, singleThread); //throw FileError
break;
case DeletionVariant::recycler:
if (!beforeOverwrite) reportInfo(replaceCpy(txtDelSymlinkRecycler_, L"%x", fmtPath(AFS::getDisplayPath(linkPath))), statReporter); //throw ThreadStopRequest
try
{
moveToRecycleBinIfExists(linkPath, relPath, singleThread); //throw FileError, RecycleBinUnavailable
}
catch (const RecycleBinUnavailable& e)
{
if (!recyclerMissingReportOnce_) //shared by threads! access under "singleThread" lock!
{
recyclerMissingReportOnce_ = true;
statReporter.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing_); //throw ThreadStopRequest
}
if (!beforeOverwrite) statReporter.logMessage(replaceCpy(txtDelSymlinkPermanent_, L"%x", fmtPath(AFS::getDisplayPath(linkPath))) +
L" [" + _("Recycle bin unavailable") + L']', PhaseCallback::MsgType::warning); //throw ThreadStopRequest
parallel::removeSymlinkIfExists(linkPath, singleThread); //throw FileError
}
break;
case DeletionVariant::versioning:
if (!beforeOverwrite) reportInfo(replaceCpy(txtDelSymlinkVersioning_, L"%x", fmtPath(AFS::getDisplayPath(linkPath))), statReporter); //throw ThreadStopRequest
parallel::revisionSymlink(getOrCreateVersioner(), linkPath, relPath, singleThread); //throw FileError
break;
}
//remain transactional as much as possible => no more callbacks that can throw after successful deletion! (next: update file model!)
//even if the source item did not exist anymore, significant I/O work was done => report unconditionally
if (!beforeOverwrite) statReporter.reportDelta(1, 0);
}
void DeletionHandler::removeDirWithCallback(const AbstractPath& folderPath, const Zstring& relPath,
AsyncItemStatReporter& statReporter, std::mutex& singleThread) //throw FileError, ThreadStopRequest
{
auto removeFolderPermanently = [&]
{
//callbacks run *outside* singleThread_ lock! => fine
auto onBeforeDeletion = [&statReporter](const std::wstring& statusText, const std::wstring& displayPath)
{
statReporter.updateStatus(replaceCpy(statusText, L"%x", fmtPath(displayPath))); //throw ThreadStopRequest
statReporter.reportDelta(1, 0); //it would be more correct to report *after* work was done!
};
static_assert(std::is_const_v<decltype(txtDelFilePermanent_)>, "callbacks better be thread-safe!");
parallel::removeFolderIfExistsRecursion(folderPath,
[&](const std::wstring& displayPath) { onBeforeDeletion(txtDelFilePermanent_, displayPath); },
[&](const std::wstring& displayPath) { onBeforeDeletion(txtDelSymlinkPermanent_, displayPath); },
[&](const std::wstring& displayPath) { onBeforeDeletion(txtDelFolderPermanent_, displayPath); }, singleThread); //throw FileError, ThreadStopRequest
};
switch (deletionVariant_)
{
case DeletionVariant::permanent:
{
reportInfo(replaceCpy(txtDelFolderPermanent_, L"%x", fmtPath(AFS::getDisplayPath(folderPath))), statReporter); //throw ThreadStopRequest
removeFolderPermanently(); //throw FileError, ThreadStopRequest
}
break;
case DeletionVariant::recycler:
reportInfo(replaceCpy(txtDelFolderRecycler_, L"%x", fmtPath(AFS::getDisplayPath(folderPath))), statReporter); //throw ThreadStopRequest
try
{
moveToRecycleBinIfExists(folderPath, relPath, singleThread); //throw FileError, RecycleBinUnavailable
statReporter.reportDelta(1, 0); //moving to recycler is ONE logical operation, irrespective of the number of child elements!
}
catch (const RecycleBinUnavailable& e)
{
if (!recyclerMissingReportOnce_) //shared by threads! access under "singleThread" lock!
{
recyclerMissingReportOnce_ = true;
statReporter.reportWarning(e.toString() + L"\n\n" + _("Ignore and delete permanently each time recycle bin is unavailable?"), warnRecyclerMissing_); //throw ThreadStopRequest
}
statReporter.logMessage(replaceCpy(txtDelFolderPermanent_, L"%x", fmtPath(AFS::getDisplayPath(folderPath))) +
L" [" + _("Recycle bin unavailable") + L']', PhaseCallback::MsgType::warning); //throw ThreadStopRequest
removeFolderPermanently(); //throw FileError, ThreadStopRequest
}
break;
case DeletionVariant::versioning:
{
reportInfo(replaceCpy(txtDelFolderVersioning_, L"%x", fmtPath(AFS::getDisplayPath(folderPath))), statReporter); //throw ThreadStopRequest
//callbacks run *outside* singleThread_ lock! => fine
auto notifyMove = [&statReporter](const std::wstring& statusText, const std::wstring& displayPathFrom, const std::wstring& displayPathTo)
{
statReporter.updateStatus(replaceCpy(replaceCpy(statusText, L"%x", L'\n' + fmtPath(displayPathFrom)), L"%y", L'\n' + fmtPath(displayPathTo))); //throw ThreadStopRequest
statReporter.reportDelta(1, 0); //it would be more correct to report *after* work was done!
};
static_assert(std::is_const_v<decltype(txtMovingFileXtoY_)>, "callbacks better be thread-safe!");
auto onBeforeFileMove = [&](const std::wstring& displayPathFrom, const std::wstring& displayPathTo) { notifyMove(txtMovingFileXtoY_, displayPathFrom, displayPathTo); };
auto onBeforeFolderMove = [&](const std::wstring& displayPathFrom, const std::wstring& displayPathTo) { notifyMove(txtMovingFolderXtoY_, displayPathFrom, displayPathTo); };
auto notifyUnbufferedIO = [&](int64_t bytesDelta) { statReporter.reportDelta(0, bytesDelta); interruptionPoint(); }; //throw ThreadStopRequest
parallel::revisionFolder(getOrCreateVersioner(), folderPath, relPath, onBeforeFileMove, onBeforeFolderMove, notifyUnbufferedIO, singleThread); //throw FileError, ThreadStopRequest
}
break;
}
}
//===================================================================================================
//===================================================================================================
class Workload
{
public:
Workload(size_t threadCount, AsyncCallback& acb) : acb_(acb), workload_(threadCount) { assert(threadCount > 0); }
using WorkItem = std::function<void() /*throw ThreadStopRequest*/>;
using WorkItems = RingBuffer<WorkItem>; //FIFO!
//blocking call: context of worker thread
WorkItem getNext(size_t threadIdx) //throw ThreadStopRequest
{
interruptionPoint(); //throw ThreadStopRequest
std::unique_lock dummy(lockWork_);
for (;;)
{
if (!workload_[threadIdx].empty())
{
auto wi = std::move(workload_[threadIdx]. front());
/**/ workload_[threadIdx].pop_front();
return wi;
}
if (!pendingWorkload_.empty())
{
workload_[threadIdx] = std::move(pendingWorkload_. front());
/**/ pendingWorkload_.pop_front();
assert(!workload_[threadIdx].empty());
}
else
{
WorkItems& items = *std::max_element(workload_.begin(), workload_.end(), [](const WorkItems& lhs, const WorkItems& rhs) { return lhs.size() < rhs.size(); });
if (!items.empty()) //=> != workload_[threadIdx]
{
//steal half of largest workload from other thread
const size_t sz = items.size(); //[!] changes during loop!
for (size_t i = 0; i < sz; ++i)
{
auto wi = std::move(items. front());
/**/ items.pop_front();
if (i % 2 == 0)
workload_[threadIdx].push_back(std::move(wi));
else
items.push_back(std::move(wi));
}
}
else //wait...
{
if (++idleThreads_ == workload_.size())
acb_.notifyAllDone(); //noexcept
ZEN_ON_SCOPE_EXIT(--idleThreads_);
auto haveNewWork = [&] { return !pendingWorkload_.empty() || std::any_of(workload_.begin(), workload_.end(), [](const WorkItems& wi) { return !wi.empty(); }); };
interruptibleWait(conditionNewWork_, dummy, [&] { return haveNewWork(); }); //throw ThreadStopRequest
//it's sufficient to notify condition in addWorkItems() only (as long as we use std::condition_variable::notify_all())
}
}
}
}
void addWorkItems(RingBuffer<WorkItems>&& buckets)
{
{
std::lock_guard dummy(lockWork_);
while (!buckets.empty())
{
pendingWorkload_.push_back(std::move(buckets. front()));
/**/ buckets.pop_front();
}
}
conditionNewWork_.notify_all();
}
private:
Workload (const Workload&) = delete;
Workload& operator=(const Workload&) = delete;
AsyncCallback& acb_;
std::mutex lockWork_;
std::condition_variable conditionNewWork_;
size_t idleThreads_ = 0;
std::vector<WorkItems> workload_; //thread-specific buckets
RingBuffer<WorkItems> pendingWorkload_; //FIFO: buckets of work items for use by any thread
};
template <SelectSide side, class List> inline
bool haveNameClash(const FileSystemObject& fsObj, const List& m)
{
return std::any_of(m.begin(), m.end(), [itemName = fsObj.getItemName<side>()](const FileSystemObject& sibling)
{ return equalNoCase(sibling.getItemName<side>(), itemName); }); //equalNoCase: when in doubt => assume name clash!
}
class FolderPairSyncer
{
public:
struct SyncCtx
{
bool verifyCopiedFiles;
bool copyFilePermissions;
bool failSafeFileCopy;
DeletionHandler& delHandlerLeft;
DeletionHandler& delHandlerRight;
};
static void runSync(SyncCtx& syncCtx, BaseFolderPair& baseFolder, PhaseCallback& cb)
{
runPass(PassNo::zero, syncCtx, baseFolder, cb); //prepare file moves
runPass(PassNo::one, syncCtx, baseFolder, cb); //delete files (or overwrite big ones with smaller ones)
runPass(PassNo::two, syncCtx, baseFolder, cb); //copy rest
}
private:
friend class Workload;
enum class PassNo
{
zero, //prepare file moves
one, //delete files
two, //create, modify
never //skip item
};
FolderPairSyncer(SyncCtx& syncCtx, std::mutex& singleThread, AsyncCallback& acb) :
delHandlerLeft_ (syncCtx.delHandlerLeft),
delHandlerRight_ (syncCtx.delHandlerRight),
verifyCopiedFiles_ (syncCtx.verifyCopiedFiles),
copyFilePermissions_(syncCtx.copyFilePermissions),
failSafeFileCopy_ (syncCtx.failSafeFileCopy),
singleThread_(singleThread),
acb_(acb) {}
static PassNo getPass(const FilePair& file);
static PassNo getPass(const SymlinkPair& symlink);
static PassNo getPass(const FolderPair& folder);
static bool needZeroPass(const FilePair& file);
static bool needZeroPass(const FolderPair& folder);
static void runPass(PassNo pass, SyncCtx& syncCtx, BaseFolderPair& baseFolder, PhaseCallback& cb); //throw X
RingBuffer<Workload::WorkItems> getFolderLevelWorkItems(PassNo pass, ContainerObject& parentFolder, Workload& workload);
static bool containsMoveTarget(const FolderPair& parent);
void executeFileMove(FilePair& file); //throw ThreadStopRequest
template <SelectSide side> void executeFileMoveImpl(FilePair& fileFrom, FilePair& fileTo); //throw ThreadStopRequest
void synchronizeFile(FilePair& file); //
template <SelectSide side> void synchronizeFileInt(FilePair& file, SyncOperation syncOp); //throw FileError, ErrorMoveUnsupported, ThreadStopRequest
void synchronizeLink(SymlinkPair& symlink); //
template <SelectSide sideTrg> void synchronizeLinkInt(SymlinkPair& symlink, SyncOperation syncOp); //throw FileError, ThreadStopRequest
void synchronizeFolder(FolderPair& folder); //
template <SelectSide sideTrg> void synchronizeFolderInt(FolderPair& folder, SyncOperation syncOp); //throw FileError, ThreadStopRequest
void reportItemInfo(const std::wstring& msgTemplate, const AbstractPath& itemPath) { reportInfo(replaceCpy(msgTemplate, L"%x", fmtPath(AFS::getDisplayPath(itemPath))), acb_); }
void reportItemInfo(const std::wstring& msgTemplate, const AbstractPath& itemPath1, const AbstractPath& itemPath2) //throw ThreadStopRequest
{
reportInfo(replaceCpy(replaceCpy(msgTemplate, L"%x", L'\n' + fmtPath(AFS::getDisplayPath(itemPath1))),
L"%y", L'\n' + fmtPath(AFS::getDisplayPath(itemPath2))), acb_); //throw ThreadStopRequest
}
//already existing after onDeleteTargetFile(): undefined behavior! (e.g. fail/overwrite/auto-rename)
AFS::FileCopyResult copyFileWithCallback(const FileDescriptor& sourceDescr,
const AbstractPath& targetPath,
const std::function<void()>& onDeleteTargetFile /*throw X*/, //optional!
AsyncItemStatReporter& statReporter, //ThreadStopRequest
const std::wstring& statusMsg); //throw FileError, ThreadStopRequest, X
DeletionHandler& delHandlerLeft_;
DeletionHandler& delHandlerRight_;
const bool verifyCopiedFiles_;
const bool copyFilePermissions_;
const bool failSafeFileCopy_;
std::mutex& singleThread_;
AsyncCallback& acb_;
//preload status texts (premature?)
const std::wstring txtCreatingFile_ {_("Creating file %x" )};
const std::wstring txtCreatingLink_ {_("Creating symbolic link %x")};
const std::wstring txtCreatingFolder_ {_("Creating folder %x" )};
const std::wstring txtUpdatingFile_ {_("Updating file %x" )};
const std::wstring txtUpdatingLink_ {_("Updating symbolic link %x")};
const std::wstring txtVerifyingFile_ {_("Verifying file %x" )};
const std::wstring txtRenamingFileXtoY_ {_("Renaming file %x to %y" )};
const std::wstring txtRenamingLinkXtoY_ {_("Renaming symbolic link %x to %y")};
const std::wstring txtRenamingFolderXtoY_{_("Renaming folder %x to %y" )};
const std::wstring txtMovingFileXtoY_ {_("Moving file %x to %y" )};
const std::wstring txtSourceItemNotExist_{_("Source item %x is not existing")};
};
//===================================================================================================
//===================================================================================================
/* ___________________________
| |
| Multithreaded File Copy |
|_________________________|
---------------- =================
|Async Callback| <-- |Worker Thread 1|
---------------- ====================
/|\ |Worker Thread 2|
| =================
============= | ... |
GUI <-- |Main Thread| \|/ \|/
Callback ============= --------------------
| Workload |
--------------------
Notes: - All threads share a single mutex, unlocked only during file I/O => do NOT require file_hierarchy.cpp classes to be thread-safe (i.e. internally synchronized)!
- Workload holds (folder-level-) items in buckets associated with each worker thread (FTP scenario: avoid CWDs)
- If a worker is idle, its Workload bucket is empty and no more pending buckets available: steal from other threads (=> take half of largest bucket)
- Maximize opportunity for parallelization ASAP: Workload buckets serve folder-items *before* files/symlinks => reduce risk of work-stealing
- Memory consumption: work items may grow indefinitely; however: test case "C:\" ~80MB per 1 million work items
*/
void FolderPairSyncer::runPass(PassNo pass, SyncCtx& syncCtx, BaseFolderPair& baseFolder, PhaseCallback& cb) //throw X
{
std::mutex singleThread; //only a single worker thread may run at a time, except for parallel file I/O
AsyncCallback acb; //
FolderPairSyncer fps(syncCtx, singleThread, acb); //manage life time: enclose InterruptibleThread's!!!
Workload workload(1, acb);
workload.addWorkItems(fps.getFolderLevelWorkItems(pass, baseFolder, workload)); //initial workload: set *before* threads get access!
std::vector<InterruptibleThread> worker;
ZEN_ON_SCOPE_EXIT( for (InterruptibleThread& wt : worker) wt.requestStop(); ); //stop *all* at the same time before join!
size_t threadIdx = 0;
Zstring threadName = Zstr("Sync");
worker.emplace_back([threadIdx, &singleThread, &acb, &workload, threadName = std::move(threadName)]
{
setCurrentThreadName(threadName);
while (/*blocking call:*/ std::function<void()> workItem = workload.getNext(threadIdx)) //throw ThreadStopRequest
{
acb.notifyTaskBegin(0 /*prio*/); //same prio, while processing only one folder pair at a time
ZEN_ON_SCOPE_EXIT(acb.notifyTaskEnd());
std::lock_guard dummy(singleThread); //protect ALL accesses to "fps" and workItem execution!
workItem(); //throw ThreadStopRequest
}
});
acb.waitUntilDone(UI_UPDATE_INTERVAL / 2 /*every ~25 ms*/, cb); //throw X
}
//thread-safe thanks to std::mutex singleThread
RingBuffer<Workload::WorkItems> FolderPairSyncer::getFolderLevelWorkItems(PassNo pass, ContainerObject& parentFolder, Workload& workload)
{
RingBuffer<Workload::WorkItems> buckets;
RingBuffer<ContainerObject*> foldersToInspect;
foldersToInspect.push_back(&parentFolder);
while (!foldersToInspect.empty())
{
ContainerObject& conObj = *foldersToInspect. front();
/**/ foldersToInspect.pop_front();
RingBuffer<std::function<void()>> workItems;
if (pass == PassNo::zero)
{
//create folders as required by file move targets:
for (FolderPair& folder : conObj.subfolders())
if (needZeroPass(folder))
workItems.push_back([this, &folder, &workload, pass]
{
tryReportingError([&] { synchronizeFolder(folder); }, acb_); //throw ThreadStopRequest
//error? => still process move targets (for delete + copy fall back!)
workload.addWorkItems(getFolderLevelWorkItems(pass, folder, workload));
});
else
foldersToInspect.push_back(&folder);
for (FilePair& file : conObj.files())
if (needZeroPass(file))
workItems.push_back([this, &file] { executeFileMove(file); /*throw ThreadStopRequest*/ });
}
else
{
//synchronize folders *first* (see comment above "Multithreaded File Copy")
for (FolderPair& folder : conObj.subfolders())
if (pass == getPass(folder))
workItems.push_back([this, &folder, &workload, pass]
{
tryReportingError([&]{ synchronizeFolder(folder); }, acb_); //throw ThreadStopRequest
workload.addWorkItems(getFolderLevelWorkItems(pass, folder, workload));
});
else
foldersToInspect.push_back(&folder);
//synchronize files:
for (FilePair& file : conObj.files())
if (pass == getPass(file))
workItems.push_back([this, &file]
{
tryReportingError([&]{ synchronizeFile(file); }, acb_); //throw ThreadStopRequest
});
//synchronize symbolic links:
for (SymlinkPair& symlink : conObj.symlinks())
if (pass == getPass(symlink))
workItems.push_back([this, &symlink]
{
tryReportingError([&] { synchronizeLink(symlink); }, acb_); //throw ThreadStopRequest
});
}
if (!workItems.empty())
buckets.push_back(std::move(workItems));
}
return buckets;
}
/* __________________________
|Move algorithm, 0th pass|
--------------------------
1. loop over hierarchy and find "move targets" => remember required parent folders
2. create required folders hierarchically:
- name-clash with other file/symlink (=> obscure!): fall back to delete and copy
- source folder missing: child items already deleted by synchronizeFolder()
- ignored error: fall back to delete and copy (in phases 1 and 2)
3. start file move (via targets)
- name-clash with other folder/symlink (=> obscure!): fall back to delete and copy
- ErrorMoveUnsupported: fall back to delete and copy
- ignored error: fall back to delete and copy
__________________
|killer-scenarios|
------------------
propagate the following move sequences:
I) a -> a/a caveat syncing parent directory first leads to circular dependency!
II) a/a -> a caveat: fixing name clash will remove source!
III) c -> d caveat: move-sequence needs to be processed in correct order!
b -> c/b
a -> b/a */
template <SelectSide side>
void FolderPairSyncer::executeFileMoveImpl(FilePair& fileFrom, FilePair& fileTo) //throw ThreadStopRequest
{
assert(fileFrom.getMovePair() == &fileTo);
const bool fallBackCopyDelete = [&]
{
//creation of parent folder has failed earlier? => fall back to delete + copy
const FolderPair* parentMissing = nullptr; //let's be more specific: go up in hierarchy until first missing parent folder
for (const FolderPair* f = dynamic_cast<const FolderPair*>(&fileTo.parent()); f && f->isEmpty<side>(); f = dynamic_cast<const FolderPair*>(&f->parent()))
parentMissing = f;
if (parentMissing)
{
reportInfo(AFS::generateMoveErrorMsg(fileFrom.getAbstractPath<side>(), fileTo.getAbstractPath<side>()) + L"\n\n" +
replaceCpy(_("Parent folder %x is not existing."), L"%x", fmtPath(AFS::getDisplayPath(parentMissing->getAbstractPath<side>()))), acb_); //throw ThreadStopRequest
return true;
}
//name clash with folders/symlinks? obscure => fall back to delete + copy
if (haveNameClash<side>(fileTo, fileTo.parent().subfolders()) ||
haveNameClash<side>(fileTo, fileTo.parent().symlinks ()))
{
reportInfo(AFS::generateMoveErrorMsg(fileFrom.getAbstractPath<side>(), fileTo.getAbstractPath<side>()) + L"\n\n" +
replaceCpy(_("The name %x is already used by another item."), L"%x", fmtPath(fileTo.getItemName<side>())), acb_); //throw ThreadStopRequest
return true;
}
bool moveSupported = true;
const std::wstring errMsg = tryReportingError([&] //throw ThreadStopRequest
{
try
{
synchronizeFile(fileTo); //throw FileError, ErrorMoveUnsupported, ThreadStopRequest
}
catch (const ErrorMoveUnsupported& e)
{
acb_.logMessage(e.toString(), PhaseCallback::MsgType::info); //let user know that move operation is not supported, then fall back:
moveSupported = false;
}
}, acb_);
return !errMsg.empty() || !moveSupported; //move failed? We cannot allow to continue and have move source's parent directory deleted, messing up statistics!
}();
if (fallBackCopyDelete)
{
auto getStats = [&]() -> std::pair<int, int64_t>
{
SyncStatistics statSrc(fileFrom);
SyncStatistics statTrg(fileTo);
return {getCUD(statSrc) + getCUD(statTrg), statSrc.getBytesToProcess() + statTrg.getBytesToProcess()};
};
const auto [itemsBefore, bytesBefore] = getStats();
fileFrom.setMovePair(nullptr);
const auto [itemsAfter, bytesAfter] = getStats();
//fix statistics total to match "copy + delete"
acb_.updateDataTotal(itemsAfter - itemsBefore, bytesAfter - bytesBefore); //noexcept
}
}
void FolderPairSyncer::executeFileMove(FilePair& file) //throw ThreadStopRequest
{
const SyncOperation syncOp = file.getSyncOperation();
switch (syncOp)
{
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
if (FilePair* fileFrom = file.getMovePair())
{
if (syncOp == SO_MOVE_LEFT_TO)
executeFileMoveImpl<SelectSide::left>(*fileFrom, file); //throw ThreadStopRequest
else
executeFileMoveImpl<SelectSide::right>(*fileFrom, file); //throw ThreadStopRequest
}
else assert(false);
break;
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
case SO_DELETE_LEFT:
case SO_DELETE_RIGHT:
case SO_MOVE_LEFT_FROM: //don't try to move more than *once* per pair
case SO_MOVE_RIGHT_FROM: //
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
assert(false); //should have been filtered out by FolderPairSyncer::needZeroPass()
break;
}
}
//---------------------------------------------------------------------------------------------------------------
bool FolderPairSyncer::containsMoveTarget(const FolderPair& parent)
{
for (const FilePair& file : parent.files())
if (needZeroPass(file))
return true;
for (const FolderPair& subFolder : parent.subfolders())
if (containsMoveTarget(subFolder))
return true;
return false;
}
//0th pass: execute file moves (+ optional fallback to delete/copy in passes 1 and 2)
bool FolderPairSyncer::needZeroPass(const FolderPair& folder)
{
switch (folder.getSyncOperation())
{
case SO_CREATE_LEFT:
return containsMoveTarget(folder) && //recursive! watch perf!
!haveNameClash<SelectSide::left>(folder, folder.parent().files ()) && //name clash with files/symlinks? obscure => skip folder creation
!haveNameClash<SelectSide::left>(folder, folder.parent().symlinks()); // => move: fall back to delete + copy
case SO_CREATE_RIGHT:
return containsMoveTarget(folder) && //recursive! watch perf!
!haveNameClash<SelectSide::right>(folder, folder.parent().files ()) && //name clash with files/symlinks? obscure => skip folder creation
!haveNameClash<SelectSide::right>(folder, folder.parent().symlinks()); // => move: fall back to delete + copy
case SO_DO_NOTHING: //implies !isEmpty<side>(); see FolderPair::getSyncOperation()
case SO_UNRESOLVED_CONFLICT: //
case SO_EQUAL:
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
assert(!containsMoveTarget(folder) || (!folder.isEmpty<SelectSide::left>() && !folder.isEmpty<SelectSide::right>()));
//we're good to move contained items
break;
case SO_DELETE_LEFT: //not possible in the context of planning to move a child item, see FolderPair::getSyncOperation()
case SO_DELETE_RIGHT: //
assert(!containsMoveTarget(folder));
break;
case SO_OVERWRITE_LEFT: //
case SO_OVERWRITE_RIGHT: //
case SO_MOVE_LEFT_FROM: //
case SO_MOVE_RIGHT_FROM: //status not possible for folder
case SO_MOVE_LEFT_TO: //
case SO_MOVE_RIGHT_TO: //
assert(false);
break;
}
return false;
}
inline
bool FolderPairSyncer::needZeroPass(const FilePair& file)
{
switch (file.getSyncOperation())
{
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
return true;
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
case SO_DELETE_LEFT:
case SO_DELETE_RIGHT:
case SO_MOVE_LEFT_FROM: //don't try to move more than *once* per pair
case SO_MOVE_RIGHT_FROM: //
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
break;
}
return false;
}
//1st, 2nd pass benefits:
// - avoid disk space shortage: 1. delete files, 2. overwrite big with small files first
// - support change in type: overwrite file by directory, symlink by file, etc.
inline
FolderPairSyncer::PassNo FolderPairSyncer::getPass(const FilePair& file)
{
switch (file.getSyncOperation()) //evaluate comparison result and sync direction
{
case SO_DELETE_LEFT:
case SO_DELETE_RIGHT:
return PassNo::one;
case SO_OVERWRITE_LEFT:
return file.getFileSize<SelectSide::left>() > file.getFileSize<SelectSide::right>() ? PassNo::one : PassNo::two;
case SO_OVERWRITE_RIGHT:
return file.getFileSize<SelectSide::left>() < file.getFileSize<SelectSide::right>() ? PassNo::one : PassNo::two;
case SO_MOVE_LEFT_FROM: //
case SO_MOVE_RIGHT_FROM: // [!]
return PassNo::never;
case SO_MOVE_LEFT_TO: //
case SO_MOVE_RIGHT_TO: //make sure 2-step move is processed in second pass, after move *target* parent directory was created!
return PassNo::two;
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
return PassNo::two;
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
return PassNo::never;
}
assert(false);
return PassNo::never;
}
inline
FolderPairSyncer::PassNo FolderPairSyncer::getPass(const SymlinkPair& symlink)
{
switch (symlink.getSyncOperation()) //evaluate comparison result and sync direction
{
case SO_DELETE_LEFT:
case SO_DELETE_RIGHT:
return PassNo::one; //make sure to delete symlinks in first pass, and equally named file or dir in second pass: usecase "overwrite symlink with regular file"!
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
return PassNo::two;
case SO_MOVE_LEFT_FROM:
case SO_MOVE_RIGHT_FROM:
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
assert(false);
[[fallthrough]];
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
return PassNo::never;
}
assert(false);
return PassNo::never;
}
inline
FolderPairSyncer::PassNo FolderPairSyncer::getPass(const FolderPair& folder)
{
switch (folder.getSyncOperation()) //evaluate comparison result and sync direction
{
case SO_DELETE_LEFT:
case SO_DELETE_RIGHT:
return PassNo::one;
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
return PassNo::two;
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
case SO_MOVE_LEFT_FROM:
case SO_MOVE_RIGHT_FROM:
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
assert(false);
[[fallthrough]];
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
return PassNo::never;
}
assert(false);
return PassNo::never;
}
//---------------------------------------------------------------------------------------------------------------
inline
void FolderPairSyncer::synchronizeFile(FilePair& file) //throw FileError, ErrorMoveUnsupported, ThreadStopRequest
{
assert(isLocked(singleThread_));
const SyncOperation syncOp = file.getSyncOperation();
if (const SyncDirection syncDir = getEffectiveSyncDir(syncOp);
syncDir != SyncDirection::none)
{
if (syncDir == SyncDirection::left)
synchronizeFileInt<SelectSide::left>(file, syncOp);
else
synchronizeFileInt<SelectSide::right>(file, syncOp);
}
}
template <SelectSide sideTrg>
void FolderPairSyncer::synchronizeFileInt(FilePair& file, SyncOperation syncOp) //throw FileError, ErrorMoveUnsupported, ThreadStopRequest
{
constexpr SelectSide sideSrc = getOtherSide<sideTrg>;
DeletionHandler& delHandlerTrg = selectParam<sideTrg>(delHandlerLeft_, delHandlerRight_);
switch (syncOp)
{
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
{
if (auto parentFolder = dynamic_cast<const FolderPair*>(&file.parent()))
if (parentFolder->isEmpty<sideTrg>()) //BaseFolderPair OTOH is always non-empty and existing in this context => else: fatal error in zen::synchronize()
return; //if parent directory creation failed, there's no reason to show more errors!
const AbstractPath targetPath = file.getAbstractPath<sideTrg>();
const std::wstring& statusMsg = replaceCpy(txtCreatingFile_, L"%x", fmtPath(AFS::getDisplayPath(targetPath)));
reportInfo(std::wstring(statusMsg), acb_); //throw ThreadStopRequest
AsyncItemStatReporter statReporter(1, file.getFileSize<sideSrc>(), acb_);
try
{
const AFS::FileCopyResult result = copyFileWithCallback({file.getAbstractPath<sideSrc>(), file.getAttributes<sideSrc>()},
targetPath,
nullptr, //onDeleteTargetFile: nothing to delete
//if existing: undefined behavior! (e.g. fail/overwrite/auto-rename)
statReporter,
statusMsg); //throw FileError, ThreadStopRequest
statReporter.reportDelta(1, 0);
//update FilePair
file.setSyncedTo<sideTrg>(result.fileSize,
result.modTime, //target time set from source
result.modTime,
result.targetFilePrint,
result.sourceFilePrint,
false, file.isFollowedSymlink<sideSrc>());
if (result.errorModTime) //log only; no popup
acb_.logMessage(result.errorModTime->toString(), PhaseCallback::MsgType::warning); //throw ThreadStopRequest
}
catch (const FileError& e)
{
bool sourceExists = true;
try { sourceExists = parallel::itemExists(file.getAbstractPath<sideSrc>(), singleThread_); /*throw FileError*/ }
//abstract context => unclear which exception is more relevant/useless:
//e could be "item not found": doh; e2 devoid of any details after SFTP error: https://freefilesync.org/forum/viewtopic.php?t=7138#p24064
catch (const FileError& e2) { throw FileError(replaceCpy(e.toString(), L"\n\n", L'\n'), replaceCpy(e2.toString(), L"\n\n", L'\n')); }
//do not check on type (symlink, file, folder) -> if there is a type change, FFS should not be quiet about it!
if (!sourceExists)
{
reportItemInfo(txtSourceItemNotExist_, file.getAbstractPath<sideSrc>()); //throw ThreadStopRequest
statReporter.reportDelta(1, 0);
//even if the source item does not exist anymore, significant I/O work was done => report
file.removeItem<sideSrc>(); //source deleted meanwhile...nothing was done (logical point of view!)
//remove only *after* evaluating "file, sideSrc"!
}
else
throw;
}
}
break;
case SO_DELETE_LEFT:
case SO_DELETE_RIGHT:
{
AsyncItemStatReporter statReporter(1, 0, acb_);
if (file.isFollowedSymlink<sideTrg>())
delHandlerTrg.removeLinkWithCallback(file.getAbstractPath<sideTrg>(), file.getRelativePath<sideTrg>(),
false /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest
else
delHandlerTrg.removeFileWithCallback({file.getAbstractPath<sideTrg>(), file.getAttributes<sideTrg>()}, file.getRelativePath<sideTrg>(),
false /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest
file.removeItem<sideTrg>(); //update FilePair
}
break;
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
if (FilePair* fileFrom = file.getMovePair())
{
FilePair* fileTo = &file;
assert((fileFrom->getSyncOperation() == SO_MOVE_LEFT_FROM && fileTo->getSyncOperation() == SO_MOVE_LEFT_TO && sideTrg == SelectSide::left) ||
(fileFrom->getSyncOperation() == SO_MOVE_RIGHT_FROM && fileTo->getSyncOperation() == SO_MOVE_RIGHT_TO && sideTrg == SelectSide::right));
const AbstractPath pathFrom = fileFrom->getAbstractPath<sideTrg>();
const AbstractPath pathTo = fileTo ->getAbstractPath<sideTrg>();
reportItemInfo(txtMovingFileXtoY_, pathFrom, pathTo); //throw ThreadStopRequest
AsyncItemStatReporter statReporter(1, 0, acb_);
//already existing: undefined behavior! (e.g. fail/overwrite)
parallel::moveAndRenameItem(pathFrom, pathTo, singleThread_); //throw FileError, ErrorMoveUnsupported
statReporter.reportDelta(1, 0);
//update FilePair
assert(fileFrom->getFileSize<sideTrg>() == fileTo->getFileSize<sideSrc>());
fileTo->setSyncedTo<sideTrg>(fileTo ->getFileSize<sideSrc>(),
fileFrom->getLastWriteTime<sideTrg>(),
fileTo ->getLastWriteTime<sideSrc>(),
fileFrom->getFilePrint<sideTrg>(),
fileTo ->getFilePrint<sideSrc>(),
fileFrom->isFollowedSymlink<sideTrg>(),
fileTo ->isFollowedSymlink<sideSrc>());
fileFrom->removeItem<sideTrg>(); //remove only *after* evaluating "fileFrom, sideTrg"!
}
else (assert(false));
break;
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
{
//respect differences in case of source object:
const AbstractPath targetPathLogical = AFS::appendRelPath(file.parent().getAbstractPath<sideTrg>(), file.getItemName<sideSrc>());
AbstractPath targetPathResolvedOld = file.getAbstractPath<sideTrg>(); //support change in case when syncing to case-sensitive SFTP on Windows!
AbstractPath targetPathResolvedNew = targetPathLogical;
if (file.isFollowedSymlink<sideTrg>()) //follow link when updating file rather than delete it and replace with regular file!!!
targetPathResolvedOld = targetPathResolvedNew = parallel::getSymlinkResolvedPath(file.getAbstractPath<sideTrg>(), singleThread_); //throw FileError
const std::wstring& statusMsg = replaceCpy(txtUpdatingFile_, L"%x", fmtPath(AFS::getDisplayPath(targetPathResolvedOld)));
reportInfo(std::wstring(statusMsg), acb_); //throw ThreadStopRequest
AsyncItemStatReporter statReporter(1, file.getFileSize<sideSrc>(), acb_);
if (file.isFollowedSymlink<sideTrg>()) //since we follow the link, we need to sync case sensitivity of the link manually!
if (!file.hasEquivalentItemNames())
//already existing: undefined behavior! (e.g. fail/overwrite)
parallel::moveAndRenameItem(file.getAbstractPath<sideTrg>(), targetPathLogical, singleThread_); //throw FileError, (ErrorMoveUnsupported)
auto onDeleteTargetFile = [&] //delete target at appropriate time
{
assert(isLocked(singleThread_));
FileAttributes followedTargetAttr = file.getAttributes<sideTrg>();
followedTargetAttr.isFollowedSymlink = false;
if (file.isFollowedSymlink<sideTrg>())
delHandlerTrg.removeLinkWithCallback(targetPathResolvedOld, file.getRelativePath<sideTrg>(),
true /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest
else
delHandlerTrg.removeFileWithCallback({targetPathResolvedOld, followedTargetAttr}, file.getRelativePath<sideTrg>(),
true /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest
//file.removeItem<sideTrg>(); -> doesn't make sense for isFollowedSymlink(); "file, sideTrg" evaluated below!
};
const AFS::FileCopyResult result = copyFileWithCallback({file.getAbstractPath<sideSrc>(), file.getAttributes<sideSrc>()},
targetPathResolvedNew,
onDeleteTargetFile,
statReporter,
statusMsg); //throw FileError, ThreadStopRequest
statReporter.reportDelta(1, 0);
//we model "delete + copy" as ONE logical operation
//update FilePair
file.setSyncedTo<sideTrg>(result.fileSize,
result.modTime, //target time set from source
result.modTime,
result.targetFilePrint,
result.sourceFilePrint,
file.isFollowedSymlink<sideTrg>(),
file.isFollowedSymlink<sideSrc>());
if (result.errorModTime) //log only; no popup
acb_.logMessage(result.errorModTime->toString(), PhaseCallback::MsgType::warning); //throw ThreadStopRequest
}
break;
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
//harmonize with file_hierarchy.cpp::getSyncOpDescription!!
reportInfo(replaceCpy(replaceCpy(txtRenamingFileXtoY_, L"%x", fmtPath(AFS::getDisplayPath(file.getAbstractPath<sideTrg>()))),
L"%y", fmtPath(file.getItemName<sideSrc>())), acb_); //throw ThreadStopRequest
{
AsyncItemStatReporter statReporter(1, 0, acb_);
if (!file.hasEquivalentItemNames())
//already existing: undefined behavior! (e.g. fail/overwrite)
parallel::moveAndRenameItem(file.getAbstractPath<sideTrg>(), //throw FileError, (ErrorMoveUnsupported)
AFS::appendRelPath(file.parent().getAbstractPath<sideTrg>(), file.getItemName<sideSrc>()), singleThread_);
else
assert(false);
#if 0 //changing file time without copying content is not justified after CompareVariant::size finds "equal" files!
//Bonus: some devices don't support setting (precise) file times anyway, e.g. FAT or MTP!
if (file.getLastWriteTime<sideTrg>() != file.getLastWriteTime<sideSrc>())
//- no need to call sameFileTime() or respect 2 second FAT/FAT32 precision in this comparison
//- do NOT read *current* source file time, but use buffered value which corresponds to time of comparison!
parallel::setModTime(file.getAbstractPath<sideTrg>(), file.getLastWriteTime<sideSrc>()); //throw FileError
#endif
statReporter.reportDelta(1, 0);
//-> both sides *should* be completely equal now...
assert(file.getFileSize<sideTrg>() == file.getFileSize<sideSrc>());
file.setSyncedTo<sideTrg>(file.getFileSize<sideSrc>(),
file.getLastWriteTime <sideTrg>(),
file.getLastWriteTime <sideSrc>(),
file.getFilePrint <sideTrg>(),
file.getFilePrint <sideSrc>(),
file.isFollowedSymlink<sideTrg>(),
file.isFollowedSymlink<sideSrc>());
}
break;
case SO_MOVE_LEFT_FROM: //use SO_MOVE_LEFT_TO/SO_MOVE_RIGHT_TO to execute move:
case SO_MOVE_RIGHT_FROM: //=> makes sure parent directory has been created
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
assert(false); //should have been filtered out by FolderPairSyncer::getPass()
return; //no update on processed data!
}
}
inline
void FolderPairSyncer::synchronizeLink(SymlinkPair& symlink) //throw FileError, ThreadStopRequest
{
assert(isLocked(singleThread_));
const SyncOperation syncOp = symlink.getSyncOperation();
if (const SyncDirection syncDir = getEffectiveSyncDir(syncOp);
syncDir != SyncDirection::none)
{
if (syncDir == SyncDirection::left)
synchronizeLinkInt<SelectSide::left>(symlink, syncOp);
else
synchronizeLinkInt<SelectSide::right>(symlink, syncOp);
}
}
template <SelectSide sideTrg>
void FolderPairSyncer::synchronizeLinkInt(SymlinkPair& symlink, SyncOperation syncOp) //throw FileError, ThreadStopRequest
{
constexpr SelectSide sideSrc = getOtherSide<sideTrg>;
DeletionHandler& delHandlerTrg = selectParam<sideTrg>(delHandlerLeft_, delHandlerRight_);
switch (syncOp)
{
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
{
if (auto parentFolder = dynamic_cast<const FolderPair*>(&symlink.parent()))
if (parentFolder->isEmpty<sideTrg>()) //BaseFolderPair OTOH is always non-empty and existing in this context => else: fatal error in zen::synchronize()
return; //if parent directory creation failed, there's no reason to show more errors!
const AbstractPath targetPath = symlink.getAbstractPath<sideTrg>();
reportItemInfo(txtCreatingLink_, targetPath); //throw ThreadStopRequest
AsyncItemStatReporter statReporter(1, 0, acb_);
try
{
parallel::copySymlink(symlink.getAbstractPath<sideSrc>(), targetPath, copyFilePermissions_, singleThread_); //throw FileError
statReporter.reportDelta(1, 0);
//update SymlinkPair
symlink.setSyncedTo<sideTrg>(symlink.getLastWriteTime<sideSrc>(), //target time set from source
symlink.getLastWriteTime<sideSrc>());
}
catch (const FileError& e)
{
bool sourceExists = true;
try { sourceExists = parallel::itemExists(symlink.getAbstractPath<sideSrc>(), singleThread_); /*throw FileError*/ }
//abstract context => unclear which exception is more relevant/useless:
catch (const FileError& e2) { throw FileError(replaceCpy(e.toString(), L"\n\n", L'\n'), replaceCpy(e2.toString(), L"\n\n", L'\n')); }
//do not check on type (symlink, file, folder) -> if there is a type change, FFS should not be quiet about it!
if (!sourceExists)
{
reportItemInfo(txtSourceItemNotExist_, symlink.getAbstractPath<sideSrc>()); //throw ThreadStopRequest
//even if the source item does not exist anymore, significant I/O work was done => report
statReporter.reportDelta(1, 0);
symlink.removeItem<sideSrc>(); //source deleted meanwhile...nothing was done (logical point of view!)
}
else
throw;
}
}
break;
case SO_DELETE_LEFT:
case SO_DELETE_RIGHT:
{
AsyncItemStatReporter statReporter(1, 0, acb_);
delHandlerTrg.removeLinkWithCallback(symlink.getAbstractPath<sideTrg>(), symlink.getRelativePath<sideTrg>(),
false /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest
symlink.removeItem<sideTrg>(); //update SymlinkPair
}
break;
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
{
reportItemInfo(txtUpdatingLink_, symlink.getAbstractPath<sideTrg>()); //throw ThreadStopRequest
AsyncItemStatReporter statReporter(1, 0, acb_);
delHandlerTrg.removeLinkWithCallback(symlink.getAbstractPath<sideTrg>(), symlink.getRelativePath<sideTrg>(),
true /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest
//symlink.removeItem<sideTrg>(); -> "symlink, sideTrg" evaluated below!
parallel::copySymlink(symlink.getAbstractPath<sideSrc>(),
AFS::appendRelPath(symlink.parent().getAbstractPath<sideTrg>(), symlink.getItemName<sideSrc>()), //respect differences in case of source object
copyFilePermissions_, singleThread_); //throw FileError
statReporter.reportDelta(1, 0); //we model "delete + copy" as ONE logical operation
//update SymlinkPair
symlink.setSyncedTo<sideTrg>(symlink.getLastWriteTime<sideSrc>(), //target time set from source
symlink.getLastWriteTime<sideSrc>());
}
break;
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
reportInfo(replaceCpy(replaceCpy(txtRenamingLinkXtoY_, L"%x", fmtPath(AFS::getDisplayPath(symlink.getAbstractPath<sideTrg>()))),
L"%y", fmtPath(symlink.getItemName<sideSrc>())), acb_); //throw ThreadStopRequest
{
AsyncItemStatReporter statReporter(1, 0, acb_);
if (!symlink.hasEquivalentItemNames())
//already existing: undefined behavior! (e.g. fail/overwrite)
parallel::moveAndRenameItem(symlink.getAbstractPath<sideTrg>(), //throw FileError, (ErrorMoveUnsupported)
AFS::appendRelPath(symlink.parent().getAbstractPath<sideTrg>(), symlink.getItemName<sideSrc>()), singleThread_);
else
assert(false);
statReporter.reportDelta(1, 0);
//-> both sides *should* be completely equal now...
symlink.setSyncedTo<sideTrg>(symlink.getLastWriteTime<sideTrg>(), //target time set from source
symlink.getLastWriteTime<sideSrc>());
}
break;
case SO_MOVE_LEFT_FROM:
case SO_MOVE_RIGHT_FROM:
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
assert(false); //should have been filtered out by FolderPairSyncer::getPass()
return; //no update on processed data!
}
}
inline
void FolderPairSyncer::synchronizeFolder(FolderPair& folder) //throw FileError, ThreadStopRequest
{
assert(isLocked(singleThread_));
const SyncOperation syncOp = folder.getSyncOperation();
if (const SyncDirection syncDir = getEffectiveSyncDir(syncOp);
syncDir != SyncDirection::none)
{
if (syncDir == SyncDirection::left)
synchronizeFolderInt<SelectSide::left>(folder, syncOp);
else
synchronizeFolderInt<SelectSide::right>(folder, syncOp);
}
}
template <SelectSide sideTrg>
void FolderPairSyncer::synchronizeFolderInt(FolderPair& folder, SyncOperation syncOp) //throw FileError, ThreadStopRequest
{
constexpr SelectSide sideSrc = getOtherSide<sideTrg>;
DeletionHandler& delHandlerTrg = selectParam<sideTrg>(delHandlerLeft_, delHandlerRight_);
switch (syncOp)
{
case SO_CREATE_LEFT:
case SO_CREATE_RIGHT:
{
if (auto parentFolder = dynamic_cast<const FolderPair*>(&folder.parent()))
if (parentFolder->isEmpty<sideTrg>()) //BaseFolderPair OTOH is always non-empty and existing in this context => else: fatal error in zen::synchronize()
return; //if parent directory creation failed, there's no reason to show more errors!
const AbstractPath targetPath = folder.getAbstractPath<sideTrg>();
reportItemInfo(txtCreatingFolder_, targetPath); //throw ThreadStopRequest
//shallow-"copying" a folder might not fail if source is missing, so we need to check this first:
if (parallel::itemExists(folder.getAbstractPath<sideSrc>(), singleThread_)) //throw FileError
{
AsyncItemStatReporter statReporter(1, 0, acb_);
try
{
//already existing: fail
parallel::copyNewFolder(folder.getAbstractPath<sideSrc>(), targetPath, copyFilePermissions_, singleThread_); //throw FileError
}
catch (FileError&)
{
bool folderAlreadyExists = false;
try { folderAlreadyExists = parallel::getItemType(targetPath, singleThread_) == AFS::ItemType::folder; } /*throw FileError*/ catch (FileError&) {}
//previous exception is more relevant; good enough? https://freefilesync.org/forum/viewtopic.php?t=5266
if (!folderAlreadyExists)
throw;
}
statReporter.reportDelta(1, 0);
//update FolderPair
folder.setSyncedTo<sideTrg>(false, //isSymlinkTrg
folder.isFollowedSymlink<sideSrc>());
}
else //source deleted meanwhile...
{
reportItemInfo(txtSourceItemNotExist_, folder.getAbstractPath<sideSrc>()); //throw ThreadStopRequest
//attention when fixing statistics due to missing folder: child items may be scheduled for move, so deletion will have move-references flip back to copy + delete!
const SyncStatistics statsBefore(folder.base()); //=> don't bother considering individual move operations, just calculate over the whole tree
folder.clearFiles(); //
folder.clearSymlinks(); //update FolderPair
folder.clearSubfolders(); //
folder.removeItem<sideSrc>(); //
const SyncStatistics statsAfter(folder.base());
acb_.updateDataProcessed(1, 0); //even if the source item does not exist anymore, significant I/O work was done => report
acb_.updateDataTotal(getCUD(statsAfter) - getCUD(statsBefore) + 1, statsAfter.getBytesToProcess() - statsBefore.getBytesToProcess()); //noexcept
}
}
break;
case SO_DELETE_LEFT:
case SO_DELETE_RIGHT:
{
const SyncStatistics subStats(folder); //counts sub-objects only!
AsyncItemStatReporter statReporter(1 + getCUD(subStats), subStats.getBytesToProcess(), acb_);
if (folder.isFollowedSymlink<sideTrg>())
delHandlerTrg.removeLinkWithCallback(folder.getAbstractPath<sideTrg>(), folder.getRelativePath<sideTrg>(),
false /*beforeOverwrite*/, statReporter, singleThread_); //throw FileError, ThreadStopRequest
else
delHandlerTrg.removeDirWithCallback(folder.getAbstractPath<sideTrg>(), folder.getRelativePath<sideTrg>(),
statReporter, singleThread_); //throw FileError, ThreadStopRequest
//TODO: implement parallel folder deletion
folder.clearFiles(); //
folder.clearSymlinks(); //update FolderPair
folder.clearSubfolders(); //
folder.removeItem<sideTrg>(); //
}
break;
case SO_RENAME_LEFT:
case SO_RENAME_RIGHT:
reportInfo(replaceCpy(replaceCpy(txtRenamingFolderXtoY_, L"%x", fmtPath(AFS::getDisplayPath(folder.getAbstractPath<sideTrg>()))),
L"%y", fmtPath(folder.getItemName<sideSrc>())), acb_); //throw ThreadStopRequest
{
AsyncItemStatReporter statReporter(1, 0, acb_);
if (!folder.hasEquivalentItemNames())
//already existing: undefined behavior! (e.g. fail/overwrite)
parallel::moveAndRenameItem(folder.getAbstractPath<sideTrg>(), //throw FileError, (ErrorMoveUnsupported)
AFS::appendRelPath(folder.parent().getAbstractPath<sideTrg>(), folder.getItemName<sideSrc>()), singleThread_);
else
assert(false);
statReporter.reportDelta(1, 0);
//-> both sides *should* be completely equal now...
folder.setSyncedTo<sideTrg>(folder.isFollowedSymlink<sideTrg>(),
folder.isFollowedSymlink<sideSrc>());
}
break;
case SO_OVERWRITE_LEFT:
case SO_OVERWRITE_RIGHT:
case SO_MOVE_LEFT_FROM:
case SO_MOVE_RIGHT_FROM:
case SO_MOVE_LEFT_TO:
case SO_MOVE_RIGHT_TO:
case SO_DO_NOTHING:
case SO_EQUAL:
case SO_UNRESOLVED_CONFLICT:
assert(false); //should have been filtered out by FolderPairSyncer::getPass()
return; //no update on processed data!
}
}
//###########################################################################################
//returns current attributes of source file
AFS::FileCopyResult FolderPairSyncer::copyFileWithCallback(const FileDescriptor& sourceDescr,
const AbstractPath& targetPath,
const std::function<void()>& onDeleteTargetFile /*throw X*/,
AsyncItemStatReporter& statReporter /*throw ThreadStopRequest*/,
const std::wstring& statusMsg) //throw FileError, ThreadStopRequest, X
{
const AbstractPath& sourcePath = sourceDescr.path;
const AFS::StreamAttributes sourceAttr{sourceDescr.attr.modTime, sourceDescr.attr.fileSize, sourceDescr.attr.filePrint};
auto copyOperation = [&](const AbstractPath& sourcePathTmp)
{
PercentStatReporter percentReporter(statusMsg, sourceDescr.attr.fileSize, statReporter);
//already existing + no onDeleteTargetFile: undefined behavior! (e.g. fail/overwrite/auto-rename)
const AFS::FileCopyResult result = parallel::copyFileTransactional(sourcePathTmp, sourceAttr, //throw FileError, ErrorFileLocked, ThreadStopRequest, X
targetPath,
copyFilePermissions_,
failSafeFileCopy_, [&]
{
if (onDeleteTargetFile) //running *outside* singleThread_ lock! => onDeleteTargetFile-callback expects lock being held:
{
std::lock_guard dummy(singleThread_);
onDeleteTargetFile(); //throw X
}
},
[&](int64_t bytesDelta) //callback runs *outside* singleThread_ lock! => fine
{
percentReporter.updateDeltaAndStatus(bytesDelta); //throw ThreadStopRequest
interruptionPoint(); //throw ThreadStopRequest => not reliably covered by PercentStatReporter::updateDeltaAndStatus()!
},
singleThread_);
//#################### Verification #############################
if (verifyCopiedFiles_)
{
reportItemInfo(txtVerifyingFile_, targetPath); //throw ThreadStopRequest
//delete target if verification fails
ZEN_ON_SCOPE_FAIL(try { parallel::removeFilePlain(targetPath, singleThread_); }
catch (const FileError& e) { statReporter.logMessage(e.toString(), PhaseCallback::MsgType::error); /*throw ThreadStopRequest*/ });
//callback runs *outside* singleThread_ lock! => fine
auto verifyCallback = [&](int64_t bytesDelta) { interruptionPoint(); }; //throw ThreadStopRequest
parallel::verifyFiles(sourcePathTmp, targetPath, verifyCallback, singleThread_); //throw FileError, ThreadStopRequest
}
//#################### /Verification #############################
return result;
};
return copyOperation(sourcePath); //throw FileError, (ErrorFileLocked), ThreadStopRequest
}
//###########################################################################################
template <SelectSide side>
bool checkBaseFolderStatus(BaseFolderPair& baseFolder, PhaseCallback& callback /*throw X*/)
{
const AbstractPath folderPath = baseFolder.getAbstractPath<side>();
switch (baseFolder.getFolderStatus<side>())
{
case BaseFolderStatus::existing:
{
const std::wstring errMsg = tryReportingError([&]
{
AFS::getItemType(folderPath); //throw FileError
}, callback); //throw X
if (!errMsg.empty())
return false;
}
break;
case BaseFolderStatus::notExisting:
{
bool folderExisting = false;
const std::wstring errMsg = tryReportingError([&]
{
folderExisting = AFS::itemExists(folderPath); //throw FileError
}, callback); //throw X
if (!errMsg.empty())
return false;
if (folderExisting) //=> somebody else created it: problem?
{
/* Is it possible we're catching a "false positive" here, could FFS have created the directory indirectly after comparison?
1. deletion handling: recycler -> no, temp directory created only at first deletion
2. deletion handling: versioning -> "
3. log file creates containing folder -> no, log only created in batch mode, and only *before* comparison
4. yes, could be us! e.g. multiple folder pairs to non-yet-existing target folder => too obscure!? */
callback.reportFatalError(replaceCpy(_("The folder %x is already existing, but was not found earlier during comparison."),
L"%x", fmtPath(AFS::getDisplayPath(folderPath)))); //throw X
return false;
}
}
break;
case BaseFolderStatus::failure:
//e.g. TEMPORARY network drop! base directory not found during comparison
//=> sync-directions are based on false assumptions! Abort.
callback.reportFatalError(replaceCpy(_("Skipping folder pair because %x could not be accessed during comparison."),
L"%x", fmtPath(AFS::getDisplayPath(folderPath)))); //throw X
return false;
}
return true;
}
template <SelectSide side> //create base directories first (if not yet existing) -> no symlink or attribute copying!
bool createBaseFolder(BaseFolderPair& baseFolder, bool copyFilePermissions, PhaseCallback& callback /*throw X*/) //return false if fatal error occurred
{
switch (baseFolder.getFolderStatus<side>())
{
case BaseFolderStatus::existing:
break;
case BaseFolderStatus::notExisting:
{
//create target directory: user presumably ignored warning "dir not yet existing" in order to have it created automatically
const AbstractPath folderPath = baseFolder.getAbstractPath<side>();
constexpr SelectSide sideSrc = getOtherSide<side>;
const std::wstring errMsg = tryReportingError([&]
{
if (baseFolder.getFolderStatus<sideSrc>() == BaseFolderStatus::existing) //copy file permissions
{
if (const std::optional<AbstractPath> parentPath = AFS::getParentPath(folderPath))
AFS::createFolderIfMissingRecursion(*parentPath); //throw FileError
AFS::copyNewFolder(baseFolder.getAbstractPath<sideSrc>(), folderPath, copyFilePermissions); //throw FileError
}
else
AFS::createFolderIfMissingRecursion(folderPath); //throw FileError
assert(baseFolder.getFolderStatus<sideSrc>() != BaseFolderStatus::failure);
baseFolder.setFolderStatus<side>(BaseFolderStatus::existing); //update our model!
}, callback); //throw X
return errMsg.empty();
}
case BaseFolderStatus::failure:
assert(false); //already skipped after checkBaseFolderStatus()
break;
}
return true;
}
}
void fff::synchronize(const std::chrono::system_clock::time_point& syncStartTime,
bool verifyCopiedFiles,
bool copyLockedFiles,
bool copyFilePermissions,
bool failSafeFileCopy,
bool runWithBackgroundPriority,
const std::vector<FolderPairSyncCfg>& syncConfig,
FolderComparison& folderCmp,
WarningDialogs& warnings,
ProcessCallback& callback /*throw X*/) //throw X
{
//PERF_START;
if (syncConfig.size() != folderCmp.size())
throw std::logic_error(std::string(__FILE__) + '[' + numberTo<std::string>(__LINE__) + "] Contract violation!");
//aggregate basic information
std::vector<SyncStatistics> folderPairStats;
{
int itemsTotal = 0;
int64_t bytesTotal = 0;
for (const BaseFolderPair& baseFolder : asRange(folderCmp))
{
SyncStatistics fpStats(baseFolder);
itemsTotal += getCUD(fpStats);
bytesTotal += fpStats.getBytesToProcess();
folderPairStats.push_back(fpStats);
}
//inform about the total amount of data that will be processed from now on
//keep at beginning so that all gui elements are initialized properly
callback.initNewPhase(itemsTotal, //throw X
bytesTotal,
ProcessPhase::sync);
}
//-------------------------------------------------------------------------------
//prevent operating system going into sleep state
std::optional<SetProcessPriority> noStandby;
try
{
noStandby.emplace(runWithBackgroundPriority ? ProcessPriority::background : ProcessPriority::normal); //throw FileError
}
catch (const FileError& e) //failure is not critical => log only
{
callback.logMessage(e.toString(), PhaseCallback::MsgType::warning); //throw X
}
//-------------------execute basic checks all at once BEFORE starting sync--------------------------------------
std::vector<unsigned char /*we really want bool*/> skipFolderPair(folderCmp.size(), false); //folder pairs may be skipped after fatal errors were found
std::vector<std::tuple<const BaseFolderPair*, int /*conflict count*/, std::vector<std::wstring>>> checkUnresolvedConflicts;
std::vector<std::tuple<const BaseFolderPair*, SelectSide, bool /*write access*/>> checkBaseFolderRaceCondition;
std::vector<std::pair<AbstractPath, AbstractPath>> checkSignificantDiffPairs;
std::vector<std::pair<AbstractPath, std::pair<int64_t, int64_t>>> checkDiskSpaceMissing; //base folder / space required / space available
std::set<AbstractPath> checkVersioningPaths;
std::vector<std::pair<AbstractPath, const PathFilter*>> checkVersioningBasePaths; //hard filter creates new logical hierarchies for otherwise equal AbstractPath...
std::set<AbstractPath> checkVersioningLimitPaths;
//------------------- start checking folder pairs -------------------
//skip incomplete folder pairs with only one folder selected, or all <empty> => don't support "deletion via <empty> source folder"
{
bool haveFullPair = false;
std::wstring partialPairList;
for (const BaseFolderPair& baseFolder : asRange(folderCmp))
{
const AbstractPath& folderPathL = baseFolder.getAbstractPath<SelectSide::left >();
const AbstractPath& folderPathR = baseFolder.getAbstractPath<SelectSide::right>();
if (AFS::isNullPath(folderPathL) != AFS::isNullPath(folderPathR))
{
partialPairList += L"\n" +
(AFS::isNullPath(folderPathL) ? L"<" + _("empty") + L">" : AFS::getDisplayPath(folderPathL)) + L" | " +
(AFS::isNullPath(folderPathR) ? L"<" + _("empty") + L">" : AFS::getDisplayPath(folderPathR));
}
else if (!AFS::isNullPath(folderPathL))
haveFullPair = true;
}
//error if: partial pairs or all empty -> single-folder comparison scenario doesn't include synchronization
if (!partialPairList.empty() || !haveFullPair)
callback.reportFatalError(trimCpy(_("A folder input field is empty.") + L" \n\n" +
_("Please select both left and right folders for synchronization.") + L"\n" + partialPairList)); //throw X
//"skipFolderPair[folderIndex] = true" will be set below
}
for (size_t folderIndex = 0; folderIndex < folderCmp.size(); ++folderIndex)
{
BaseFolderPair& baseFolder = folderCmp[folderIndex].ref();
const FolderPairSyncCfg& folderPairCfg = syncConfig[folderIndex];
const SyncStatistics& folderPairStat = folderPairStats[folderIndex];
//=============== start with checks that may SKIP folder pairs ===============
//============================================================================
//skip incomplete folder pairs (fatal error already reported above)
if (AFS::isNullPath(baseFolder.getAbstractPath<SelectSide::left >()) ||
AFS::isNullPath(baseFolder.getAbstractPath<SelectSide::right>()))
{
skipFolderPair[folderIndex] = true;
continue;
}
//exclude a few pathological cases
if (baseFolder.getAbstractPath<SelectSide::left >() ==
baseFolder.getAbstractPath<SelectSide::right>())
{
skipFolderPair[folderIndex] = true;
continue;
}
//skip folder pair if there is nothing to do (except when DB files need to be updated for two-way mode and move-detection)
//=> avoid redundant errors in checkBaseFolderStatus() if base folder existence test failed during comparison
if (getCUD(folderPairStat) == 0 && !folderPairCfg.saveSyncDB)
{
skipFolderPair[folderIndex] = true;
continue;
}
//check for network drops after comparison
// - convenience: exit sync right here instead of showing tons of errors during file copy
// - early failure! there's no point in evaluating subsequent warnings
if (!checkBaseFolderStatus<SelectSide::left >(baseFolder, callback) ||
!checkBaseFolderStatus<SelectSide::right>(baseFolder, callback))
{
skipFolderPair[folderIndex] = true;
continue;
}
//allow propagation of deletions only from *empty* or *existing* source folder:
auto sourceFolderMissing = [&](const AbstractPath& baseFolderPath, BaseFolderStatus folderStatus) //we need to evaluate existence status from time of comparison!
{
//PERMANENT network drop: avoid data loss when source directory is not found AND user chose to ignore errors (else we wouldn't arrive here)
if (folderPairStat.deleteCount() > 0) //check deletions only... (respect filtered items!)
//folderPairStat.conflictCount() == 0 && -> there COULD be conflicts for <Two way> variant if directory existence check fails, but loading sync.ffs_db succeeds
//https://sourceforge.net/tracker/?func=detail&atid=1093080&aid=3531351&group_id=234430 -> fixed, but still better not consider conflicts!
if (folderStatus != BaseFolderStatus::existing) //avoid race-condition: we need to evaluate existence status from time of comparison!
{
callback.reportFatalError(replaceCpy(_("Source folder %x not found."), L"%x", fmtPath(AFS::getDisplayPath(baseFolderPath))));
return true;
}
return false;
};
if (sourceFolderMissing(baseFolder.getAbstractPath<SelectSide::left >(), baseFolder.getFolderStatus<SelectSide:: left>()) ||
sourceFolderMissing(baseFolder.getAbstractPath<SelectSide::right>(), baseFolder.getFolderStatus<SelectSide::right>()))
{
skipFolderPair[folderIndex] = true;
continue;
}
//check if user-defined directory for deletion was specified
const AbstractPath versioningFolderPath = createAbstractPath(folderPairCfg.versioningFolderPhrase);
if (folderPairCfg.handleDeletion == DeletionVariant::versioning)
if (AFS::isNullPath(versioningFolderPath))
{
callback.reportFatalError(_("Please enter a target folder.")); //user should never see this: already checked in SyncCfgDialog
skipFolderPair[folderIndex] = true;
continue;
}
//================= Warnings (*after* folder pair skips) =================
//========================================================================
//prepare conflict preview:
if (folderPairStat.conflictCount() > 0)
checkUnresolvedConflicts.emplace_back(&baseFolder, folderPairStat.conflictCount(), folderPairStat.getConflictsPreview());
//prepare: check if some files are used by multiple pairs in read/write access
const bool writeLeft = folderPairStat.createCount<SelectSide::left>() +
folderPairStat.updateCount<SelectSide::left>() +
folderPairStat.deleteCount<SelectSide::left>() > 0;
const bool writeRight = folderPairStat.createCount<SelectSide::right>() +
folderPairStat.updateCount<SelectSide::right>() +
folderPairStat.deleteCount<SelectSide::right>() > 0;
checkBaseFolderRaceCondition.emplace_back(&baseFolder, SelectSide::left, writeLeft);
checkBaseFolderRaceCondition.emplace_back(&baseFolder, SelectSide::right, writeRight);
//prepare: check if versioning path itself will be synchronized (and was not excluded via filter)
if (folderPairCfg.handleDeletion == DeletionVariant::versioning)
checkVersioningPaths.insert(versioningFolderPath);
checkVersioningBasePaths.emplace_back(baseFolder.getAbstractPath<SelectSide::left >(), &baseFolder.getFilter());
checkVersioningBasePaths.emplace_back(baseFolder.getAbstractPath<SelectSide::right>(), &baseFolder.getFilter());
//prepare: versioning folder paths differing only in case
if (folderPairCfg.handleDeletion == DeletionVariant::versioning &&
folderPairCfg.versioningStyle != VersioningStyle::replace)
if (folderPairCfg.versionMaxAgeDays > 0 || folderPairCfg.versionCountMax > 0) //same check as in applyVersioningLimit()
checkVersioningLimitPaths.insert(versioningFolderPath);
//check if more than 50% of total number of files/dirs will be created/overwritten/deleted
if (significantDifferenceDetected(folderPairStat))
checkSignificantDiffPairs.emplace_back(baseFolder.getAbstractPath<SelectSide::left >(),
baseFolder.getAbstractPath<SelectSide::right>());
//check for sufficient free diskspace (folderPath might not yet exist!)
auto checkSpace = [&](const AbstractPath& baseFolderPath, int64_t minSpaceNeeded)
{
if (minSpaceNeeded > 0)
try
{
const int64_t freeSpace = AFS::getFreeDiskSpace(baseFolderPath); //throw FileError, returns < 0 if not available
if (0 <= freeSpace &&
freeSpace < minSpaceNeeded)
checkDiskSpaceMissing.push_back({baseFolderPath, {minSpaceNeeded, freeSpace}});
}
catch (const FileError& e) //not critical => log only
{
callback.logMessage(e.toString(), PhaseCallback::MsgType::warning); //throw X
}
};
const std::pair<int64_t, int64_t> spaceNeeded = MinimumDiskSpaceNeeded::calculate(baseFolder);
if (baseFolder.getFolderStatus<SelectSide::left >() != BaseFolderStatus::failure) checkSpace(baseFolder.getAbstractPath<SelectSide::left >(), spaceNeeded.first);
if (baseFolder.getFolderStatus<SelectSide::right>() != BaseFolderStatus::failure) checkSpace(baseFolder.getAbstractPath<SelectSide::right>(), spaceNeeded.second);
}
//--------------------------------------------------------------------------------------
//check if unresolved conflicts exist
if (!checkUnresolvedConflicts.empty())
{
//distribute CONFLICTS_PREVIEW_MAX over all pairs, not *per* pair, or else log size with many folder pairs can blow up!
std::vector<std::vector<std::wstring>> conflictPreviewTrim(checkUnresolvedConflicts.size());
size_t previewRemain = CONFLICTS_PREVIEW_MAX;
for (size_t i = 0; ; ++i)
{
const size_t previewRemainOld = previewRemain;
for (size_t j = 0; j < checkUnresolvedConflicts.size(); ++j)
{
const auto& [baseFolder, conflictCount, conflictPreview] = checkUnresolvedConflicts[j];
if (i < conflictPreview.size())
{
conflictPreviewTrim[j].push_back(conflictPreview[i]);
if (--previewRemain == 0)
goto break2; //sigh
}
}
if (previewRemain == previewRemainOld)
break;
}
break2:
std::wstring msg = _("The following items have unresolved conflicts and will not be synchronized:");
auto itPrevi = conflictPreviewTrim.begin();
for (const auto& [baseFolder, conflictCount, conflictPreview] : checkUnresolvedConflicts)
{
msg += L"\n\n" + _("Folder pair:") + L' ' +
AFS::getDisplayPath(baseFolder->getAbstractPath<SelectSide::left >()) + L" <-> " +
AFS::getDisplayPath(baseFolder->getAbstractPath<SelectSide::right>());
for (const std::wstring& conflictMsg : *itPrevi)
msg += L'\n' + conflictMsg;
if (makeUnsigned(conflictCount) > itPrevi->size())
msg += L"\n [...] " + replaceCpy(_P("Showing %y of 1 item", "Showing %y of %x items", conflictCount), //%x used as plural form placeholder!
L"%y", formatNumber(itPrevi->size()));
++itPrevi;
}
callback.reportWarning(msg, warnings.warnUnresolvedConflicts); //throw X
}
//check if user accidentally selected wrong directories for sync
if (!checkSignificantDiffPairs.empty())
{
std::wstring msg = _("The following folders are significantly different. Please check that the correct folders are selected for synchronization.");
for (const auto& [folderPathL, folderPathR] : checkSignificantDiffPairs)
msg += L"\n\n" +
AFS::getDisplayPath(folderPathL) + L" <-> " + L'\n' +
AFS::getDisplayPath(folderPathR);
callback.reportWarning(msg, warnings.warnSignificantDifference); //throw X
}
//check for sufficient free diskspace
if (!checkDiskSpaceMissing.empty())
{
std::wstring msg = _("Not enough free disk space available in:");
for (const auto& [folderPath, space] : checkDiskSpaceMissing)
msg += L"\n\n" + AFS::getDisplayPath(folderPath) + L'\n' +
TAB_SPACE + _("Required:") + L' ' + formatFilesizeShort(space.first) + L'\n' +
TAB_SPACE + _("Available:") + L' ' + formatFilesizeShort(space.second);
callback.reportWarning(msg, warnings.warnNotEnoughDiskSpace); //throw X
}
//check if folders are used by multiple pairs in read/write access
{
std::vector<PathRaceItem> pathRaceItems;
//race condition := multiple accesses of which at least one is a write
//=> use "writeAccess" to reduce list of - not necessarily conflicting - candidates to check (=> perf!)
for (auto it = checkBaseFolderRaceCondition.begin(); it != checkBaseFolderRaceCondition.end(); ++it)
if (const auto& [baseFolder1, side1, writeAccess1] = *it;
writeAccess1)
for (auto it2 = checkBaseFolderRaceCondition.begin(); it2 != checkBaseFolderRaceCondition.end(); ++it2)
{
const auto& [baseFolder2, side2, writeAccess2] = *it2;
if (!writeAccess2 ||
it < it2) //avoid duplicate comparisons
{
//"The Things We Do for [Perf]"
/**/ if (side1 == SelectSide::left && side2 == SelectSide::left ) checkPathRaceCondition<SelectSide::left, SelectSide::left >(*baseFolder1, *baseFolder2, pathRaceItems);
else if (side1 == SelectSide::left && side2 == SelectSide::right) checkPathRaceCondition<SelectSide::left, SelectSide::right>(*baseFolder1, *baseFolder2, pathRaceItems);
else if (side1 == SelectSide::right && side2 == SelectSide::left ) checkPathRaceCondition<SelectSide::right, SelectSide::left >(*baseFolder1, *baseFolder2, pathRaceItems);
else checkPathRaceCondition<SelectSide::right, SelectSide::right>(*baseFolder1, *baseFolder2, pathRaceItems);
}
}
removeDuplicates(pathRaceItems);
//create mapping table for folder pair positions
std::unordered_map<const BaseFolderPair*, size_t> folderPairIdxs;
for (size_t folderIndex = 0; folderIndex < folderCmp.size(); ++folderIndex)
folderPairIdxs[&folderCmp[folderIndex].ref()] = folderIndex;
std::partial_sort(pathRaceItems.begin(),
pathRaceItems.begin() + std::min(pathRaceItems.size(), CONFLICTS_PREVIEW_MAX),
pathRaceItems.end(), [&](const PathRaceItem& lhs, const PathRaceItem& rhs)
{
if (const std::weak_ordering cmp = comparePathNoCase(lhs, rhs);
cmp != std::weak_ordering::equivalent)
return cmp < 0; //1. order by device, and case-insensitive path
return folderPairIdxs.find(&lhs.fsObj->base())->second < //2. order by folder pair position
folderPairIdxs.find(&rhs.fsObj->base())->second;
});
if (!pathRaceItems.empty())
{
std::wstring msg = _("Some files will be synchronized as part of multiple folder pairs.") + L'\n' +
_("To avoid conflicts, set up exclude filters so that each updated file is included by only one folder pair.") + L"\n\n";
auto prevItem = pathRaceItems[0];
std::for_each(pathRaceItems.begin(), pathRaceItems.begin() + std::min(pathRaceItems.size(), CONFLICTS_PREVIEW_MAX), [&](const PathRaceItem& item)
{
if (comparePathNoCase(item, prevItem) != std::weak_ordering::equivalent)
msg += L"\n"; //visually separate path groups
msg += formatRaceItem(item) + L"\n";
prevItem = item;
});
if (pathRaceItems.size() > CONFLICTS_PREVIEW_MAX)
msg += L"\n[...] " + replaceCpy(_P("Showing %y of 1 item", "Showing %y of %x items", pathRaceItems.size()), //%x used as plural form placeholder!
L"%y", formatNumber(CONFLICTS_PREVIEW_MAX));
msg += L"\n💾: " + _("Write access") + L" 👓: " + _("Read access");
callback.reportWarning(msg, warnings.warnDependentBaseFolders); //throw X
}
}
//check if versioning folder itself will be synchronized (and was not excluded via filter)
{
std::wstring msg;
bool shouldExclude = false;
for (const AbstractPath& versioningFolderPath : checkVersioningPaths)
{
std::set<AbstractPath> foldersWithWarnings; //=> at most one msg per base folder (*and* per versioningFolderPath)
for (const auto& [folderPath, filter] : checkVersioningBasePaths) //may contain duplicate paths, but with *different* hard filter!
if (std::optional<PathDependency> pd = getFolderPathDependency(versioningFolderPath, NullFilter(), folderPath, *filter))
if (const auto [it, inserted] = foldersWithWarnings.insert(folderPath);
inserted)
{
msg += L"\n\n" +
_("Selected folder:") + L" \t" + AFS::getDisplayPath(folderPath) + L'\n' +
_("Versioning folder:") + L" \t" + AFS::getDisplayPath(versioningFolderPath);
if (pd->itemPathParent == folderPath) //if versioning folder is a subfolder of a base folder
if (!pd->relPath.empty()) //this can be fixed via an exclude filter
{
assert(pd->itemPathParent == folderPath); //otherwise: what the fuck!?
shouldExclude = true;
msg += std::wstring() + L'\n' +
L"" + _("Exclude:") + L" \t" + utfTo<std::wstring>(FILE_NAME_SEPARATOR + pd->relPath + FILE_NAME_SEPARATOR);
}
}
}
if (!msg.empty())
callback.reportWarning(_("The versioning folder must not be part of the synchronization.") +
(shouldExclude ? L' ' + _("The folder should be excluded via filter.") : L"") +
msg, warnings.warnVersioningFolderPartOfSync); //throw X
}
//warn if versioning folder paths differ only in case => possible pessimization for applyVersioningLimit()
{
std::map<std::pair<AfsDevice, ZstringNoCase>, std::set<AbstractPath>> ciPathAliases;
for (const AbstractPath& folderPath : checkVersioningLimitPaths)
ciPathAliases[std::pair(folderPath.afsDevice, folderPath.afsPath.value)].insert(folderPath);
if (std::any_of(ciPathAliases.begin(), ciPathAliases.end(), [](const auto& item) { return item.second/*aliases*/.size() > 1; }))
{
std::wstring msg = _("The following folder paths differ in case. Please use a single form in order to avoid duplicate accesses.");
for (const auto& [key, aliases] : ciPathAliases)
if (aliases.size() > 1)
{
msg += L'\n';
for (const AbstractPath& aliasPath : aliases)
msg += L'\n' + AFS::getDisplayPath(aliasPath);
}
callback.reportWarning(msg, warnings.warnFoldersDifferInCase); //throw X
}
//what about /folder and /Folder/subfolder? => yes, inconsistent, but doesn't matter for FFS
}
//-------------------end of basic checks------------------------------------------
std::set<VersioningLimitFolder> versionLimitFolders;
bool recyclerMissingReportOnce = false; //prompt user only *once* per sync, not per failed item!
class PcbNoThrow : public PhaseCallback
{
public:
explicit PcbNoThrow(ProcessCallback& cb) : cb_(cb) {}
void updateDataProcessed(int itemsDelta, int64_t bytesDelta) override {} //sync DB/del-handler: logically not part of sync data, so let's ignore
void updateDataTotal (int itemsDelta, int64_t bytesDelta) override {} //
void requestUiUpdate(bool force) override { try { cb_.requestUiUpdate(force); /*throw X*/} catch (...) {}; }
void updateStatus(std::wstring&& msg) override { try { cb_.updateStatus(std::move(msg)); /*throw X*/} catch (...) {}; }
void logMessage(const std::wstring& msg, MsgType type) override { try { cb_.logMessage(msg, type); /*throw X*/} catch (...) {}; }
void reportWarning(const std::wstring& msg, bool& warningActive) override { logMessage(msg, MsgType::warning); /*ignore*/ }
Response reportError (const ErrorInfo& errorInfo) override { logMessage(errorInfo.msg, MsgType::error); return Response::ignore; }
void reportFatalError(const std::wstring& msg) override { logMessage(msg, MsgType::error); /*ignore*/ }
private:
ProcessCallback& cb_;
} callbackNoThrow(callback);
try
{
//loop through all directory pairs
for (size_t folderIndex = 0; folderIndex < folderCmp.size(); ++folderIndex)
{
BaseFolderPair& baseFolder = folderCmp[folderIndex].ref();
const FolderPairSyncCfg& folderPairCfg = syncConfig[folderIndex];
const SyncStatistics& folderPairStat = folderPairStats[folderIndex];
if (skipFolderPair[folderIndex]) //folder pairs may be skipped after fatal errors were found
continue;
//------------------------------------------------------------------------------------------
//checking a second time: 1. a long time may have passed since syncing the previous folder pairs!
// 2. expected to be run directly *before* createBaseFolder()!
if (!checkBaseFolderStatus<SelectSide::left >(baseFolder, callback) ||
!checkBaseFolderStatus<SelectSide::right>(baseFolder, callback))
continue;
//create base folders if not yet existing
if (folderPairStat.createCount() > 0 || folderPairCfg.saveSyncDB) //else: temporary network drop leading to deletions already caught by "sourceFolderMissing" check!
if (!createBaseFolder<SelectSide::left >(baseFolder, copyFilePermissions, callback) || //+ detect temporary network drop!!
!createBaseFolder<SelectSide::right>(baseFolder, copyFilePermissions, callback)) //
continue;
//------------------------------------------------------------------------------------------
//update database even when sync is cancelled (or "nothing to sync"):
auto guardDbSave = makeGuard<ScopeGuardRunMode::onFail>([&]
{
if (folderPairCfg.saveSyncDB)
saveLastSynchronousState(baseFolder, failSafeFileCopy,
callbackNoThrow);
});
//------------------------------------------------------------------------------------------
//execute synchronization recursively
if (getCUD(folderPairStat) > 0)
{
callback.logMessage(_("Synchronizing folder pair:") + L' ' + getVariantNameWithSymbol(folderPairCfg.syncVar) + L'\n' + //throw X
TAB_SPACE + AFS::getDisplayPath(baseFolder.getAbstractPath<SelectSide::left >()) + L'\n' +
TAB_SPACE + AFS::getDisplayPath(baseFolder.getAbstractPath<SelectSide::right>()), PhaseCallback::MsgType::info);
//guarantee removal of invalid entries (where element is empty on both sides)
ZEN_ON_SCOPE_EXIT(baseFolder.removeDoubleEmpty());
bool copyPermissionsFp = false;
tryReportingError([&]
{
copyPermissionsFp = copyFilePermissions && //copy permissions only if asked for and supported by *both* sides!
AFS::supportPermissionCopy(baseFolder.getAbstractPath<SelectSide::left>(),
baseFolder.getAbstractPath<SelectSide::right>()); //throw FileError
}, callback); //throw X
const AbstractPath versioningFolderPath = createAbstractPath(folderPairCfg.versioningFolderPhrase);
DeletionHandler delHandlerL(baseFolder.getAbstractPath<SelectSide::left>(),
recyclerMissingReportOnce,
warnings.warnRecyclerMissing,
folderPairCfg.handleDeletion,
versioningFolderPath,
folderPairCfg.versioningStyle,
std::chrono::system_clock::to_time_t(syncStartTime));
DeletionHandler delHandlerR(baseFolder.getAbstractPath<SelectSide::right>(),
recyclerMissingReportOnce,
warnings.warnRecyclerMissing,
folderPairCfg.handleDeletion,
versioningFolderPath,
folderPairCfg.versioningStyle,
std::chrono::system_clock::to_time_t(syncStartTime));
//always (try to) clean up, even if synchronization is aborted!
auto guardDelCleanup = makeGuard<ScopeGuardRunMode::onFail>([&]
{
delHandlerL.tryCleanup(callbackNoThrow);
delHandlerR.tryCleanup(callbackNoThrow);
});
FolderPairSyncer::SyncCtx syncCtx =
{
verifyCopiedFiles, copyPermissionsFp, failSafeFileCopy,
delHandlerL, delHandlerR,
};
FolderPairSyncer::runSync(syncCtx, baseFolder, callback);
//(try to gracefully) clean up temporary Recycle Bin folders and versioning
delHandlerL.tryCleanup(callback); //throw X
delHandlerR.tryCleanup(callback); //
guardDelCleanup.dismiss();
if (folderPairCfg.handleDeletion == DeletionVariant::versioning &&
folderPairCfg.versioningStyle != VersioningStyle::replace)
versionLimitFolders.insert(
{
versioningFolderPath,
folderPairCfg.versionMaxAgeDays,
folderPairCfg.versionCountMin,
folderPairCfg.versionCountMax
});
}
//(try to gracefully) write database file
if (folderPairCfg.saveSyncDB)
{
saveLastSynchronousState(baseFolder, failSafeFileCopy,
callback /*throw X*/); //throw X
guardDbSave.dismiss(); //[!] dismiss *after* "graceful" try: user might cancel during DB write: ensure DB is still written
}
}
//-----------------------------------------------------------------------------------------------------
applyVersioningLimit(versionLimitFolders,
callback /*throw X*/); //throw X
}
catch (const std::exception& e)
{
callback.reportFatalError(utfTo<std::wstring>(e.what()));
}
}