// ***************************************************************************** // * This file is part of the FreeFileSync project. It is distributed under * // * GNU General Public License: https://www.gnu.org/licenses/gpl-3.0 * // * Copyright (C) Zenju (zenju AT freefilesync DOT org) - All Rights Reserved * // ***************************************************************************** #include "db_file.h" #include //std::endian #include #include #include #include "../afs/native.h" #include "status_handler_impl.h" using namespace zen; using namespace fff; namespace { //------------------------------------------------------------------------------------------------------------------------------- const char DB_FILE_DESCR[] = "FreeFileSync"; const int DB_FILE_VERSION = 11; //2020-02-07 const int DB_STREAM_VERSION = 5; //2023-07-29 //------------------------------------------------------------------------------------------------------------------------------- struct SessionData { bool isLeadStream = false; std::string rawStream; bool operator==(const SessionData&) const = default; }; using UniqueId = std::string; using DbStreams = std::unordered_map; //list of streams by session GUID /*------------------------------------------------------------------------------ | ensure 32/64 bit portability: use fixed size data types only e.g. uint32_t | ------------------------------------------------------------------------------*/ template inline AbstractPath getDatabaseFilePath(const BaseFolderPair& baseFolder) { static_assert(std::endian::native == std::endian::little); /* Windows, Linux, macOS considerations for uniform database format: - different file IDs: no, but the volume IDs are different! - problem with case sensitivity: no - are UTC file times identical: yes (at least with 1 sec precision) - endianess: FFS currently not running on any big-endian platform - precomposed/decomposed UTF: differences already ignored - 32 vs 64-bit: already handled => give DB files different names: */ const Zstring dbName = Zstr(".sync"); //files beginning with dots are usually hidden return AFS::appendRelPath(baseFolder.getAbstractPath(), dbName + SYNC_DB_FILE_ENDING); } //####################################################################################################################################### void saveStreams(const DbStreams& streamList, const AbstractPath& dbPath, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, X { MemoryStreamOut memStreamOut; //write FreeFileSync file identifier writeArray(memStreamOut, DB_FILE_DESCR, sizeof(DB_FILE_DESCR)); //save file format version writeNumber(memStreamOut, DB_FILE_VERSION); //write stream list writeNumber(memStreamOut, static_cast(streamList.size())); for (const auto& [sessionID, sessionData] : streamList) { writeContainer(memStreamOut, sessionID); writeNumber(memStreamOut, sessionData.isLeadStream); writeContainer (memStreamOut, sessionData.rawStream); } writeNumber(memStreamOut, getCrc32(memStreamOut.ref())); //------------------------------------------------------------------------------------------------------------------------ //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) const std::unique_ptr byteStreamOut = AFS::getOutputStream(dbPath, memStreamOut.ref().size(), std::nullopt /*modTime*/); //throw FileError unbufferedSave(memStreamOut.ref(), [&](const void* buffer, size_t bytesToWrite) { return byteStreamOut->tryWrite(buffer, bytesToWrite, notifyUnbufferedIO); //throw FileError, X }, byteStreamOut->getBlockSize()); //throw FileError, X byteStreamOut->finalize(notifyUnbufferedIO); //throw FileError, X } DEFINE_NEW_FILE_ERROR(FileErrorDatabaseNotExisting) DEFINE_NEW_FILE_ERROR(FileErrorDatabaseCorrupted) DbStreams loadStreams(const AbstractPath& dbPath, const IoCallback& notifyUnbufferedIO /*throw X*/) //throw FileError, FileErrorDatabaseNotExisting, FileErrorDatabaseCorrupted, X { std::string byteStream; try { const std::unique_ptr fileIn = AFS::getInputStream(dbPath); //throw FileError, ErrorFileLocked byteStream = unbufferedLoad([&](void* buffer, size_t bytesToRead) { return fileIn->tryRead(buffer, bytesToRead, notifyUnbufferedIO); //throw FileError, ErrorFileLocked, X; may return short, only 0 means EOF! }, fileIn->getBlockSize()); //throw FileError, X } catch (const FileError& e) { bool dbNotYetExisting = false; try { dbNotYetExisting = !AFS::itemExists(dbPath); /*throw FileError*/ } //abstract context => unclear which exception is more relevant/useless: catch (const FileError& e2) { throw FileError(replaceCpy(e.toString(), L"\n\n", L'\n'), replaceCpy(e2.toString(), L"\n\n", L'\n')); } //caveat: merging FileError might create redundant error message: https://freefilesync.org/forum/viewtopic.php?t=9377 if (dbNotYetExisting) //throw FileError throw FileErrorDatabaseNotExisting(replaceCpy(_("Database file %x does not yet exist."), L"%x", fmtPath(AFS::getDisplayPath(dbPath)))); else throw; } //------------------------------------------------------------------------------------------------------------------------ try { MemoryStreamIn memStreamIn(byteStream); char formatDescr[sizeof(DB_FILE_DESCR)] = {}; readArray(memStreamIn, formatDescr, sizeof(formatDescr)); //throw SysErrorUnexpectedEos if (!std::equal(DB_FILE_DESCR, DB_FILE_DESCR + sizeof(DB_FILE_DESCR), formatDescr)) throw SysError(_("File content is corrupted.") + L" (invalid header)"); const int version = readNumber(memStreamIn); //throw SysErrorUnexpectedEos if (version == 9 || //TODO: remove migration code at some time! v9 used until 2017-02-01 version == 10) //TODO: remove migration code at some time! v10 used until 2020-02-07 ; else if (version == DB_FILE_VERSION) //catch data corruption ASAP + don't rely on std::bad_alloc for consistency checking // => only "partially" useful for container/stream metadata since the streams data is zlib-compressed { assert(byteStream.size() >= sizeof(uint32_t)); //obviously in this context! MemoryStreamOut crcStreamOut; writeNumber(crcStreamOut, getCrc32(byteStream.begin(), byteStream.end() - sizeof(uint32_t))); if (!endsWith(byteStream, crcStreamOut.ref())) throw SysError(_("File content is corrupted.") + L" (invalid checksum)"); } else throw SysError(_("Unsupported data format.") + L' ' + replaceCpy(_("Version: %x"), L"%x", numberTo(version))); DbStreams output; //read stream list size_t streamCount = readNumber(memStreamIn); //throw SysErrorUnexpectedEos while (streamCount-- != 0) { std::string sessionID = readContainer(memStreamIn); //throw SysErrorUnexpectedEos SessionData sessionData = {}; if (version == 9) //TODO: remove migration code at some time! v9 used until 2017-02-01 { sessionData.rawStream = readContainer(memStreamIn); //throw SysErrorUnexpectedEos MemoryStreamIn streamIn(sessionData.rawStream); const int streamVersion = readNumber(streamIn); //throw SysErrorUnexpectedEos if (streamVersion != 2) //don't throw here due to old stream formats continue; sessionData.isLeadStream = readNumber(streamIn) != 0; //throw SysErrorUnexpectedEos } else { sessionData.isLeadStream = readNumber (memStreamIn) != 0; //throw SysErrorUnexpectedEos sessionData.rawStream = readContainer(memStreamIn); // } output[sessionID] = std::move(sessionData); } return output; } catch (const SysError& e) { throw FileErrorDatabaseCorrupted(replaceCpy(_("Cannot read database file %x."), L"%x", fmtPath(AFS::getDisplayPath(dbPath))), e.toString()); } } //####################################################################################################################################### class StreamGenerator { public: static void execute(const InSyncFolder& dbFolder, //throw FileError const std::wstring& displayFilePathL, //used for diagnostics only const std::wstring& displayFilePathR, std::string& streamL, std::string& streamR) { MemoryStreamOut outL; MemoryStreamOut outR; //save format version writeNumber(outL, DB_STREAM_VERSION); writeNumber(outR, DB_STREAM_VERSION); auto compStream = [&](const std::string& stream) //throw FileError { try { /* Zlib: optimal level - test case 1 million files level|size [MB]|time [ms] 0 49.54 272 (uncompressed) 1 14.53 1013 2 14.13 1106 3 13.76 1288 - best compromise between speed and compression 4 13.20 1526 5 12.73 1916 6 12.58 2765 7 12.54 3633 8 12.51 9032 9 12.50 19698 (maximal compression) */ return compress(stream, 3 /*level*/); //throw SysError } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(displayFilePathL + L"/" + displayFilePathR)), e.toString()); } }; StreamGenerator generator; //PERF_START generator.recurse(dbFolder); //PERF_STOP const std::string bufText = compStream(generator.streamOutText_ .ref()); const std::string bufSmallNum = compStream(generator.streamOutSmallNum_.ref()); const std::string bufBigNum = compStream(generator.streamOutBigNum_ .ref()); MemoryStreamOut streamOut; writeContainer(streamOut, bufText); writeContainer(streamOut, bufSmallNum); writeContainer(streamOut, bufBigNum); const std::string& buf = streamOut.ref(); //distribute "outputBoth" over left and right streams: const size_t size1stPart = buf.size() / 2; const size_t size2ndPart = buf.size() - size1stPart; writeNumber(outL, size1stPart); writeNumber(outR, size2ndPart); if (size1stPart > 0) writeArray(outL, buf.c_str(), size1stPart); if (size2ndPart > 0) writeArray(outR, buf.c_str() + size1stPart, size2ndPart); streamL = std::move(outL.ref()); streamR = std::move(outR.ref()); } private: void recurse(const InSyncFolder& container) { writeNumber(streamOutSmallNum_, static_cast(container.files.size())); for (const auto& [itemName, inSyncData] : container.files) { writeItemName(itemName.normStr); writeNumber(streamOutSmallNum_, static_cast(inSyncData.cmpVar)); writeNumber(streamOutSmallNum_, inSyncData.fileSize); writeFileDescr(inSyncData.left); writeFileDescr(inSyncData.right); } writeNumber(streamOutSmallNum_, static_cast(container.symlinks.size())); for (const auto& [itemName, inSyncData] : container.symlinks) { writeItemName(itemName.normStr); writeNumber(streamOutSmallNum_, static_cast(inSyncData.cmpVar)); writeNumber(streamOutBigNum_, inSyncData.left .modTime); writeNumber(streamOutBigNum_, inSyncData.right.modTime); } writeNumber(streamOutSmallNum_, static_cast(container.folders.size())); for (const auto& [itemName, inSyncData] : container.folders) { writeItemName(itemName.normStr); recurse(inSyncData); } } void writeItemName(const Zstring& str) { writeContainer(streamOutText_, utfTo(str)); } void writeFileDescr(const InSyncDescrFile& descr) { writeNumber(streamOutBigNum_, descr.modTime); writeNumber(streamOutBigNum_, descr.filePrint); static_assert(sizeof(descr.modTime) <= sizeof(int64_t)); //ensure cross-platform compatibility! } /* maximize zlib compression by grouping similar data (=> 20% size reduction!) -> further ~5% reduction possible by having one container per data type other ideas: - avoid left/right side interleaving in writeFileDescr() => pessimization! - convert CompareVariant/InSyncStatus to "enum : unsigned char" => only 0,4% size reduction! - split up writeItemName() to use streamOutSmallNum_ + streamOutText_ => pessimization! - use null-termination in writeItemName() => 5% size reduction (embedded zeros impossible?) - use empty item name as sentinel => only 0,17% size reduction! - save fileSize using instreamOutBigNum_ => pessimization! */ MemoryStreamOut streamOutText_; // MemoryStreamOut streamOutSmallNum_; //data with bias to lead side (= always left in this context) MemoryStreamOut streamOutBigNum_; // }; class StreamParser { public: static SharedRef execute(bool leadStreamLeft, //throw FileError const std::string& streamL, const std::string& streamR, const std::wstring& displayFilePathL, //for diagnostics only const std::wstring& displayFilePathR) { try { MemoryStreamIn streamInL(streamL); MemoryStreamIn streamInR(streamR); const int streamVersion = readNumber(streamInL); //throw SysErrorUnexpectedEos const int streamVersionR = readNumber(streamInR); // if (streamVersion != streamVersionR) throw SysError(_("File content is corrupted.") + L" (different stream formats)"); //TODO: remove migration code at some time! 2017-02-01 if (streamVersion == 2) { const bool has1stPartL = readNumber(streamInL) != 0; //throw SysErrorUnexpectedEos const bool has1stPartR = readNumber(streamInR) != 0; // if (has1stPartL == has1stPartR) throw SysError(_("File content is corrupted.") + L" (second stream part missing)"); if (has1stPartL != leadStreamLeft) throw SysError(_("File content is corrupted.") + L" (has1stPartL != leadStreamLeft)"); MemoryStreamIn& in1stPart = leadStreamLeft ? streamInL : streamInR; MemoryStreamIn& in2ndPart = leadStreamLeft ? streamInR : streamInL; const size_t size1stPart = static_cast(readNumber(in1stPart)); const size_t size2ndPart = static_cast(readNumber(in2ndPart)); std::string tmpB(size1stPart + size2ndPart, '\0'); //throw std::bad_alloc readArray(in1stPart, tmpB.data(), size1stPart); //stream always non-empty readArray(in2ndPart, tmpB.data() + size1stPart, size2ndPart); //throw SysErrorUnexpectedEos const std::string tmpL = readContainer(streamInL); const std::string tmpR = readContainer(streamInR); auto output = makeSharedRef(); StreamParserV2 parser(decompress(tmpL), // decompress(tmpR), //throw SysError decompress(tmpB)); // parser.recurse(output.ref()); //throw SysError return output; } else if (streamVersion == 3 || //TODO: remove migration code at some time! 2021-02-14 streamVersion == 4 || //TODO: remove migration code at some time! 2023-07-29 streamVersion == DB_STREAM_VERSION) { MemoryStreamIn& streamInPart1 = leadStreamLeft ? streamInL : streamInR; MemoryStreamIn& streamInPart2 = leadStreamLeft ? streamInR : streamInL; const size_t sizePart1 = static_cast(readNumber(streamInPart1)); const size_t sizePart2 = static_cast(readNumber(streamInPart2)); std::string buf(sizePart1 + sizePart2, '\0'); if (sizePart1 > 0) readArray(streamInPart1, buf.data(), sizePart1); //throw SysErrorUnexpectedEos if (sizePart2 > 0) readArray(streamInPart2, buf.data() + sizePart1, sizePart2); // MemoryStreamIn streamIn(buf); const std::string bufText = readContainer(streamIn); // const std::string bufSmallNum = readContainer(streamIn); //throw SysErrorUnexpectedEos const std::string bufBigNum = readContainer(streamIn); // auto output = makeSharedRef(); StreamParser parser(streamVersion, decompress(bufText), // decompress(bufSmallNum), //throw SysError decompress(bufBigNum)); // if (leadStreamLeft) parser.recurse(output.ref()); //throw SysError else parser.recurse(output.ref()); //throw SysError return output; } else throw SysError(_("Unsupported data format.") + L' ' + replaceCpy(_("Version: %x"), L"%x", numberTo(streamVersion))); } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read database file %x."), L"%x", fmtPath(displayFilePathL) + L", " + fmtPath(displayFilePathR)), e.toString()); } } private: StreamParser(int streamVersion, std::string&& bufText, std::string&& bufSmallNumbers, std::string&& bufBigNumbers) : streamVersion_(streamVersion), bufText_ (std::move(bufText)), bufSmallNumbers_(std::move(bufSmallNumbers)), bufBigNumbers_ (std::move(bufBigNumbers)) {} template void recurse(InSyncFolder& container) //throw SysError { size_t fileCount = readNumber(streamInSmallNum_); //throw SysErrorUnexpectedEos while (fileCount-- != 0) { const Zstring itemName = readItemName(); // const auto cmpVar = static_cast(readNumber(streamInSmallNum_)); // const uint64_t fileSize = readNumber(streamInSmallNum_); // const InSyncDescrFile descrL = readFileDescr(); //throw SysErrorUnexpectedEos const InSyncDescrFile descrT = readFileDescr(); // container.addFile(itemName, selectParam(descrL, descrT), selectParam(descrT, descrL), cmpVar, fileSize); } size_t linkCount = readNumber(streamInSmallNum_); while (linkCount-- != 0) { const Zstring itemName = readItemName(); // const auto cmpVar = static_cast(readNumber(streamInSmallNum_)); // const InSyncDescrLink descrL{static_cast(readNumber(streamInBigNum_))}; //throw SysErrorUnexpectedEos const InSyncDescrLink descrT{static_cast(readNumber(streamInBigNum_))}; // container.addSymlink(itemName, selectParam(descrL, descrT), selectParam(descrT, descrL), cmpVar); } size_t dirCount = readNumber(streamInSmallNum_); // while (dirCount-- != 0) { const Zstring itemName = readItemName(); // if (streamVersion_ <= 4) //TODO: remove migration code at some time! 2023-07-29 /*const auto status = static_cast(*/ readNumber(streamInSmallNum_); InSyncFolder& dbFolder = container.addFolder(itemName); recurse(dbFolder); } } Zstring readItemName() { return utfTo(readContainer(streamInText_)); } //throw SysErrorUnexpectedEos InSyncDescrFile readFileDescr() //throw SysErrorUnexpectedEos { const auto modTime = static_cast(readNumber(streamInBigNum_)); //throw SysErrorUnexpectedEos AFS::FingerPrint filePrint = 0; if (streamVersion_ == 3) //TODO: remove migration code at some time! 2021-02-14 { const auto& devFileId = readContainer(streamInBigNum_); //throw SysErrorUnexpectedEos ino_t fileIndex = 0; if (devFileId.size() == sizeof(dev_t) + sizeof(fileIndex)) { std::memcpy(&fileIndex, &devFileId[devFileId.size() - sizeof(fileIndex)], sizeof(fileIndex)); filePrint = fileIndex; } else assert(devFileId.empty()); } else filePrint = readNumber(streamInBigNum_); //throw SysErrorUnexpectedEos return {modTime, filePrint}; } //TODO: remove migration code at some time! 2017-02-01 class StreamParserV2 { public: StreamParserV2(std::string&& bufferL, std::string&& bufferR, std::string&& bufferB) : bufL_(std::move(bufferL)), bufR_(std::move(bufferR)), bufB_(std::move(bufferB)) {} void recurse(InSyncFolder& container) //throw SysError { size_t fileCount = readNumber(inputBoth_); while (fileCount-- != 0) { const Zstring itemName = utfTo(readContainer(inputBoth_)); const auto cmpVar = static_cast(readNumber(inputBoth_)); const uint64_t fileSize = readNumber(inputBoth_); const auto modTimeL = static_cast(readNumber(inputLeft_)); /*const auto fileIdL =*/ readContainer(inputLeft_); const auto modTimeR = static_cast(readNumber(inputRight_)); /*const auto fileIdR =*/ readContainer(inputRight_); container.addFile(itemName, InSyncDescrFile{modTimeL, AFS::FingerPrint()}, InSyncDescrFile{modTimeR, AFS::FingerPrint()}, cmpVar, fileSize); } size_t linkCount = readNumber(inputBoth_); while (linkCount-- != 0) { const Zstring itemName = utfTo(readContainer(inputBoth_)); const auto cmpVar = static_cast(readNumber(inputBoth_)); const auto modTimeL = static_cast(readNumber(inputLeft_)); const auto modTimeR = static_cast(readNumber(inputRight_)); container.addSymlink(itemName, InSyncDescrLink{modTimeL}, InSyncDescrLink{modTimeR}, cmpVar); } size_t dirCount = readNumber(inputBoth_); while (dirCount-- != 0) { const Zstring itemName = utfTo(readContainer(inputBoth_)); /*const auto status = static_cast(*/ readNumber(inputBoth_); InSyncFolder& dbFolder = container.addFolder(itemName); recurse(dbFolder); } } private: const std::string bufL_; const std::string bufR_; const std::string bufB_; MemoryStreamIn inputLeft_ {bufL_}; //data related to one side only MemoryStreamIn inputRight_{bufR_}; // MemoryStreamIn inputBoth_ {bufB_}; //data concerning both sides }; const int streamVersion_; const std::string bufText_; const std::string bufSmallNumbers_; const std::string bufBigNumbers_ ; MemoryStreamIn streamInText_ {bufText_}; // MemoryStreamIn streamInSmallNum_{bufSmallNumbers_}; //data with bias to lead side MemoryStreamIn streamInBigNum_ {bufBigNumbers_}; // }; //####################################################################################################################################### class LastSynchronousStateUpdater { /* 1. filter by file name does *not* create a new hierarchy, but merely gives a different *view* on the existing file hierarchy => only update database entries matching this view! 2. Symlink handling *does* create a new (asymmetric) hierarchy during comparison => update all database entries! */ public: static void execute(const BaseFolderPair& baseFolder, InSyncFolder& dbFolder) { LastSynchronousStateUpdater updater(baseFolder.getCompVariant(), baseFolder.getFilter()); updater.recurse(baseFolder, Zstring(), dbFolder); } private: LastSynchronousStateUpdater(CompareVariant activeCmpVar, const PathFilter& filter) : filter_(filter), activeCmpVar_(activeCmpVar) {} void recurse(const ContainerObject& conObj, const Zstring& relPath, InSyncFolder& dbFolder) { processFiles (conObj, relPath, dbFolder.files); processLinks (conObj, relPath, dbFolder.symlinks); processFolders(conObj, relPath, dbFolder.folders); } void processFiles(const ContainerObject& conObj, const Zstring& parentRelPath, InSyncFolder::FileList& dbFiles) { std::unordered_set toPreserve; for (const FilePair& file : conObj.files()) if (!file.isPairEmpty()) { if (file.getCategory() == FILE_EQUAL) //data in sync: write current state { //Caveat: If FILE_EQUAL, we *implicitly* assume equal left and right file names matching case: InSyncFolder's mapping tables use file name as a key! //This makes us silently dependent from code in algorithm.h!!! assert(file.hasEquivalentItemNames()); const Zstring& fileName = file.getItemName(); assert(file.getFileSize() == file.getFileSize()); //create or update new "in-sync" state dbFiles.insert_or_assign(fileName, InSyncFile { .left = InSyncDescrFile{file.getLastWriteTime(), file.getFilePrint()}, .right = InSyncDescrFile{file.getLastWriteTime(), file.getFilePrint()}, .cmpVar = activeCmpVar_, .fileSize = file.getFileSize(), }); toPreserve.insert(fileName); } else //not in sync: preserve last synchronous state { toPreserve.insert(file.getItemName()); //left/right may differ in case! toPreserve.insert(file.getItemName()); // } } //delete removed items (= "in-sync") from database std::erase_if(dbFiles, [&](const InSyncFolder::FileList::value_type& v) { if (toPreserve.contains(v.first)) return false; //all items not existing in "currentFiles" have either been deleted meanwhile or been excluded via filter: const Zstring& itemRelPath = appendPath(parentRelPath, v.first.normStr); return filter_.passFileFilter(itemRelPath); //note: items subject to traveral errors are also excluded by this file filter here! see comparison.cpp, modified file filter for read errors }); } void processLinks(const ContainerObject& conObj, const Zstring& parentRelPath, InSyncFolder::SymlinkList& dbSymlinks) { std::unordered_set toPreserve; for (const SymlinkPair& symlink : conObj.symlinks()) if (!symlink.isPairEmpty()) { if (symlink.getLinkCategory() == SYMLINK_EQUAL) //data in sync: write current state { assert(symlink.hasEquivalentItemNames()); const Zstring& linkName = symlink.getItemName(); //create or update new "in-sync" state dbSymlinks.insert_or_assign(linkName, InSyncSymlink { .left = InSyncDescrLink{symlink.getLastWriteTime()}, .right = InSyncDescrLink{symlink.getLastWriteTime()}, .cmpVar = activeCmpVar_, }); toPreserve.insert(linkName); } else //not in sync: preserve last synchronous state { toPreserve.insert(symlink.getItemName()); //left/right may differ in case! toPreserve.insert(symlink.getItemName()); // } } //delete removed items (= "in-sync") from database std::erase_if(dbSymlinks, [&](const InSyncFolder::SymlinkList::value_type& v) { if (toPreserve.contains(v.first)) return false; //all items not existing in "currentSymlinks" have either been deleted meanwhile or been excluded via filter: const Zstring& itemRelPath = appendPath(parentRelPath, v.first.normStr); return filter_.passFileFilter(itemRelPath); }); } void processFolders(const ContainerObject& conObj, const Zstring& parentRelPath, InSyncFolder::FolderList& dbFolders) { std::unordered_map toPreserve; for (const FolderPair& folder : conObj.subfolders()) if (!folder.isPairEmpty()) { if (folder.getDirCategory() == DIR_EQUAL) { assert(folder.hasEquivalentItemNames()); const Zstring& folderName = folder.getItemName(); //create directory entry if not existing (but do *not touch* existing child elements!!!) dbFolders.try_emplace(folderName); toPreserve.emplace(folderName, &folder); } else //not in sync: preserve last synchronous state { toPreserve.emplace(folder.getItemName(), &folder); //names differing (in case)? => treat like any other folder rename toPreserve.emplace(folder.getItemName(), &folder); //=> no *new* database entries even if child items are in sync //BUT: update existing one: there should be only *one* DB entry after a folder rename (matching either folder name on left or right) } } //delete removed items (= "in-sync") from database eraseIf(dbFolders, [&](InSyncFolder::FolderList::value_type& v) { const Zstring& itemRelPath = appendPath(parentRelPath, v.first.normStr); if (auto it = toPreserve.find(v.first); it != toPreserve.end()) { recurse(*(it->second), itemRelPath, v.second); //required even if e.g. DIR_LEFT_ONLY: //existing child-items may not be in sync, but items deleted on both sides *are* in-sync!!! return false; } //if folder is not included in "current folders", it is either not existing anymore, in which case it should be deleted from database //or it was excluded via filter and the database entry should be preserved bool childItemMightMatch = true; const bool passFilter = filter_.passDirFilter(itemRelPath, &childItemMightMatch); if (!passFilter && childItemMightMatch) dbSetEmptyState(v.second, appendSeparator(itemRelPath)); //child items might match, e.g. *.txt include filter! return passFilter; }); } //delete all entries for removed folder (= "in-sync") from database void dbSetEmptyState(InSyncFolder& dbFolder, const Zstring& parentRelPathPf) { std::erase_if(dbFolder.files, [&](const InSyncFolder::FileList ::value_type& v) { return filter_.passFileFilter(parentRelPathPf + v.first.normStr); }); std::erase_if(dbFolder.symlinks, [&](const InSyncFolder::SymlinkList::value_type& v) { return filter_.passFileFilter(parentRelPathPf + v.first.normStr); }); eraseIf(dbFolder.folders, [&](InSyncFolder::FolderList::value_type& v) { const Zstring& itemRelPath = parentRelPathPf + v.first.normStr; bool childItemMightMatch = true; const bool passFilter = filter_.passDirFilter(itemRelPath, &childItemMightMatch); if (!passFilter && childItemMightMatch) dbSetEmptyState(v.second, appendSeparator(itemRelPath)); return passFilter; }); } const PathFilter& filter_; //filter used while scanning directory: generates view on actual files! const CompareVariant activeCmpVar_; }; struct StreamStatusNotifier { StreamStatusNotifier(const std::wstring& statusMsg, AsyncCallback& acb /*throw ThreadStopRequest*/) : msgPrefix_(statusMsg + L' '), acb_(acb) {} void operator()(int64_t bytesDelta) //throw ThreadStopRequest { bytesTotal_ += bytesDelta; const auto now = std::chrono::steady_clock::now(); if (now >= lastUpdate_ + UI_UPDATE_INTERVAL / 2) //every ~25 ms { lastUpdate_ = now; acb_.updateStatus(msgPrefix_ + formatFilesizeShort(bytesTotal_)); //throw ThreadStopRequest } } private: const std::wstring msgPrefix_; int64_t bytesTotal_ = 0; AsyncCallback& acb_; std::chrono::steady_clock::time_point lastUpdate_; }; std::pair findCommonSession(const DbStreams& streamsLeft, const DbStreams& streamsRight, //throw FileError const std::wstring& displayFilePathL, //used for diagnostics only const std::wstring& displayFilePathR) { auto itCommonL = streamsLeft .end(); auto itCommonR = streamsRight.end(); for (auto itL = streamsLeft.begin(); itL != streamsLeft.end(); ++itL) { auto itR = streamsRight.find(itL->first); if (itR != streamsRight.end()) /* handle case when db file is loaded together with a (former) copy of itself: - some streams may have been updated in the meantime => must not discard either db file! - since db file was copied, multiple streams may have matching sessionID => IGNORE all of them: one of them may be used later against other sync targets! */ if (itL->second.isLeadStream != itR->second.isLeadStream) { if (itCommonL != streamsLeft.end()) //should not be possible! throw FileError(replaceCpy(_("Cannot read database file %x."), L"%x", fmtPath(displayFilePathL) + L", " + fmtPath(displayFilePathR)), _("File content is corrupted.") + L" (multiple common sessions found)"); itCommonL = itL; itCommonR = itR; } } return {itCommonL, itCommonR}; } } //####################################################################################################################################### std::unordered_map> fff::loadLastSynchronousState(const std::vector& baseFolders, PhaseCallback& callback /*throw X*/) //throw X { std::set dbFilePaths; for (const BaseFolderPair* baseFolder : baseFolders) //avoid race condition with directory existence check: reading sync.ffs_db may succeed although first dir check had failed => conflicts! if (baseFolder->getFolderStatus() == BaseFolderStatus::existing && baseFolder->getFolderStatus() == BaseFolderStatus::existing) { dbFilePaths.insert(getDatabaseFilePath(*baseFolder)); dbFilePaths.insert(getDatabaseFilePath(*baseFolder)); } //else: ignore; there's no value in reporting it other than to confuse users std::map dbStreamsByPath; //------------ (try to) load DB files in parallel ------------------------- { Protected&> protDbStreamsByPath(dbStreamsByPath); std::vector> parallelWorkload; for (const AbstractPath& dbPath : dbFilePaths) parallelWorkload.emplace_back(dbPath, [&protDbStreamsByPath](ParallelContext& ctx) //throw ThreadStopRequest { tryReportingError([&] //throw ThreadStopRequest { StreamStatusNotifier notifyLoad(replaceCpy(_("Loading file %x..."), L"%x", fmtPath(AFS::getDisplayPath(ctx.itemPath))), ctx.acb); try { DbStreams dbStreams = ::loadStreams(ctx.itemPath, notifyLoad); //throw FileError, FileErrorDatabaseNotExisting, FileErrorDatabaseCorrupted, ThreadStopRequest protDbStreamsByPath.access([&](auto& dbStreamsByPath2) { dbStreamsByPath2.emplace(ctx.itemPath, std::move(dbStreams)); }); } catch (FileErrorDatabaseNotExisting&) {} //redundant info => no reportInfo() }, ctx.acb); }); massParallelExecute(parallelWorkload, Zstr("Load sync.ffs_db"), callback /*throw X*/); //throw X } //---------------------------------------------------------------- std::unordered_map> output; for (const BaseFolderPair* baseFolder : baseFolders) if (baseFolder->getFolderStatus() == BaseFolderStatus::existing && baseFolder->getFolderStatus() == BaseFolderStatus::existing) { const AbstractPath dbPathL = getDatabaseFilePath(*baseFolder); const AbstractPath dbPathR = getDatabaseFilePath(*baseFolder); auto itL = dbStreamsByPath.find(dbPathL); auto itR = dbStreamsByPath.find(dbPathR); if (itL != dbStreamsByPath.end() && itR != dbStreamsByPath.end()) try { const DbStreams& streamsL = itL->second; const DbStreams& streamsR = itR->second; //find associated session: there can be at most one session within intersection of left and right IDs const auto [itStreamL, itStreamR] = findCommonSession(streamsL, streamsR, AFS::getDisplayPath(dbPathL), AFS::getDisplayPath(dbPathR)); //throw FileError if (itStreamL != streamsL.end()) { assert(itStreamL->second.isLeadStream != itStreamR->second.isLeadStream); SharedRef lastSyncState = StreamParser::execute(itStreamL->second.isLeadStream, itStreamL->second.rawStream, itStreamR->second.rawStream, AFS::getDisplayPath(dbPathL), AFS::getDisplayPath(dbPathR)); //throw FileError output.emplace(baseFolder, lastSyncState); } } catch (const FileError& e) { callback.reportFatalError(e.toString()); } //throw X } return output; } void fff::saveLastSynchronousState(const BaseFolderPair& baseFolder, bool transactionalCopy, PhaseCallback& callback /*throw X*/) //throw X { const AbstractPath dbPathL = getDatabaseFilePath(baseFolder); const AbstractPath dbPathR = getDatabaseFilePath(baseFolder); //------------ (try to) load DB files in parallel ------------------------- DbStreams streamsL; //list of session ID + DirInfo-stream DbStreams streamsR; // { bool loadSuccessL = false; bool loadSuccessR = false; std::vector> parallelWorkload; for (const auto& [dbPath, streamsOut, loadSuccess] : { std::tuple(dbPathL, &streamsL, &loadSuccessL), std::tuple(dbPathR, &streamsR, &loadSuccessR) }) parallelWorkload.emplace_back(dbPath, [&streamsOut = *streamsOut, &loadSuccess = *loadSuccess](ParallelContext& ctx) //throw ThreadStopRequest { const std::wstring errMsg = tryReportingError([&] //throw ThreadStopRequest { StreamStatusNotifier notifyLoad(replaceCpy(_("Loading file %x..."), L"%x", fmtPath(AFS::getDisplayPath(ctx.itemPath))), ctx.acb); try { streamsOut = ::loadStreams(ctx.itemPath, notifyLoad); } //throw FileError, FileErrorDatabaseNotExisting, FileErrorDatabaseCorrupted, ThreadStopRequest catch (FileErrorDatabaseNotExisting&) {} catch (FileErrorDatabaseCorrupted&) {} //=> just overwrite corrupted DB file: error already reported by loadLastSynchronousState() }, ctx.acb); loadSuccess = errMsg.empty(); }); massParallelExecute(parallelWorkload, Zstr("Load sync.ffs_db"), callback /*throw X*/); //throw X if (!loadSuccessL || !loadSuccessR) return; /* don't continue when one of the two files failed to load (e.g. network drop): no common session would be found, (although it may exist!) => a) if file also fails to save: new orphan session in the other file created b) if file saves successfully: previous stream sessions lost + old session in other file not cleaned up (orphan) */ } //---------------------------------------------------------------- //load last synchrounous state auto itStreamOldL = streamsL.cend(); auto itStreamOldR = streamsR.cend(); InSyncFolder lastSyncState; try { //find associated session: there can be at most one session within intersection of left and right IDs std::tie(itStreamOldL, itStreamOldR) = findCommonSession(streamsL, streamsR, AFS::getDisplayPath(dbPathL), AFS::getDisplayPath(dbPathR)); //throw FileError if (itStreamOldL != streamsL.end()) lastSyncState = std::move(StreamParser::execute(itStreamOldL->second.isLeadStream /*leadStreamLeft*/, itStreamOldL->second.rawStream, itStreamOldR->second.rawStream, AFS::getDisplayPath(dbPathL), AFS::getDisplayPath(dbPathR)).ref()); //throw FileError } catch (const FileError& e) { callback.reportFatalError(e.toString()); } //throw X //if database files are corrupted: just overwrite! User is already informed about errors right after comparing! //update last synchrounous state LastSynchronousStateUpdater::execute(baseFolder, lastSyncState); //serialize again SessionData sessionDataL = {}; SessionData sessionDataR = {}; sessionDataL.isLeadStream = true; sessionDataR.isLeadStream = false; if (const std::wstring errMsg = tryReportingError([&] //throw X { StreamGenerator::execute(lastSyncState, //throw FileError AFS::getDisplayPath(dbPathL), AFS::getDisplayPath(dbPathR), sessionDataL.rawStream, sessionDataR.rawStream); }, callback /*throw X*/); !errMsg.empty()) return; //check if there is some work to do at all if (itStreamOldL != streamsL.end() && itStreamOldL->second == sessionDataL && itStreamOldR != streamsR.end() && itStreamOldR->second == sessionDataR) return; //some users monitor the *.ffs_db file with RTS => don't touch the file if it isnt't strictly needed //erase old session data if (itStreamOldL != streamsL.end()) streamsL.erase(itStreamOldL); if (itStreamOldR != streamsR.end()) streamsR.erase(itStreamOldR); //create new session data const std::string sessionID = generateGUID(); streamsL[sessionID] = std::move(sessionDataL); streamsR[sessionID] = std::move(sessionDataR); //------------ save DB files in parallel ------------------------- //1. create *both* ffs_tmp files first (caveat: *not* necessarily in parallel, depending on deviceParallelOps!) //2. if successful, rename both files (almost) transactionally! bool saveSuccessL = false; bool saveSuccessR = false; std::optional dbPathTmpL; std::optional dbPathTmpR; ZEN_ON_SCOPE_EXIT ( //*INDENT-OFF* if (dbPathTmpL) try { AFS::removeFilePlain(*dbPathTmpL); } catch (const FileError& e) { logExtraError(e.toString()); } if (dbPathTmpR) try { AFS::removeFilePlain(*dbPathTmpR); } catch (const FileError& e) { logExtraError(e.toString()); } //*INDENT-ON* ) std::vector> parallelWorkloadSave, parallelWorkloadMove; for (const auto& [dbPath, streams, saveSuccess, dbPathTmp] : { std::tuple(dbPathL, &streamsL, &saveSuccessL, &dbPathTmpL), std::tuple(dbPathR, &streamsR, &saveSuccessR, &dbPathTmpR) }) { parallelWorkloadSave.emplace_back(dbPath, [&streams = *streams, &saveSuccess = *saveSuccess, &dbPathTmp = *dbPathTmp, transactionalCopy](ParallelContext& ctx) //throw ThreadStopRequest { const std::wstring errMsg = tryReportingError([&] //throw ThreadStopRequest { StreamStatusNotifier notifySave(replaceCpy(_("Saving file %x..."), L"%x", fmtPath(AFS::getDisplayPath(ctx.itemPath))), ctx.acb); if (transactionalCopy && !AFS::hasNativeTransactionalCopy(ctx.itemPath)) //=> write (both?) DB files as a transaction { const Zstring shortGuid = printNumber(Zstr("%04x"), static_cast(getCrc16(generateGUID()))); const AbstractPath tmpPath = AFS::appendRelPath(*AFS::getParentPath(ctx.itemPath), AFS::getItemName(ctx.itemPath) + Zstr('.') + shortGuid + AFS::TEMP_FILE_ENDING); saveStreams(streams, tmpPath, notifySave); //throw FileError, ThreadStopRequest dbPathTmp = tmpPath; //pass file ownership } else //some MTP devices don't even allow renaming files: https://freefilesync.org/forum/viewtopic.php?t=6531 { AFS::removeFileIfExists(ctx.itemPath); //throw FileError saveStreams(streams, ctx.itemPath, notifySave); //throw FileError, ThreadStopRequest } }, ctx.acb); saveSuccess = errMsg.empty(); }); //---------------------------------------------------------------------------- if (transactionalCopy && !AFS::hasNativeTransactionalCopy(dbPath)) parallelWorkloadMove.emplace_back(dbPath, [&dbPathTmp = *dbPathTmp](ParallelContext& ctx) //throw ThreadStopRequest { tryReportingError([&] //throw ThreadStopRequest { //rename temp file (almost) transactionally: without write access, file creation would have failed AFS::removeFileIfExists(ctx.itemPath); //throw FileError AFS::moveAndRenameItem(*dbPathTmp, ctx.itemPath); //throw FileError, (ErrorMoveUnsupported) dbPathTmp = std::nullopt; //basically a "ScopeGuard::dismiss()" }, ctx.acb); }); } massParallelExecute(parallelWorkloadSave, Zstr("Save sync.ffs_db"), callback /*throw X*/); //throw X //---------------------------------------------------------------- if (saveSuccessL && saveSuccessR) massParallelExecute(parallelWorkloadMove, Zstr("Move sync.ffs_db"), callback /*throw X*/); //throw X }