// ***************************************************************************** // * This file is part of the FreeFileSync project. It is distributed under * // * GNU General Public License: https://www.gnu.org/licenses/gpl-3.0 * // * Copyright (C) Zenju (zenju AT freefilesync DOT org) - All Rights Reserved * // ***************************************************************************** #include "ftp.h" #include #include #include #include #include //DON'T include directly! #include "init_curl_libssh2.h" #include "ftp_common.h" #include "abstract_impl.h" //#include #include using namespace zen; using namespace fff; using AFS = AbstractFileSystem; namespace { //Extensions to FTP: https://tools.ietf.org/html/rfc3659 //FTP commands: https://en.wikipedia.org/wiki/List_of_FTP_commands constexpr std::chrono::seconds FTP_SESSION_MAX_IDLE_TIME (20); constexpr std::chrono::seconds FTP_SESSION_CLEANUP_INTERVAL(4); const size_t FTP_BLOCK_SIZE_DOWNLOAD = 64 * 1024; //libcurl returns blocks of only 16 kB as returned by recv() even if we request larger blocks via CURLOPT_BUFFERSIZE const size_t FTP_BLOCK_SIZE_UPLOAD = 64 * 1024; //libcurl requests blocks of 64 kB. larger blocksizes set via CURLOPT_UPLOAD_BUFFERSIZE do not seem to make a difference const size_t FTP_STREAM_BUFFER_SIZE = 1024 * 1024; //unit: [byte] //stream buffer should be big enough to facilitate prefetching during alternating read/write operations => e.g. see serialize.h::unbufferedStreamCopy() constexpr ZstringView ftpPrefix = Zstr("ftp:"); enum class ServerEncoding { unknown, utf8, ansi, }; inline uint16_t getEffectivePort(int portOption) { if (portOption > 0) return static_cast(portOption); return DEFAULT_PORT_FTP; } struct FtpDeviceId //= what defines a unique FTP location { FtpDeviceId(const FtpLogin& login) : server(login.server), port(getEffectivePort(login.portCfg)), username(login.username) {} Zstring server; uint16_t port; //must be valid port! Zstring username; }; std::weak_ordering operator<=>(const FtpDeviceId& lhs, const FtpDeviceId& rhs) { //exactly the type of case insensitive comparison we need for server names! https://docs.microsoft.com/en-us/windows/win32/api/ws2tcpip/nf-ws2tcpip-getaddrinfow#IDNs if (const std::weak_ordering cmp = compareAsciiNoCase(lhs.server, rhs.server); cmp != std::weak_ordering::equivalent) return cmp; return std::tie(lhs.port, lhs.username) <=> //username: case sensitive! std::tie(rhs.port, rhs.username); } //also needed by compareDeviceSameAfsType(), so can't just replace with hash and use std::unordered_map struct FtpSessionCfg //= config for buffered FTP session { FtpDeviceId deviceId; Zstring password; bool useTls = false; }; bool operator==(const FtpSessionCfg& lhs, const FtpSessionCfg& rhs) { if (lhs.deviceId <=> rhs.deviceId != std::weak_ordering::equivalent) return false; return std::tie(lhs.password, lhs.useTls) == //password: case sensitive! std::tie(rhs.password, rhs.useTls); } Zstring concatenateFtpFolderPathPhrase(const FtpLogin& login, const AfsPath& itemPath); //noexcept Zstring ansiToUtfEncoding(const std::string_view& str) //throw SysError { if (str.empty()) return {}; gsize bytesWritten = 0; //not including the terminating null GError* error = nullptr; ZEN_ON_SCOPE_EXIT(if (error) ::g_error_free(error)); //https://developer.gnome.org/glib/stable/glib-Character-Set-Conversion.html#g-convert gchar* utfStr = ::g_convert(str.data(), //const gchar* str str.size(), //gssize len "UTF-8", //const gchar* to_codeset "LATIN1", //const gchar* from_codeset nullptr, //gsize* bytes_read &bytesWritten, //gsize* bytes_written &error); //GError** error if (!utfStr) throw SysError(formatGlibError("g_convert(" + std::string(str) + ", LATIN1 -> UTF-8)", error)); ZEN_ON_SCOPE_EXIT(::g_free(utfStr)); return {utfStr, bytesWritten}; } std::string utfToAnsiEncoding(const Zstring& str) //throw SysError { if (str.empty()) return {}; const Zstring& strNorm = getUnicodeNormalForm(str); //convert to pre-composed *before* attempting conversion gsize bytesWritten = 0; //not including the terminating null GError* error = nullptr; ZEN_ON_SCOPE_EXIT(if (error) ::g_error_free(error)); //fails for: 1. broken UTF-8 2. not-ANSI-encodable Unicode gchar* ansiStr = ::g_convert(strNorm.c_str(), //const gchar* str strNorm.size(), //gssize len "LATIN1", //const gchar* to_codeset "UTF-8", //const gchar* from_codeset nullptr, //gsize* bytes_read &bytesWritten, //gsize* bytes_written &error); //GError** error if (!ansiStr) throw SysError(formatGlibError("g_convert(" + utfTo(strNorm) + ", UTF-8 -> LATIN1)", error)); ZEN_ON_SCOPE_EXIT(::g_free(ansiStr)); return {ansiStr, bytesWritten}; } std::wstring getCurlDisplayPath(const FtpDeviceId& deviceId, const AfsPath& itemPath) { Zstring displayPath = Zstring(ftpPrefix) + Zstr("//"); if (!deviceId.username.empty()) //show username! consider AFS::compareDeviceSameAfsType() displayPath += deviceId.username + Zstr('@'); //if (parseIpv6Address(deviceId.server) && deviceId.port != DEFAULT_PORT_FTP) // displayPath += Zstr('[') + deviceId.server + Zstr(']'); //else displayPath += deviceId.server; //if (deviceId.port != DEFAULT_PORT_FTP) // displayPath += Zstr(':') + numberTo(deviceId.port); const Zstring& relPath = getServerRelPath(itemPath); if (relPath != Zstr("/")) displayPath += relPath; return utfTo(displayPath); } std::vector splitFtpResponse(std::string&&) = delete; std::vector splitFtpResponse(const std::string& buf) { std::vector lines; split2(buf, [](const char c) { return isLineBreak(c) || c == '\0'; }, //is 0-char check even needed? [&lines](const std::string_view block) { if (!block.empty()) //consider Windows' lines.push_back(block); }); return lines; } class FtpLineParser { public: explicit FtpLineParser(const std::string_view& line) : it_(line.begin()), itEnd_(line.end()) {} /**/ FtpLineParser(std::string_view&&) = delete; template std::string_view readRange(size_t count, Function acceptChar) //throw SysError { if (static_cast(count) > itEnd_ - it_) throw SysError(L"Unexpected end of line."); const auto rngEnd = it_ + count; if (!std::all_of(it_, rngEnd, acceptChar)) throw SysError(L"Expected char type not found."); return makeStringView(std::exchange(it_, rngEnd), rngEnd); } template //expects non-empty range! std::string_view readRange(Function acceptChar) //throw SysError { auto rngEnd = std::find_if_not(it_, itEnd_, acceptChar); if (rngEnd == it_) throw SysError(L"Expected char range not found."); return makeStringView(std::exchange(it_, rngEnd), rngEnd); } char peekNextChar() const { return it_ == itEnd_ ? '\0' : *it_; } private: /**/ std::string_view::const_iterator it_; const std::string_view::const_iterator itEnd_; }; //---------------------------------------------------------------------------------------------------------------- std::wstring formatFtpStatus(int sc) { const wchar_t* statusText = [&] //https://en.wikipedia.org/wiki/List_of_FTP_server_return_codes { switch (sc) { case 400: return L"The command was not accepted but the error condition is temporary."; case 421: return L"Service not available, closing control connection."; case 425: return L"Cannot open data connection."; case 426: return L"Connection closed; transfer aborted."; case 430: return L"Invalid username or password."; case 431: return L"Need some unavailable resource to process security."; case 434: return L"Requested host unavailable."; case 450: return L"Requested file action not taken."; case 451: return L"Local error in processing."; case 452: return L"Insufficient storage space in system. File unavailable, e.g. file busy."; case 500: return L"Syntax error, command unrecognized or command line too long."; case 501: return L"Syntax error in parameters or arguments."; case 502: return L"Command not implemented."; case 503: return L"Bad sequence of commands."; case 504: return L"Command not implemented for that parameter."; case 521: return L"Data connection cannot be opened with this PROT setting."; case 522: return L"Server does not support the requested network protocol."; case 530: return L"User not logged in."; case 532: return L"Need account for storing files."; case 533: return L"Command protection level denied for policy reasons."; case 534: return L"Could not connect to server; issue regarding SSL."; case 535: return L"Failed security check."; case 536: return L"Requested PROT level not supported by mechanism."; case 537: return L"Command protection level not supported by security mechanism."; case 550: return L"File unavailable, e.g. file not found, no access."; case 551: return L"Requested action aborted. Page type unknown."; case 552: return L"Requested file action aborted. Exceeded storage allocation."; case 553: return L"File name not allowed."; default: return L""; } }(); if (strLength(statusText) == 0) return trimCpy(replaceCpy(L"FTP status %x.", L"%x", numberTo(sc))); else return trimCpy(replaceCpy(L"FTP status %x: ", L"%x", numberTo(sc)) + statusText); } //================================================================================================================ //================================================================================================================ struct SysErrorFtpProtocol : public zen::SysError { SysErrorFtpProtocol(const std::wstring& msg, long ftpError) : SysError(msg), ftpErrorCode(ftpError) {} long ftpErrorCode; }; DEFINE_NEW_SYS_ERROR(SysErrorPassword) constinit Global globalFtpSessionCount; GLOBAL_RUN_ONCE(globalFtpSessionCount.set(createUniSessionCounter())); class FtpSession { public: explicit FtpSession(const FtpSessionCfg& sessionCfg) : //throw SysError sessionCfg_(sessionCfg) { lastSuccessfulUseTime_ = std::chrono::steady_clock::now(); } ~FtpSession() { if (easyHandle_) ::curl_easy_cleanup(easyHandle_); } const FtpSessionCfg& getSessionCfg() const { return sessionCfg_; } //set *before* calling any of the subsequent functions; see FtpSessionManager::access() void setContextTimeout(const std::weak_ptr& timeoutSec) { timeoutSec_ = timeoutSec; } //returns server response (header data) std::string perform(const AfsPath& itemPath, bool isDir, long pathMethod, const std::vector& extraOptions, bool requestUtf8) //throw SysError, SysErrorPassword, SysErrorFtpProtocol { if (requestUtf8) //avoid endless recursion initUtf8(); //throw SysError, SysErrorFtpProtocol if (!easyHandle_) { easyHandle_ = ::curl_easy_init(); if (!easyHandle_) throw SysError(formatSystemError("curl_easy_init", formatCurlStatusCode(CURLE_OUT_OF_MEMORY), L"")); } else ::curl_easy_reset(easyHandle_); auto setCurlOption = [easyHandle = easyHandle_](const CurlOption& curlOpt) //throw SysError { if (const CURLcode rc = ::curl_easy_setopt(easyHandle, curlOpt.option, curlOpt.value); rc != CURLE_OK) throw SysError(formatSystemError("curl_easy_setopt(" + numberTo(static_cast(curlOpt.option)) + ")", formatCurlStatusCode(rc), utfTo(::curl_easy_strerror(rc)))); }; char curlErrorBuf[CURL_ERROR_SIZE] = {}; setCurlOption({CURLOPT_ERRORBUFFER, curlErrorBuf}); //throw SysError std::string headerData; curl_write_callback onHeaderReceived = [](/*const*/ char* buffer, size_t size, size_t nitems, void* callbackData) { auto& output = *static_cast(callbackData); output.append(buffer, size * nitems); return size * nitems; }; setCurlOption({CURLOPT_HEADERDATA, &headerData}); //throw SysError setCurlOption({CURLOPT_HEADERFUNCTION, onHeaderReceived}); //throw SysError setCurlOption({CURLOPT_URL, getCurlUrlPath(itemPath, isDir).c_str()}); //throw SysError assert(pathMethod != CURLFTPMETHOD_MULTICWD); //too slow! setCurlOption({CURLOPT_FTP_FILEMETHOD, pathMethod}); //throw SysError if (!sessionCfg_.deviceId.username.empty()) //else: libcurl will default to CURL_DEFAULT_USER("anonymous") and CURL_DEFAULT_PASSWORD("ftp@example.com") { //ANSI or UTF encoding? // "modern" FTP servers (implementing RFC 2640) have UTF8 enabled by default => pray and hope for the best. // What about ANSI-FTP servers and "Microsoft FTP Service" which requires "OPTS UTF8 ON"? => *psh* // CURLOPT_PREQUOTE to the rescue? Nope, issued long after USER/PASS setCurlOption({CURLOPT_USERNAME, utfTo(sessionCfg_.deviceId.username).c_str()}); //throw SysError setCurlOption({CURLOPT_PASSWORD, utfTo(sessionCfg_.password ).c_str()}); //throw SysError //curious: libcurl will *not* default to CURL_DEFAULT_USER when setting password but no username } setCurlOption({CURLOPT_PORT, sessionCfg_.deviceId.port}); //throw SysError //thread-safety: https://curl.haxx.se/libcurl/c/threadsafe.html setCurlOption({CURLOPT_NOSIGNAL, 1}); //throw SysError //allow PASV IP: some FTP servers really use IP different from control connection setCurlOption({CURLOPT_FTP_SKIP_PASV_IP, 0}); //throw SysError //let's not hold our breath until Curl adds a reasonable PASV handling => patch libcurl accordingly! //https://github.com/curl/curl/issues/1455 //https://github.com/curl/curl/pull/1470 //support broken servers like this one: https://freefilesync.org/forum/viewtopic.php?t=4301 const std::shared_ptr timeoutSec = timeoutSec_.lock(); assert(timeoutSec); if (!timeoutSec) throw std::runtime_error(std::string(__FILE__) + '[' + numberTo(__LINE__) + "] FtpSession: Timeout duration was not set."); setCurlOption({CURLOPT_CONNECTTIMEOUT, *timeoutSec}); //throw SysError //CURLOPT_TIMEOUT: "Since this puts a hard limit for how long time a request is allowed to take, it has limited use in dynamic use cases with varying transfer times." setCurlOption({CURLOPT_LOW_SPEED_TIME, *timeoutSec}); //throw SysError setCurlOption({CURLOPT_LOW_SPEED_LIMIT, 1 /*[bytes]*/}); //throw SysError //can't use "0" which means "inactive", so use some low number setCurlOption({CURLOPT_SERVER_RESPONSE_TIMEOUT, *timeoutSec}); //throw SysError //FTP only; unlike CURLOPT_TIMEOUT, this one is NOT a limit on the total transfer time //CURLOPT_ACCEPTTIMEOUT_MS? => only relevant for "active" FTP connections //long-running file uploads require keep-alives for the TCP control connection: https://freefilesync.org/forum/viewtopic.php?t=6928 setCurlOption({CURLOPT_TCP_KEEPALIVE, 1}); //throw SysError //=> CURLOPT_TCP_KEEPIDLE (=delay until sending first keepalive probe) and // CURLOPT_TCP_KEEPINTVL (interval between probes) both default to 60 sec, // CURLOPT_TCP_KEEPCNT (number of probes with *no server response* before dropping connection) defaults to 9 std::optional socketException; //libcurl does *not* set FD_CLOEXEC for us! https://github.com/curl/curl/issues/2252 auto onSocketCreate = [&](curl_socket_t curlfd, curlsocktype purpose) { assert(::fcntl(curlfd, F_GETFD) == 0); if (::fcntl(curlfd, F_SETFD, FD_CLOEXEC) == -1) //=> RACE-condition if other thread calls fork/execv before this thread sets FD_CLOEXEC! { socketException = SysError(formatSystemError("fcntl(FD_CLOEXEC)", errno)); return CURL_SOCKOPT_ERROR; } return CURL_SOCKOPT_OK; }; using SocketCbType = decltype(onSocketCreate); using SocketCbWrapperType = int (*)(SocketCbType* clientp, curl_socket_t curlfd, curlsocktype purpose); //needed for cdecl function pointer cast SocketCbWrapperType onSocketCreateWrapper = [](SocketCbType* clientp, curl_socket_t curlfd, curlsocktype purpose) { return (*clientp)(curlfd, purpose); //free this poor little C-API from its shackles and redirect to a proper lambda }; setCurlOption({CURLOPT_SOCKOPTFUNCTION, onSocketCreateWrapper}); //throw SysError setCurlOption({CURLOPT_SOCKOPTDATA, &onSocketCreate}); //throw SysError //Use share interface? https://curl.haxx.se/libcurl/c/libcurl-share.html //perf test, 4 and 8 parallel threads: // CURL_LOCK_DATA_DNS => no measurable total time difference // CURL_LOCK_DATA_SSL_SESSION => freefilesync.org; not working at all: lots of CURLE_RECV_ERROR (seems nobody ever tested this with truly parallel FTP accesses!) #if 0 do not include this into release! static CURLSH* curlShare = [] { struct ShareLocks { std::mutex lockIntenal; std::mutex lockDns; std::mutex lockSsl; }; static ShareLocks globalLocksTestingOnly; using LockFunType = void (*)(CURL* handle, curl_lock_data data, curl_lock_access access, void* userptr); //needed for cdecl function pointer cast LockFunType lockFun = [](CURL* handle, curl_lock_data data, curl_lock_access access, void* userptr) { auto& locks = *static_cast(userptr); switch (data) { case CURL_LOCK_DATA_SHARE: return locks.lockIntenal.lock(); case CURL_LOCK_DATA_DNS: return locks.lockDns.lock(); case CURL_LOCK_DATA_SSL_SESSION: return locks.lockSsl.lock(); } assert(false); }; using UnlockFunType = void (*)(CURL *handle, curl_lock_data data, void* userptr); UnlockFunType unlockFun = [](CURL *handle, curl_lock_data data, void* userptr) { auto& locks = *static_cast(userptr); switch (data) { case CURL_LOCK_DATA_SHARE: return locks.lockIntenal.unlock(); case CURL_LOCK_DATA_DNS: return locks.lockDns.unlock(); case CURL_LOCK_DATA_SSL_SESSION: return locks.lockSsl.unlock(); } assert(false); }; CURLSH* cs = ::curl_share_init(); assert(cs); CURLSHcode rc = CURLSHE_OK; rc = ::curl_share_setopt(cs, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); assert(rc == CURLSHE_OK); rc = ::curl_share_setopt(cs, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); //buggy!? assert(rc == CURLSHE_OK); rc = ::curl_share_setopt(cs, CURLSHOPT_LOCKFUNC, lockFun); assert(rc == CURLSHE_OK); rc = ::curl_share_setopt(cs, CURLSHOPT_UNLOCKFUNC, unlockFun); assert(rc == CURLSHE_OK); rc = ::curl_share_setopt(cs, CURLSHOPT_USERDATA, &globalLocksTestingOnly); assert(rc == CURLSHE_OK); return cs; }(); //CURLSHcode ::curl_share_cleanup(curlShare); setCurlOption({CURLOPT_SHARE, curlShare}); //throw SysError #endif //TODO: FTP option to require certificate checking? #if 0 setCurlOption({CURLOPT_CAINFO, "cacert.pem"}); //throw SysError //hopefully latest version from https://curl.haxx.se/docs/caextract.html //libcurl forwards this char-string to OpenSSL as is, which (thank god) accepts UTF8 #else setCurlOption({CURLOPT_CAINFO, 0}); //throw SysError //be explicit: "even when [CURLOPT_SSL_VERIFYPEER] is disabled [...] curl may still load the certificate file specified in CURLOPT_CAINFO." //check if server certificate can be trusted? (Default: 1L) // => may fail with: "CURLE_PEER_FAILED_VERIFICATION: SSL certificate problem: certificate has expired" setCurlOption({CURLOPT_SSL_VERIFYPEER, 0}); //throw SysError //check that server name matches the name in the certificate? (Default: 2L) // => may fail with: "CURLE_PEER_FAILED_VERIFICATION: SSL: no alternative certificate subject name matches target host name 'freefilesync.org'" setCurlOption({CURLOPT_SSL_VERIFYHOST, 0}); //throw SysError #endif if (sessionCfg_.useTls) //https://tools.ietf.org/html/rfc4217 { //require SSL for both control and data: setCurlOption({CURLOPT_USE_SSL, CURLUSESSL_ALL}); //throw SysError //try TLS first, then SSL (currently: CURLFTPAUTH_DEFAULT == CURLFTPAUTH_SSL): setCurlOption({CURLOPT_FTPSSLAUTH, CURLFTPAUTH_TLS}); //throw SysError } //support older FTP servers with less than 2048 bit TLS keys ("CURLE_SSL_CONNECT_ERROR: TLS connect error: error:0A00018A:SSL routines::dh key too small") //=> OpenSSL defaults to security level 2 (unless OPENSSL_TLS_SECURITY_LEVEL=level is defined during compilation) https://docs.openssl.org/master/man3/SSL_CTX_set_security_level/ setCurlOption({CURLOPT_SSL_CIPHER_LIST, "DEFAULT:@SECLEVEL=1"}); //throw SysError for (const CurlOption& option : extraOptions) setCurlOption(option); //throw SysError //======================================================================================================= const CURLcode rcPerf = ::curl_easy_perform(easyHandle_); //WTF: curl_easy_perform() considers FTP response codes >= 400 as failure, but for HTTP response codes 4XX are considered success!! CONSISTENCY, people!!! //note: CURLOPT_FAILONERROR(default:off) is only available for HTTP => BUT at least we can prefix FTP commands with * for same effect: https://curl.se/libcurl/c/CURLOPT_QUOTE.html if (socketException) throw* socketException; //throw SysError //======================================================================================================= if (rcPerf != CURLE_OK) { std::wstring errorMsg = trimCpy(utfTo(curlErrorBuf)); //optional if (const std::vector& headerLines = splitFtpResponse(headerData); !headerLines.empty()) if (const std::string_view& response = trimCpy(headerLines.back()); //that *should* be the server's error response !response.empty()) errorMsg += (errorMsg.empty() ? L"" : L"\n") + utfTo(response); #if 0 //utfTo(::curl_easy_strerror(ec)) is uninteresting //use CURLINFO_OS_ERRNO ?? https://curl.haxx.se/libcurl/c/CURLINFO_OS_ERRNO.html long nativeErrorCode = 0; if (::curl_easy_getinfo(easyHandle_, CURLINFO_OS_ERRNO, &nativeErrorCode) == CURLE_OK) if (nativeErrorCode != 0) errorMsg += (errorMsg.empty() ? L"" : L"\n") + std::wstring(L"Native error code: ") + numberTo(nativeErrorCode); #endif if (rcPerf == CURLE_LOGIN_DENIED) throw SysErrorPassword(formatSystemError("curl_easy_perform", formatCurlStatusCode(rcPerf), errorMsg)); long ftpStatusCode = 0; //optional /*const CURLcode rc =*/ ::curl_easy_getinfo(easyHandle_, CURLINFO_RESPONSE_CODE, &ftpStatusCode); //https://en.wikipedia.org/wiki/List_of_FTP_server_return_codes assert(rcPerf == CURLE_OPERATION_TIMEDOUT || rcPerf == CURLE_ABORTED_BY_CALLBACK || ftpStatusCode == 0 || 400 <= ftpStatusCode && ftpStatusCode < 600); if (ftpStatusCode != 0) throw SysErrorFtpProtocol(formatSystemError("curl_easy_perform", formatCurlStatusCode(rcPerf), errorMsg), ftpStatusCode); throw SysError(formatSystemError("curl_easy_perform", formatCurlStatusCode(rcPerf), errorMsg)); } lastSuccessfulUseTime_ = std::chrono::steady_clock::now(); return headerData; } //returns server response (header data) std::string runSingleFtpCommand(const std::string& ftpCmd, bool requestUtf8) //throw SysError, SysErrorFtpProtocol { curl_slist* quote = nullptr; ZEN_ON_SCOPE_EXIT(::curl_slist_free_all(quote)); quote = ::curl_slist_append(quote, ftpCmd.c_str()); return perform(AfsPath(), true /*isDir*/, CURLFTPMETHOD_NOCWD /*avoid needless CWDs*/, { {CURLOPT_NOBODY, 1L}, {CURLOPT_QUOTE, quote}, }, requestUtf8); //throw SysError, SysErrorPassword, SysErrorFtpProtocol } void testConnection() //throw SysError { /* https://en.wikipedia.org/wiki/List_of_FTP_commands FEAT: are there servers that don't support this command? fuck, yes: "550 FEAT: Operation not permitted" => buggy server not granting access, despite support! PWD? will fail if last access deleted the working dir! "TYPE I"? might interfere with libcurls internal handling, but that's an improvement, right? right? :> => but "HELP", and "NOOP" work, right?? Fuck my life: even "HELP" is not always implemented: https://freefilesync.org/forum/viewtopic.php?t=6002 => are there servers supporting neither FEAT nor HELP? only time will tell... ... and it tells! FUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU https://freefilesync.org/forum/viewtopic.php?t=8041 */ //=> '*' to the rescue: as long as we get an FTP response - *any* FTP response (including 550) - the connection itself is fine! const std::string& featBuf = runSingleFtpCommand("*FEAT", false /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol for (const std::string_view& line : splitFtpResponse(featBuf)) if (startsWith(line, "211 ") || startsWith(line, "500 ") || startsWith(line, "550 ")) return; //ever get here? throw SysError(L"Unexpected FTP response. (" + utfTo(featBuf) + L')'); } AfsPath getHomePath() //throw SysError { if (!homePathCached_) homePathCached_ = [&] { if (easyHandle_) { const char* homePathCurl = nullptr; //not owned /*CURLcode rc =*/ ::curl_easy_getinfo(easyHandle_, CURLINFO_FTP_ENTRY_PATH, &homePathCurl); if (homePathCurl && isAsciiString(homePathCurl)) return sanitizeDeviceRelativePath(utfTo(homePathCurl)); //home path with non-ASCII chars: libcurl issues PWD right after login *before* server was set up for UTF8 //=> CURLINFO_FTP_ENTRY_PATH could be in any encoding => useless! // Test case: Windows 10 IIS FTP with non-Ascii entry path //=> start new FTP session and parse PWD *after* UTF8 is enabled: ::curl_easy_cleanup(easyHandle_); easyHandle_ = nullptr; } const std::string& pwdBuf = runSingleFtpCommand("PWD", true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol for (const std::string_view& line : splitFtpResponse(pwdBuf)) if (startsWith(line, "257 ")) { /* 257[rubbish]"" according to libcurl "The directory name can contain any character; embedded double-quotes should be escaped by double-quotes (the "quote-doubling" convention)." https://tools.ietf.org/html/rfc959 */ auto itBegin = std::find(line.begin(), line.end(), '"'); if (itBegin != line.end()) for (auto it = ++itBegin; it != line.end(); ++it) if (*it == '"') { if (it + 1 != line.end() && it[1] == '"') ++it; //skip double quote else { const std::string homePathRaw = replaceCpy(std::string{itBegin, it}, "\"\"", '"'); const Zstring homePathUtf = serverToUtfEncoding(homePathRaw); //throw SysError return sanitizeDeviceRelativePath(homePathUtf); } } break; } throw SysError(L"Unexpected FTP response. (" + utfTo(pwdBuf) + L')'); }(); return *homePathCached_; } void ensureBinaryMode() //throw SysError { if (std::optional currentSocket = getActiveSocket()) //throw SysError if (*currentSocket == binaryEnabledSocket_) return; runSingleFtpCommand("TYPE I", false /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol //make sure our binary-enabled session is still there (== libcurl behaves as we expect) std::optional currentSocket = getActiveSocket(); //throw SysError if (currentSocket) binaryEnabledSocket_ = *currentSocket; //remember what we did //libcurl already buffers "conn->proto.ftpc.transfertype" but selfishly keeps it for itself! //=> pray libcurl doesn't internally set "TYPE A"! //=> this seems to be the only place where it does: https://github.com/curl/curl/issues/4342 else throw SysError(L"Curl failed to cache FTP session."); //why is libcurl not caching the session??? } //------------------------------------------------------------------------------------------------------------ bool supportsMlsd() { return getFeatureSupport(&Features::mlsd); } // bool supportsMfmt() { return getFeatureSupport(&Features::mfmt); } //throw SysError bool supportsClnt() { return getFeatureSupport(&Features::clnt); } // bool supportsUtf8() { if (getFeatureSupport(&Features::utf8)) return true; initUtf8(); //vsFTPd (ftp.sunet.se): supports UTF8 via "OPTS UTF8 ON", even if "UTF8" is missing from "FEAT" return socketUsesUtf8_; } bool isHealthy() const { return std::chrono::steady_clock::now() - lastSuccessfulUseTime_ <= FTP_SESSION_MAX_IDLE_TIME; } std::string getServerPathInternal(const AfsPath& itemPath) //throw SysError { const Zstring serverPath = getServerRelPath(itemPath); if (itemPath.value.empty()) //endless recursion caveat!! utfToServerEncoding() transitively depends on getServerPathInternal() return utfTo(serverPath); return utfToServerEncoding(serverPath); //throw SysError } Zstring serverToUtfEncoding(const std::string_view& str) //throw SysError { if (isAsciiString(str)) //fast path return {str.begin(), str.end()}; switch (encoding_) //throw SysError { case ServerEncoding::unknown: /* "UTF-8 encodings [2] contain enough internal structure that it is always, in practice, possible to determine whether a UTF-8 or raw encoding has been used" - https://www.rfc-editor.org/rfc/rfc3659#section-2.2 "encoding rules make it very unlikely that a character sequence from a different character set will be mistaken for a UTF-8 encoded character sequence." - https://www.rfc-editor.org/rfc/rfc2640#section-2.2 => auto-detect encoding even if FEAT does not advertize UTF8: https://freefilesync.org/forum/viewtopic.php?t=9564 */ encoding_ = supportsUtf8() || isValidUtf(str) ? ServerEncoding::utf8 : ServerEncoding::ansi; return serverToUtfEncoding(str); //throw SysError case ServerEncoding::utf8: if (!isValidUtf(str)) throw SysError(_("Invalid character encoding:") + L' ' + utfTo(str) + L' ' + _("Expected:") + L" [UTF-8]"); return utfTo(str); case ServerEncoding::ansi: return ansiToUtfEncoding(str); //throw SysError } assert(false); return {}; } std::string utfToServerEncoding(const Zstring& str) //throw SysError { if (isAsciiString(str)) //fast path return {str.begin(), str.end()}; switch (encoding_) //throw SysError { case ServerEncoding::unknown: if (!supportsUtf8()) throw SysError(_("Failed to auto-detect character encoding:") + L' ' + utfTo(str)); //might be ANSI or UTF8 with non-compliant server... encoding_ = ServerEncoding::utf8; return utfToServerEncoding(str); //throw SysError case ServerEncoding::utf8: //validate! we consider REPLACEMENT_CHAR as indication for server using ANSI encoding in serverToUtfEncoding() if (!isValidUtf(str)) throw SysError(_("Invalid character encoding:") + L' ' + utfTo(str) + L' ' + _("Expected:") + (sizeof(str[0]) == 1 ? L" [UTF-8]" : L" [UTF-16]")); static_assert(sizeof(str[0]) == 1 || sizeof(str[0]) == 2); return utfTo(str); case ServerEncoding::ansi: return utfToAnsiEncoding(str); //throw SysError } assert(false); return {}; } private: FtpSession (const FtpSession&) = delete; FtpSession& operator=(const FtpSession&) = delete; std::string getCurlUrlPath(const AfsPath& itemPath /*optional*/, bool isDir) //throw SysError { std::string curlRelPath; //libcurl expects encoded paths (except for '/' char!!!) => bug: https://github.com/curl/curl/pull/4423 split(getServerPathInternal(itemPath), //throw SysError '/', [&](std::string_view comp) { if (!comp.empty()) { char* compFmt = ::curl_easy_escape(easyHandle_, comp.data(), static_cast(comp.size())); if (!compFmt) throw SysError(formatSystemError(std::string("curl_easy_escape(") + comp + ')', L"", L"Conversion failure")); ZEN_ON_SCOPE_EXIT(::curl_free(compFmt)); if (!curlRelPath.empty()) curlRelPath += '/'; curlRelPath += compFmt; } }); if (trimCpy(sessionCfg_.deviceId.server).empty()) throw SysError(_("Server name must not be empty.")); static_assert(LIBCURL_VERSION_MAJOR > 7 || (LIBCURL_VERSION_MAJOR == 7 && LIBCURL_VERSION_MINOR >= 67)); /* 1. CURLFTPMETHOD_NOCWD requires absolute paths to unconditionally skip CWDs: https://github.com/curl/curl/pull/4382 2. CURLFTPMETHOD_SINGLECWD requires absolute paths to skip one needless "CWD entry path": https://github.com/curl/curl/pull/4332 => https://curl.se/docs/faq.html#How_do_I_list_the_root_directory => use // because /%2f had bugs (but they should be fixed: https://github.com/curl/curl/pull/4348) */ std::string path = utfTo(Zstring(ftpPrefix) + Zstr("//") + sessionCfg_.deviceId.server) + "//" + curlRelPath; if (isDir && !endsWith(path, '/')) //curl-FTP needs directory paths to end with a slash path += '/'; return path; } void initUtf8() //throw SysError, SysErrorFtpProtocol { /* 1. Some RFC-2640-non-compliant servers require UTF8 to be explicitly enabled: https://wiki.filezilla-project.org/Character_Encoding#Conflicting_specification - e.g. Microsoft FTP Service: https://freefilesync.org/forum/viewtopic.php?t=4303 2. Others do not advertize "UTF8" in "FEAT", but *still* allow enabling it via "OPTS UTF8 ON": - https://freefilesync.org/forum/viewtopic.php?t=9564 - vsFTPd: ftp.sunet.se https://security.appspot.com/vsftpd.html#download "OPTS UTF8 ON" needs to be activated each time libcurl internally creates a new session hopyfully libcurl will offer a better solution: https://github.com/curl/curl/issues/1457 */ if (std::optional currentSocket = getActiveSocket()) //throw SysError if (*currentSocket == utf8RequestedSocket_) //caveat: a non-UTF8-enabled session might already exist, e.g. from a previous call to supportsMlsd() return; //some (broken!?) servers require "CLNT" before accepting "OPTS UTF8 ON": https://social.msdn.microsoft.com/Forums/en-US/d602574f-8a69-4d69-b337-52b6081902cf/problem-with-ftpwebrequestopts-utf8-on-501-please-clnt-first if (supportsClnt()) //throw SysError runSingleFtpCommand("CLNT FreeFileSync", false /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol //"prefix the command with an asterisk to make libcurl continue even if the command fails" //-> ignore if server does not know this legacy command (but report all *other* issues; else getActiveSocket() below won't have a socket and we've hidden the real error!) const std::string& optsBuf = runSingleFtpCommand("*OPTS UTF8 ON", false /*requestUtf8*/); //throw SysError, (SysErrorFtpProtocol) //get *last* FTP status code (can there be more than one!?) int ftpStatusCode = 0; for (const std::string_view& line : splitFtpResponse(optsBuf)) if (line.size() >= 4 && isDigit(line[0]) && isDigit(line[1]) && isDigit(line[2]) && line[3] == ' ') ftpStatusCode = stringTo(line); socketUsesUtf8_ = ftpStatusCode == 200 || //"200 Always in UTF8 mode." "200 UTF8 set to on" ftpStatusCode == 202; //"202 UTF8 mode is always enabled." //make sure our Unicode-enabled session is still there (== libcurl behaves as we expect) std::optional currentSocket = getActiveSocket(); //throw SysError if (currentSocket) utf8RequestedSocket_ = *currentSocket; //remember what we did else throw SysError(L"Curl failed to cache FTP session."); //why is libcurl not caching the session??? } std::optional getActiveSocket() //throw SysError { if (easyHandle_) { curl_socket_t currentSocket = 0; const CURLcode rc = ::curl_easy_getinfo(easyHandle_, CURLINFO_ACTIVESOCKET, ¤tSocket); if (rc != CURLE_OK) throw SysError(formatSystemError("curl_easy_getinfo(CURLINFO_ACTIVESOCKET)", formatCurlStatusCode(rc), utfTo(::curl_easy_strerror(rc)))); if (currentSocket != CURL_SOCKET_BAD) return currentSocket; } return {}; } struct Features { bool mlsd = false; bool mfmt = false; bool clnt = false; bool utf8 = false; }; using FeatureList = std::unordered_map; bool getFeatureSupport(bool Features::* status) //throw SysError { if (!featureCache_) { static constinit FunStatGlobal> globalServerFeatures; globalServerFeatures.setOnce([] { return std::make_unique>(); }); const auto sf = globalServerFeatures.get(); if (!sf) throw SysError(formatSystemError("FtpSession::getFeatureSupport", L"", L"Function call not allowed during application shutdown.")); sf->access([&](const FeatureList& featList) { auto it = featList.find(sessionCfg_.deviceId.server); if (it != featList.end()) featureCache_ = it->second; }); if (!featureCache_) { //*: ignore error if server does not support/allow FEAT featureCache_ = parseFeatResponse(runSingleFtpCommand("*FEAT", false /*requestUtf8*/)); //throw SysError, (SysErrorFtpProtocol) //used by initUtf8()! => requestUtf8 = false!!! sf->access([&](FeatureList& feat) { feat.emplace(sessionCfg_.deviceId.server, *featureCache_); }); } } return (*featureCache_).*status; } static Features parseFeatResponse(const std::string& featResponse) { Features output; //FEAT command: https://tools.ietf.org/html/rfc2389#page-4 std::vector lines = splitFtpResponse(featResponse); auto it = std::find_if(lines.begin(), lines.end(), [](const std::string_view& line) { return startsWith(line, "211-") || startsWith(line, "211 "); }); if (it != lines.end()) { ++it; for (; it != lines.end(); ++it) { if (equalAsciiNoCase (*it, "211 End") || //Serv-U: "211 End (for details use "HELP commmand" where command is the command of interest)" startsWithAsciiNoCase(*it, "211 End ")) //Home Ftp Server: "211 End of extentions." break; std::string line(*it); //suppport ProFTPD with "MultilineRFC2228 = on" https://freefilesync.org/forum/viewtopic.php?t=7243 if (startsWith(line, "211-")) line = ' ' + afterFirst(line, '-', IfNotFoundReturn::none); //https://tools.ietf.org/html/rfc3659#section-7.8 //"a server-FTP process that supports MLST, and MLSD [...] MUST indicate that this support exists" //"there is no distinct FEAT output for MLSD. The presence of the MLST feature indicates that both MLST and MLSD are supported" if (equalAsciiNoCase (line, " MLST") || startsWithAsciiNoCase(line, " MLST ") || //SP "MLST" [SP factlist] CRLF //so much the theory. In practice FTP server implementers can't read (specs): https://freefilesync.org/forum/viewtopic.php?t=6752 equalAsciiNoCase(line, " MLSD")) output.mlsd = true; //https://tools.ietf.org/html/draft-somers-ftp-mfxx-04#section-3.3 //"Where a server-FTP process supports the MFMT command [...] it MUST include the response to the FEAT command" else if (equalAsciiNoCase(line, " MFMT")) //SP "MFMT" CRLF output.mfmt = true; else if (equalAsciiNoCase(line, " UTF8") || equalAsciiNoCase(line, " UTF8 ON") || //support non-compliant servers: https://freefilesync.org/forum/viewtopic.php?t=7355#p24694 equalAsciiNoCase(line, " UTF-8")) //Android 12: "File Manager" by Xiaomi output.utf8 = true; else if (equalAsciiNoCase(line, " CLNT")) output.clnt = true; } } return output; } const FtpSessionCfg sessionCfg_; CURL* easyHandle_ = nullptr; curl_socket_t utf8RequestedSocket_ = 0; curl_socket_t binaryEnabledSocket_ = 0; bool socketUsesUtf8_ = false; ServerEncoding encoding_ = ServerEncoding::unknown; std::optional featureCache_; std::optional homePathCached_; const std::shared_ptr libsshCurlUnifiedInitCookie_{getLibsshCurlUnifiedInitCookie(globalFtpSessionCount)}; //throw SysError std::chrono::steady_clock::time_point lastSuccessfulUseTime_; std::weak_ptr timeoutSec_; }; //================================================================================================================ //================================================================================================================ class FtpSessionManager //reuse (healthy) FTP sessions globally { struct FtpSessionCache; public: FtpSessionManager() : sessionCleaner_([this] { setCurrentThreadName(Zstr("Session Cleaner[FTP]")); runGlobalSessionCleanUp(); /*throw ThreadStopRequest*/ }) {} void access(const FtpLogin& login, const std::function& useFtpSession /*throw X*/) //throw SysError, X { Protected& sessionCache = getSessionCache(login); std::unique_ptr ftpSession; //either or std::optional sessionCfg; // sessionCache.access([&](FtpSessionCache& cache) { if (!cache.activeCfg) //AFS::authenticateAccess() not called => authenticate implicitly! setActiveConfig(cache, login); //assume "isHealthy()" to avoid hitting server connection limits: (clean up of !isHealthy() after use, idle sessions via worker thread) if (!cache.idleFtpSessions.empty()) { ftpSession = std::move(cache.idleFtpSessions.back ()); /**/ cache.idleFtpSessions.pop_back(); } else sessionCfg = *cache.activeCfg; }); //create new FTP session outside the lock: 1. don't block other threads 2. non-atomic regarding "sessionCache"! => one session too many is not a problem! if (!ftpSession) ftpSession = std::make_unique(*sessionCfg); //throw SysError const std::shared_ptr timeoutSec = std::make_shared(login.timeoutSec); //context option: valid only for duration of this call! ftpSession->setContextTimeout(timeoutSec); ZEN_ON_SCOPE_EXIT ( //*INDENT-OFF* if (ftpSession->isHealthy()) //thread that created the "!isHealthy()" session is responsible for clean up (avoid hitting server connection limits!) sessionCache.access([&](FtpSessionCache& cache) { if (ftpSession->getSessionCfg() == *cache.activeCfg) //created outside the lock => check *again* cache.idleFtpSessions.push_back(std::move(ftpSession)); //pass ownership }); //*INDENT-ON* ); useFtpSession(*ftpSession); //throw X } void setActiveConfig(const FtpLogin& login) { getSessionCache(login).access([&](FtpSessionCache& cache) { setActiveConfig(cache, login); }); } void setSessionPassword(const FtpLogin& login, const Zstring& password) { getSessionCache(login).access([&](FtpSessionCache& cache) { cache.sessionPassword = password; setActiveConfig(cache, login); }); } private: FtpSessionManager (const FtpSessionManager&) = delete; FtpSessionManager& operator=(const FtpSessionManager&) = delete; Protected& getSessionCache(const FtpDeviceId& deviceId) { //single global session cache per login; life-time bound to globalInstance => never remove a sessionCache!!! Protected* sessionCache = nullptr; globalSessionCache_.access([&](GlobalFtpSessions& sessionsById) { sessionCache = &sessionsById[deviceId]; //get or create }); static_assert(std::is_same_v>>, "require std::map so that the pointers we return remain stable"); return *sessionCache; } void setActiveConfig(FtpSessionCache& cache, const FtpLogin& login) { if (cache.activeCfg) assert(std::all_of(cache.idleFtpSessions.begin(), cache.idleFtpSessions.end(), [&](const std::unique_ptr& session) { return session->getSessionCfg() == cache.activeCfg; })); else assert(cache.idleFtpSessions.empty()); const std::optional prevCfg = cache.activeCfg; cache.activeCfg = { .deviceId{login}, .password = login.password ? *login.password : cache.sessionPassword, .useTls = login.useTls, }; /* remove incompatible sessions: - avoid hitting FTP connection limit if some config uses TLS, but not the other: https://freefilesync.org/forum/viewtopic.php?t=8532 - logically consistent with AFS::compareDevice() - don't allow different authentication methods, when authenticateAccess() is called *once* per device in getFolderStatusParallel() - what user expects, e.g. when tesing changed settings in FTP login dialog */ if (cache.activeCfg != prevCfg) cache.idleFtpSessions.clear(); //run ~FtpSession *inside* the lock! => avoid hitting server limits! } //run a dedicated clean-up thread => it's unclear when the server let's a connection time out, so we do it preemptively //context of worker thread: void runGlobalSessionCleanUp() //throw ThreadStopRequest { std::chrono::steady_clock::time_point lastCleanupTime; for (;;) { const auto now = std::chrono::steady_clock::now(); if (now < lastCleanupTime + FTP_SESSION_CLEANUP_INTERVAL) interruptibleSleep(lastCleanupTime + FTP_SESSION_CLEANUP_INTERVAL - now); //throw ThreadStopRequest lastCleanupTime = std::chrono::steady_clock::now(); std::vector*> sessionCaches; //pointers remain stable, thanks to std::map<> globalSessionCache_.access([&](GlobalFtpSessions& sessionsById) { for (auto& [sessionId, idleSession] : sessionsById) sessionCaches.push_back(&idleSession); }); for (Protected* sessionCache : sessionCaches) for (;;) { bool done = false; sessionCache->access([&](FtpSessionCache& cache) { for (std::unique_ptr& ftpSession : cache.idleFtpSessions) if (!ftpSession->isHealthy()) //!isHealthy() sessions are destroyed after use => in this context this means they have been idle for too long { ftpSession.swap(cache.idleFtpSessions.back()); /**/ cache.idleFtpSessions.pop_back(); //run ~FtpSession *inside* the lock! => avoid hitting server limits! return; //don't hold lock for too long: delete only one session at a time, then yield... } done = true; }); if (done) break; std::this_thread::yield(); //outside the lock } } } struct FtpSessionCache { //invariant: all cached sessions correspond to activeCfg at any time! std::vector> idleFtpSessions; //extract *temporarily* from this list during use std::optional activeCfg; Zstring sessionPassword; }; using GlobalFtpSessions = std::map>; Protected globalSessionCache_; InterruptibleThread sessionCleaner_; }; //-------------------------------------------------------------------------------------- UniInitializer globalStartupInitFtp(*globalFtpSessionCount.get()); constinit Global globalFtpSessionManager; //caveat: life time must be subset of static UniInitializer! //-------------------------------------------------------------------------------------- void accessFtpSession(const FtpLogin& login, const std::function& useFtpSession /*throw X*/) //throw SysError, X { if (const std::shared_ptr mgr = globalFtpSessionManager.get()) mgr->access(login, useFtpSession); //throw SysError, X else throw SysError(formatSystemError("accessFtpSession", L"", L"Function call not allowed during init/shutdown.")); } //=========================================================================================================================== struct FtpItem { AFS::ItemType type = AFS::ItemType::file; Zstring itemName; uint64_t fileSize = 0; time_t modTime = 0; AFS::FingerPrint filePrint = 0; //optional }; //get info about *existing* symlink! FtpItem getFtpSymlinkInfo(const FtpLogin& login, const AfsPath& linkPath) //throw FileError { try { FtpItem output; assert(output.type == AFS::ItemType::file); output.itemName = AFS::getItemName(linkPath); std::string mdtmBuf; accessFtpSession(login, [&](FtpSession& session) //throw SysError { /* first test if we have a file; if it's a folder expect FTP code 550 alternative: assume folder and try traversal? NOPE: this can *succeed* for file symlinks with MLSD! (e.g. on freefilesync.org FTP) -> can't replace SIZE + MDTM with MLSD which doesn't follow symlinks! */ session.ensureBinaryMode(); //throw SysError //...or some server return ASCII size or fail with '550 SIZE not allowed in ASCII mode: https://freefilesync.org/forum/viewtopic.php?t=7669&start=30#p27742 const std::string sizeBuf = session.runSingleFtpCommand("*SIZE " + session.getServerPathInternal(linkPath), true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol //alternative: use libcurl + CURLINFO_CONTENT_LENGTH_DOWNLOAD_T? => nah, surprise (motherfucker)! libcurl adds needless "REST 0" command! for (const std::string_view& line : splitFtpResponse(sizeBuf)) if (startsWith(line, "213 ")) // 213[rubbish] according to libcurl { if (isDigit(line.back())) //https://tools.ietf.org/html/rfc3659#section-4 { auto it = std::find_if(line.rbegin(), line.rend(), [](const char c) { return !isDigit(c); }); output.fileSize = stringTo(makeStringView(it.base(), line.end())); mdtmBuf = session.runSingleFtpCommand("MDTM " + session.getServerPathInternal(linkPath), true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol return; } break; } else if (startsWith(line, "550 ")) //e.g. "550 I can only retrieve regular files" { output.type = AFS::ItemType::folder; return; } throw SysError(L"Unexpected FTP response. (" + utfTo(sizeBuf) + L')'); }); if (output.type == AFS::ItemType::folder) return output; output.modTime = [&] //https://tools.ietf.org/html/rfc3659#section-3 { for (const std::string_view& line : splitFtpResponse(mdtmBuf)) if (startsWith(line, "213 ")) // 213 YYYYMMDDHHMMSS[.sss] "Time values are always represented in UTC (GMT)" ...and libcurl thinks so, too { const auto itStart = line.begin() + 4; const auto itEnd = std::find(itStart, line.end(), '.'); if (const TimeComp tc = parseTime("%Y%m%d%H%M%S", makeStringView(itStart, itEnd)); tc != TimeComp()) if (const auto [modTime, timeValid] = utcToTimeT(tc); timeValid) return modTime; break; } throw SysError(L"Unexpected FTP response. (" + utfTo(mdtmBuf) + L')'); }(); return output; } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(getCurlDisplayPath(login, linkPath))), e.toString()); } } class FtpDirectoryReader { public: static std::vector execute(const FtpLogin& login, const AfsPath& dirPath) //throw SysError, SysErrorFtpProtocol { std::string rawListing; //get raw FTP directory listing curl_write_callback onBytesReceived = [](/*const*/ char* buffer, size_t size, size_t nitems, void* callbackData) { auto& listing = *static_cast(callbackData); listing.append(buffer, size * nitems); return size * nitems; //folder reading might take up to a minute in extreme cases (50,000 files): https://freefilesync.org/forum/viewtopic.php?t=5312 }; std::vector output; accessFtpSession(login, [&](FtpSession& session) //throw SysError { std::vector options = { {CURLOPT_WRITEDATA, &rawListing}, {CURLOPT_WRITEFUNCTION, onBytesReceived}, }; long pathMethod = CURLFTPMETHOD_SINGLECWD; if (session.supportsMlsd()) //throw SysError { options.emplace_back(CURLOPT_CUSTOMREQUEST, "MLSD"); //some FTP servers abuse https://tools.ietf.org/html/rfc3659#section-7.1 //and process wildcards characters inside the "dirpath"; see http://www.proftpd.org/docs/howto/Globbing.html // [] matches any character in the character set enclosed in the brackets // * (not between brackets) matches any string, including the empty string // ? (not between brackets) matches any single character // //of course this "helpfulness" blows up with MLSD + paths that incidentally contain wildcards: https://freefilesync.org/forum/viewtopic.php?t=5575 const bool pathHasWildcards = //=> globbing is reproducible even with freefilesync.org's FTP! contains(afterFirst(dirPath.value, Zstr('['), IfNotFoundReturn::none), Zstr(']')) || contains(dirPath.value, Zstr('*')) || contains(dirPath.value, Zstr('?')); if (!pathHasWildcards) pathMethod = CURLFTPMETHOD_NOCWD; //16% faster traversal compared to CURLFTPMETHOD_SINGLECWD (35% faster than CURLFTPMETHOD_MULTICWD) } //else: use "LIST" + CURLFTPMETHOD_SINGLECWD //caveat: let's better not use LIST parameters: https://cr.yp.to/ftp/list.html session.perform(dirPath, true /*isDir*/, pathMethod, options, true /*requestUtf8*/); //throw SysError, SysErrorPassword, SysErrorFtpProtocol if (session.supportsMlsd()) //throw SysError output = parseMlsd(rawListing, session); //throw SysError else output = parseUnknown(rawListing, session); //throw SysError }); return output; } private: FtpDirectoryReader (const FtpDirectoryReader&) = delete; FtpDirectoryReader& operator=(const FtpDirectoryReader&) = delete; static std::vector parseMlsd(const std::string& buf, FtpSession& session) //throw SysError { std::vector output; for (const std::string_view& line : splitFtpResponse(buf)) { FtpItem item = parseMlstLine(line, session); //throw SysError if (item.itemName != Zstr(".") && item.itemName != Zstr("..")) output.push_back(std::move(item)); } return output; } static FtpItem parseMlstLine(const std::string_view& rawLine, FtpSession& session) //throw SysError { /* https://tools.ietf.org/html/rfc3659 type=cdir;sizd=4096;modify=20170116230740;UNIX.mode=0755;UNIX.uid=874;UNIX.gid=869;unique=902g36e1c55; . type=pdir;sizd=4096;modify=20170116230740;UNIX.mode=0755;UNIX.uid=874;UNIX.gid=869;unique=902g36e1c55; .. type=file;size=4;modify=20170113063314;UNIX.mode=0600;UNIX.uid=874;UNIX.gid=869;unique=902g36e1c5d; readme.txt type=dir;sizd=4096;modify=20170117144634;UNIX.mode=0755;UNIX.uid=874;UNIX.gid=869;unique=902g36e418a; folder */ try { FtpItem item; auto itBegin = rawLine.begin(); if (startsWith(rawLine, ' ')) //leading blank is already trimmed if MLSD was processed by curl ++itBegin; auto itBlank = std::find(itBegin, rawLine.end(), ' '); if (itBlank == rawLine.end()) throw SysError(L"Item name not available."); const std::string_view facts = makeStringView(itBegin, itBlank); item.itemName = session.serverToUtfEncoding(makeStringView(itBlank + 1, rawLine.end())); //throw SysError std::string_view typeFact; std::string_view fileSize; split(facts, ';', [&](const std::string_view fact) { if (!fact.empty()) { if (startsWithAsciiNoCase(fact, "type=")) //must be case-insensitive!!! { const std::string_view tmp = afterFirst(fact, '=', IfNotFoundReturn::none); typeFact = beforeFirst(tmp, ':', IfNotFoundReturn::all); } else if (startsWithAsciiNoCase(fact, "size=")) fileSize = afterFirst(fact, '=', IfNotFoundReturn::none); else if (startsWithAsciiNoCase(fact, "modify=")) { std::string_view modifyFact = afterFirst(fact, '=', IfNotFoundReturn::none); modifyFact = beforeLast(modifyFact, '.', IfNotFoundReturn::all); //truncate millisecond precision if available const TimeComp tc = parseTime("%Y%m%d%H%M%S", modifyFact); if (tc == TimeComp()) throw SysError(L"Modification time is invalid."); if (const auto [modTime, timeValid] = utcToTimeT(tc); timeValid) item.modTime = modTime; else throw SysError(L"Modification time is invalid."); } else if (startsWithAsciiNoCase(fact, "unique=")) { /* https://tools.ietf.org/html/rfc3659#section-7.5.2 "The mapping between files, and unique fact tokens should be maintained, [...] for *at least* the lifetime of the control connection from user-PI to server-PI." => not necessarily *persistent* as far as the RFC goes! BUT: practially this will be the inode ID/file index, so we can assume persistence */ const std::string_view uniqueId = afterFirst(fact, '=', IfNotFoundReturn::none); assert(!uniqueId.empty()); item.filePrint = hashString(uniqueId); //other metadata to hash e.g. create fact? => not available on Linux-hosted FTP! } } }); if (equalAsciiNoCase(typeFact, "cdir")) return {AFS::ItemType::folder, Zstr("."), 0, 0}; if (equalAsciiNoCase(typeFact, "pdir")) return {AFS::ItemType::folder, Zstr(".."), 0, 0}; if (equalAsciiNoCase(typeFact, "dir")) item.type = AFS::ItemType::folder; else if (equalAsciiNoCase(typeFact, "OS.unix=slink") || //the OS.unix=slink:/target syntax is a hack and often skips equalAsciiNoCase(typeFact, "OS.unix=symlink")) //the target path after the colon: http://www.proftpd.org/docs/modules/mod_facts.html item.type = AFS::ItemType::symlink; //It may be a good idea to NOT check for type "file" explicitly: see comment in native.cpp //evaluate parsing errors right now (+ report raw entry in error message!) if (item.itemName.empty()) throw SysError(L"Item name not available."); if (item.type == AFS::ItemType::file) { if (fileSize.empty() || !std::all_of(fileSize.begin(), fileSize.end(), &isDigit)) throw SysError(L"File size not available."); //crazy, but can be "-1": https://freefilesync.org/forum/viewtopic.php?t=9720#p35757 item.fileSize = stringTo(fileSize); } return item; } catch (const SysError& e) { throw SysError(L"Unexpected FTP response. (" + utfTo(rawLine) + L") " + e.toString()); } } static std::vector parseUnknown(const std::string& buf, FtpSession& session) //throw SysError { if (!buf.empty() && isDigit(buf[0])) //lame test to distinguish Unix/Dos formats as internally used by libcurl return parseWindows(buf, session); //throw SysError return parseUnix(buf, session); // } //"ls -l" static std::vector parseUnix(const std::string& buf, FtpSession& session) //throw SysError { const std::vector lines = splitFtpResponse(buf); auto it = lines.begin(); if (it != lines.end() && startsWith(*it, "total ")) ++it; const time_t utcTimeNow = std::time(nullptr); const TimeComp tc = getUtcTime(utcTimeNow); if (tc == TimeComp()) throw SysError(L"Failed to determine current time: " + numberTo(utcTimeNow)); const int utcCurrentYear = tc.year; //different listing formats: better store at session level!? std::optional dirOwnerGroupCount; // std::optional fileOwnerGroupCount; //caveat: differentiate per item type: see alternative formats! std::optional linkOwnerGroupCount; // std::vector output; std::for_each(it, lines.end(), [&](const std::string_view line) { auto& ownerGroupCount = [&]() -> std::optional& { assert(!line.empty()); //see splitFtpResponse() switch (line[0]) { case 'd': return dirOwnerGroupCount; case 'l': return linkOwnerGroupCount; default : return fileOwnerGroupCount; } }(); //unix listing without group: https://freefilesync.org/forum/viewtopic.php?t=4306 if (!ownerGroupCount) ownerGroupCount = [&] { std::optional firstError; for (int i = 3; i-- > 0;) try { parseUnixLine(line, utcTimeNow, utcCurrentYear, i /*ownerGroupCount*/, session); //throw SysError return i; } catch (const SysError& e) { if (!firstError) firstError = e; } throw* firstError; //most likely the relevant one: https://freefilesync.org/forum/viewtopic.php?t=10798 }(); const FtpItem item = parseUnixLine(line, utcTimeNow, utcCurrentYear, *ownerGroupCount, session); //throw SysError if (item.itemName != Zstr(".") && item.itemName != Zstr("..")) output.push_back(item); }); return output; } static FtpItem parseUnixLine(const std::string_view& rawLine, time_t utcTimeNow, int utcCurrentYear, int ownerGroupCount, FtpSession& session) //throw SysError { /* Unix standard listing: "ls -l --all" total 4953 <- optional first line drwxr-xr-x 1 root root 4096 Jan 10 11:58 version -rwxr-xr-x 1 root root 1084 Sep 2 01:17 Unit Test.vcxproj.user -rwxr-xr-x 1 1000 300 2217 Feb 28 2016 win32.manifest lrwxr-xr-x 1 root root 18 Apr 26 15:17 Projects -> /mnt/hgfs/Projects file type: -:file l:symlink d:directory b:block device p:named pipe c:char device s:socket permissions: (r|-)(w|-)(x|s|S|-) user (r|-)(w|-)(x|s|S|-) group s := S + x S = Setgid (r|-)(w|-)(x|t|T|-) others t := T + x T = sticky bit Alternative formats ------------------- No group: "ls -l --no-group" https://freefilesync.org/forum/viewtopic.php?t=4306 dr-xr-xr-x 2 root 512 Apr 8 1994 etc No owner, no group, trailing slash (but only for directories!????): "ls -g --no-group --file-type" https://freefilesync.org/forum/viewtopic.php?t=10227 -rwxrwxrwx 1 ownername groupname 8064383 Mar 30 11:58 file.mp3 drwxrwxrwx 1 0 Jan 1 00:00 dirname/ Yet to be seen in the wild: Netware: d [R----F--] supervisor 512 Jan 16 18:53 login - [R----F--] rhesus 214059 Oct 20 15:27 cx.exe NetPresenz for the Mac: -------r-- 326 1391972 1392298 Nov 22 1995 MegaPhone.sit drwxrwxr-x folder 2 May 10 1996 network */ try { FtpLineParser parser(rawLine); const std::string_view typeTag = parser.readRange(1, [](const char c) //throw SysError { return c == '-' || c == 'b' || c == 'c' || c == 'd' || c == 'l' || c == 'p' || c == 's'; }); //------------------------------------------------------------------------------------ //permissions parser.readRange(9, [](const char c) //throw SysError { return c == '-' || c == 'r' || c == 'w' || c == 'x' || c == 's' || c == 'S' || c == 't' || c == 'T'; }); parser.readRange(&isWhiteSpace); //throw SysError //------------------------------------------------------------------------------------ //hard-link count (no separators) parser.readRange(&isDigit); //throw SysError parser.readRange(&isWhiteSpace); //throw SysError //------------------------------------------------------------------------------------ //both owner + group, owner only, or none at all assert(0 <= ownerGroupCount && ownerGroupCount <=2); for (int i = 0; i < ownerGroupCount; ++i) { parser.readRange(std::not_fn(isWhiteSpace)); //throw SysError parser.readRange(&isWhiteSpace); //throw SysError } //------------------------------------------------------------------------------------ //file size (no separators) const uint64_t fileSize = stringTo(parser.readRange(&isDigit)); //throw SysError parser.readRange(&isWhiteSpace); //throw SysError //------------------------------------------------------------------------------------ const std::string_view monthStr = parser.readRange(std::not_fn(isWhiteSpace)); //throw SysError parser.readRange(&isWhiteSpace); //throw SysError const char* months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; auto itMonth = std::find_if(std::begin(months), std::end(months), [&](const char* name) { return equalAsciiNoCase(name, monthStr); }); if (itMonth == std::end(months)) throw SysError(L"Failed to parse month name."); //------------------------------------------------------------------------------------ const int day = stringTo(parser.readRange(&isDigit)); //throw SysError parser.readRange(&isWhiteSpace); //throw SysError if (day < 1 || day > 31) throw SysError(L"Failed to parse day of month."); //------------------------------------------------------------------------------------ const std::string_view timeOrYear = parser.readRange([](const char c) { return c == ':' || isDigit(c); }); //throw SysError parser.readRange(&isWhiteSpace); //throw SysError TimeComp timeComp; timeComp.month = 1 + static_cast(itMonth - std::begin(months)); timeComp.day = day; if (contains(timeOrYear, ':')) { const int hour = stringTo(beforeFirst(timeOrYear, ':', IfNotFoundReturn::none)); const int minute = stringTo(afterFirst (timeOrYear, ':', IfNotFoundReturn::none)); if (hour < 0 || hour > 23 || minute < 0 || minute > 59) throw SysError(L"Failed to parse modification time."); timeComp.hour = hour; timeComp.minute = minute; timeComp.year = utcCurrentYear; //tentatively const auto [serverLocalTime, timeValid] = utcToTimeT(timeComp); if (!timeValid) throw SysError(L"Modification time is invalid."); if (serverLocalTime > utcTimeNow + 24 * 3600 ) //time-zones range from UTC-12:00 to UTC+14:00, consider DST; FileZilla uses 1 day tolerance --timeComp.year; //"more likely" this time is from last year } else if (timeOrYear.size() == 4) { timeComp.year = stringTo(timeOrYear); if (timeComp.year < 1600 || timeComp.year >= 3000) throw SysError(L"Failed to parse modification time."); } else throw SysError(L"Failed to parse modification time."); //let's pretend the time listing is UTC (same behavior as FileZilla): hopefully MLSD will make this mess obsolete soon... // => find exact offset with some MDTM hackery? yes, could do that, but this doesn't solve the bigger problem of imprecise LIST file times, so why bother? const auto [modTime, timeValid] = utcToTimeT(timeComp); if (!timeValid) throw SysError(L"Modification time is invalid."); //------------------------------------------------------------------------------------ const std::string_view trail = parser.readRange([](char) { return true; }); //throw SysError std::string_view itemName; if (typeTag == "l") itemName = beforeFirst(trail, " -> ", IfNotFoundReturn::none); else itemName = trail; if (itemName.empty()) throw SysError(L"Item name not available."); if (itemName == "." || itemName == "..") //sometimes returned, e.g. by freefilesync.org return {AFS::ItemType::folder, utfTo(itemName), 0, 0}; //------------------------------------------------------------------------------------ FtpItem item; if (typeTag == "d") item.type = AFS::ItemType::folder; else if (typeTag == "l") item.type = AFS::ItemType::symlink; else item.fileSize = fileSize; item.itemName = session.serverToUtfEncoding(itemName); //throw SysError if (item.type == AFS::ItemType::folder && endsWith(item.itemName, Zstr('/'))) item.itemName.pop_back(); item.modTime = modTime; return item; } catch (const SysError& e) { throw SysError(L"Unexpected FTP response. (" + utfTo(rawLine) + L") [ownerGroupCount: " + numberTo(ownerGroupCount) + L"] " + e.toString()); } } //"dir" static std::vector parseWindows(const std::string& buf, FtpSession& session) //throw SysError { /* Test server: test.rebex.net username:demo pw:password useTls = true listing supported by libcurl (US server) 10-27-15 03:46AM pub 04-08-14 03:09PM 11,399 readme.txt Datalogic Windows CE 5.0 01-01-98 13:00 Storage Card IIS option "four-digit years" 06-22-2017 04:25PM test 06-20-2017 12:50PM 1875499 zstring.obj Alternative formats (yet to be seen in the wild) "dir" on Windows, US: 10/27/2015 03:46 AM pub 04/08/2014 03:09 PM 11,399 readme.txt "dir" on Windows, German: 21.09.2016 18:31 Favorites 12.01.2017 19:57 11.399 gsview64.ini */ const TimeComp tc = getUtcTime(); if (tc == TimeComp()) throw SysError(L"Failed to determine current time: " + numberTo(std::time(nullptr))); const int utcCurrentYear = tc.year; std::vector output; for (const std::string_view& line : splitFtpResponse(buf)) { try { FtpLineParser parser(line); const int month = stringTo(parser.readRange(2, &isDigit)); //throw SysError parser.readRange(1, [](const char c) { return c == '-' || c == '/'; }); //throw SysError const int day = stringTo(parser.readRange(2, &isDigit)); //throw SysError parser.readRange(1, [](const char c) { return c == '-' || c == '/'; }); //throw SysError const std::string_view yearString = parser.readRange(&isDigit); //throw SysError parser.readRange(&isWhiteSpace); //throw SysError if (month < 1 || month > 12 || day < 1 || day > 31) throw SysError(L"Failed to parse modification time."); int year = 0; if (yearString.size() == 2) { year = (utcCurrentYear / 100) * 100 + stringTo(yearString); if (year > utcCurrentYear + 1 /*local time leeway*/) year -= 100; } else if (yearString.size() == 4) year = stringTo(yearString); else throw SysError(L"Failed to parse modification time."); //------------------------------------------------------------------------------------ int hour = stringTo(parser.readRange(2, &isDigit)); //throw SysError parser.readRange(1, [](const char c) { return c == ':'; }); //throw SysError const int minute = stringTo(parser.readRange(2, &isDigit)); //throw SysError if (!isWhiteSpace(parser.peekNextChar())) { const std::string_view period = parser.readRange(2, [](const char c) { return c == 'A' || c == 'P' || c == 'M'; }); //throw SysError if (period == "PM") { if (0 <= hour && hour < 12) hour += 12; } else if (hour == 12) hour = 0; } parser.readRange(&isWhiteSpace); //throw SysError if (hour < 0 || hour > 23 || minute < 0 || minute > 59) throw SysError(L"Failed to parse modification time."); //------------------------------------------------------------------------------------ TimeComp timeComp; timeComp.year = year; timeComp.month = month; timeComp.day = day; timeComp.hour = hour; timeComp.minute = minute; //let's pretend the time listing is UTC (same behavior as FileZilla): hopefully MLSD will make this mess obsolete soon... // => find exact offset with some MDTM hackery? yes, could do that, but this doesn't solve the bigger problem of imprecise LIST file times, so why bother? const auto [modTime, timeValid] = utcToTimeT(timeComp); if (!timeValid) throw SysError(L"Modification time is invalid."); //------------------------------------------------------------------------------------ const std::string_view dirTagOrSize = parser.readRange(std::not_fn(isWhiteSpace)); //throw SysError parser.readRange(&isWhiteSpace); //throw SysError const bool isDir = dirTagOrSize == ""; uint64_t fileSize = 0; if (!isDir) { std::string sizeStr(dirTagOrSize); replace(sizeStr, ',', ""); replace(sizeStr, '.', ""); if (sizeStr.empty() || !std::all_of(sizeStr.begin(), sizeStr.end(), &isDigit)) throw SysError(L"Failed to parse file size."); fileSize = stringTo(sizeStr); } //------------------------------------------------------------------------------------ const std::string_view itemName = parser.readRange([](char) { return true; }); //throw SysError if (itemName.empty()) throw SysError(L"Folder contains an item without name."); //------------------------------------------------------------------------------------ if (itemName != "." && itemName != "..") { FtpItem item; if (isDir) item.type = AFS::ItemType::folder; item.itemName = session.serverToUtfEncoding(itemName); //throw SysError item.fileSize = fileSize; item.modTime = modTime; output.push_back(item); } } catch (const SysError& e) { throw SysError(L"Unexpected FTP response. (" + utfTo(line) + L") " + e.toString()); } } return output; } }; class SingleFolderTraverser { public: SingleFolderTraverser(const FtpLogin& login, const std::vector>>& workload /*throw X*/) : workload_(workload), login_(login) { while (!workload_.empty()) { auto wi = std::move(workload_. back()); //yes, no strong exception guarantee (std::bad_alloc) /**/ workload_.pop_back(); // const auto& [folderPath, cb] = wi; tryReportingDirError([&] //throw X { traverseWithException(folderPath, *cb); //throw FileError, X }, *cb); } } private: SingleFolderTraverser (const SingleFolderTraverser&) = delete; SingleFolderTraverser& operator=(const SingleFolderTraverser&) = delete; void traverseWithException(const AfsPath& dirPath, AFS::TraverserCallback& cb) //throw FileError, X { std::vector items; try { items = FtpDirectoryReader::execute(login_, dirPath); //throw SysError, SysErrorFtpProtocol } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtPath(getCurlDisplayPath(login_, dirPath))), e.toString()); } for (const FtpItem& item : items) { const AfsPath itemPath(appendPath(dirPath.value, item.itemName)); switch (item.type) { case AFS::ItemType::file: cb.onFile({item.itemName, item.fileSize, item.modTime, item.filePrint, false /*isFollowedSymlink*/}); //throw X break; case AFS::ItemType::folder: if (std::shared_ptr cbSub = cb.onFolder({item.itemName, false /*isFollowedSymlink*/})) //throw X workload_.push_back({itemPath, std::move(cbSub)}); break; case AFS::ItemType::symlink: switch (cb.onSymlink({item.itemName, item.modTime})) //throw X { case AFS::TraverserCallback::HandleLink::follow: { FtpItem target = {}; if (!tryReportingItemError([&] //throw X { target = getFtpSymlinkInfo(login_, itemPath); //throw FileError }, cb, item.itemName)) continue; if (target.type == AFS::ItemType::folder) { if (std::shared_ptr cbSub = cb.onFolder({item.itemName, true /*isFollowedSymlink*/})) //throw X workload_.push_back({itemPath, std::move(cbSub)}); } else //a file or named pipe, etc. cb.onFile({item.itemName, target.fileSize, target.modTime, item.filePrint, true /*isFollowedSymlink*/}); //throw X } break; case AFS::TraverserCallback::HandleLink::skip: break; } break; } } } std::vector>> workload_; const FtpLogin login_; }; void traverseFolderRecursiveFTP(const FtpLogin& login, const std::vector>>& workload /*throw X*/, size_t) //throw X { SingleFolderTraverser dummy(login, workload); //throw X } //=========================================================================================================================== //=========================================================================================================================== void ftpFileDownload(const FtpLogin& login, const AfsPath& afsFilePath, //throw FileError, X const std::function& writeBlock /*throw X*/) { std::exception_ptr exception; auto onBytesReceived = [&](const void* buffer, size_t bytesToWrite) { try { writeBlock(buffer, bytesToWrite); //throw X //[!] let's NOT use "incomplete write Posix semantics" for libcurl! //who knows if libcurl buffers properly, or if it sends incomplete packages!? return bytesToWrite; } catch (...) { exception = std::current_exception(); return bytesToWrite + 1; //signal error condition => CURLE_WRITE_ERROR } }; curl_write_callback onBytesReceivedWrapper = [](char* buffer, size_t size, size_t nitems, void* callbackData) { return (*static_cast(callbackData))(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda }; try { accessFtpSession(login, [&](FtpSession& session) //throw SysError { session.perform(afsFilePath, false /*isDir*/, CURLFTPMETHOD_NOCWD, //are there any servers that require CURLFTPMETHOD_SINGLECWD? let's find out { {CURLOPT_WRITEDATA, &onBytesReceived}, {CURLOPT_WRITEFUNCTION, onBytesReceivedWrapper}, {CURLOPT_IGNORE_CONTENT_LENGTH, 1L}, //skip FTP "SIZE" command before download (=> download until actual EOF if file size changes) //{CURLOPT_BUFFERSIZE, 256 * 1024} -> default is 16 kB which seems to correspond to TLS packet size //=> setting larger buffer size does nothing (recv still returns only 16 kB) }, true /*requestUtf8*/); //throw SysError, SysErrorPassword, SysErrorFtpProtocol }); } catch (const SysError& e) { if (exception) std::rethrow_exception(exception); throw FileError(replaceCpy(_("Cannot read file %x."), L"%x", fmtPath(getCurlDisplayPath(login, afsFilePath))), e.toString()); } } /* File already existing: freefilesync.org: overwrites FileZilla Server: overwrites Windows IIS: overwrites */ void ftpFileUpload(const FtpLogin& login, const AfsPath& afsFilePath, const std::function& readBlock /*throw X*/) //throw FileError, X; return "bytesToRead" bytes unless end of stream { std::exception_ptr exception; auto getBytesToSend = [&](void* buffer, size_t bytesToRead) -> size_t { try { /* libcurl calls back until 0 bytes are returned (Posix read() semantics), or, if CURLOPT_INFILESIZE_LARGE was set, after exactly this amount of bytes [!] let's NOT use "incomplete read Posix semantics" for libcurl! who knows if libcurl buffers properly, or if it requests incomplete packages!? */ const size_t bytesRead = readBlock(buffer, bytesToRead); //throw X; return "bytesToRead" bytes unless end of stream assert(bytesRead == bytesToRead || bytesRead == 0 || readBlock(buffer, bytesToRead) == 0); return bytesRead; } catch (...) { exception = std::current_exception(); return CURL_READFUNC_ABORT; //signal error condition => CURLE_ABORTED_BY_CALLBACK } }; curl_read_callback getBytesToSendWrapper = [](char* buffer, size_t size, size_t nitems, void* callbackData) { return (*static_cast(callbackData))(buffer, size * nitems); //free this poor little C-API from its shackles and redirect to a proper lambda }; try { accessFtpSession(login, [&](FtpSession& session) //throw SysError { /* curl_slist* quote = nullptr; ZEN_ON_SCOPE_EXIT(::curl_slist_free_all(quote)); //"prefix the command with an asterisk to make libcurl continue even if the command fails" quote = ::curl_slist_append(quote, ("*DELE " + session.getServerPathInternal(afsFilePath)).c_str()); //throw SysError //optimize fail-safe copy with RNFR/RNTO as CURLOPT_POSTQUOTE? -> even slightly *slower* than RNFR/RNTO as additional curl_easy_perform() */ session.perform(afsFilePath, false /*isDir*/, CURLFTPMETHOD_NOCWD, //are there any servers that require CURLFTPMETHOD_SINGLECWD? let's find out { {CURLOPT_UPLOAD, 1L}, {CURLOPT_READDATA, &getBytesToSend}, {CURLOPT_READFUNCTION, getBytesToSendWrapper}, //{CURLOPT_UPLOAD_BUFFERSIZE, 256 * 1024} -> defaults is 64 kB. apparently no performance improvement for larger buffers like 256 kB //{CURLOPT_INFILESIZE_LARGE, static_cast(inputBuffer.size())}, //=> CURLOPT_INFILESIZE_LARGE does not issue a specific FTP command, but is used by libcurl only! //{CURLOPT_PREQUOTE, quote}, //{CURLOPT_POSTQUOTE, quote}, }, true /*requestUtf8*/); //throw SysError, SysErrorPassword, SysErrorFtpProtocol }); } catch (const SysError& e) { if (exception) std::rethrow_exception(exception); throw FileError(replaceCpy(_("Cannot write file %x."), L"%x", fmtPath(getCurlDisplayPath(login, afsFilePath))), e.toString()); } } //=========================================================================================================================== struct InputStreamFtp : public AFS::InputStream { InputStreamFtp(const FtpLogin& login, const AfsPath& filePath) { worker_ = InterruptibleThread([asyncStreamOut = this->asyncStreamIn_, login, filePath] { setCurrentThreadName(Zstr("Istream ") + utfTo(getCurlDisplayPath(login, filePath))); try { auto writeBlock = [&](const void* buffer, size_t bytesToWrite) { asyncStreamOut->write(buffer, bytesToWrite); //throw ThreadStopRequest }; ftpFileDownload(login, filePath, writeBlock); //throw FileError, ThreadStopRequest asyncStreamOut->closeStream(); } catch (FileError&) { asyncStreamOut->setWriteError(std::current_exception()); } //let ThreadStopRequest pass through! }); } ~InputStreamFtp() { asyncStreamIn_->setReadError(std::make_exception_ptr(ThreadStopRequest())); } size_t getBlockSize() override { return FTP_BLOCK_SIZE_DOWNLOAD; } //throw (FileError) //may return short; only 0 means EOF! CONTRACT: bytesToRead > 0! size_t tryRead(void* buffer, size_t bytesToRead, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, (ErrorFileLocked), X { const size_t bytesRead = asyncStreamIn_->tryRead(buffer, bytesToRead); //throw FileError reportBytesProcessed(notifyUnbufferedIO); //throw X return bytesRead; //no need for asyncStreamIn_->checkWriteErrors(): once end of stream is reached, asyncStreamOut->closeStream() was called => no errors occured } std::optional tryGetAttributesFast() override { return {}; }//throw FileError //there is no stream handle => no buffered attribute access! //PERF: get attributes during file download? // CURLOPT_FILETIME: test case 77 files, 4MB: overall copy time increases by 12% // CURLOPT_PREQUOTE/CURLOPT_PREQUOTE/CURLOPT_POSTQUOTE + MDTM: test case 77 files, 4MB: overall copy time increases by 12% private: void reportBytesProcessed(const IoCallback& notifyUnbufferedIO /*throw X*/) //throw X { const int64_t bytesDelta = makeSigned(asyncStreamIn_->getTotalBytesWritten()) - totalBytesReported_; totalBytesReported_ += bytesDelta; if (notifyUnbufferedIO) notifyUnbufferedIO(bytesDelta); //throw X } int64_t totalBytesReported_ = 0; std::shared_ptr asyncStreamIn_ = std::make_shared(FTP_STREAM_BUFFER_SIZE); InterruptibleThread worker_; }; //=========================================================================================================================== //CAVEAT: if upload fails due to already existing, OutputStreamFtp constructor does not fail, but OutputStreamFtp::write() does! // => ~OutputStreamImpl() will delete the already existing file! struct OutputStreamFtp : public AFS::OutputStreamImpl { OutputStreamFtp(const FtpLogin& login, const AfsPath& filePath, std::optional modTime) : login_(login), filePath_(filePath), modTime_(modTime) { std::promise promUploadDone; futUploadDone_ = promUploadDone.get_future(); worker_ = InterruptibleThread([login, filePath, asyncStreamIn = this->asyncStreamOut_, pUploadDone = std::move(promUploadDone)]() mutable { setCurrentThreadName(Zstr("Ostream ") + utfTo(getCurlDisplayPath(login, filePath))); try { auto readBlock = [&](void* buffer, size_t bytesToRead) { return asyncStreamIn->read(buffer, bytesToRead); //throw ThreadStopRequest }; ftpFileUpload(login, filePath, readBlock); //throw FileError, ThreadStopRequest assert(asyncStreamIn->getTotalBytesRead() == asyncStreamIn->getTotalBytesWritten()); pUploadDone.set_value(); } catch (FileError&) { const std::exception_ptr exptr = std::current_exception(); asyncStreamIn->setReadError(exptr); //set both! pUploadDone.set_exception(exptr); // } //let ThreadStopRequest pass through! }); } ~OutputStreamFtp() { if (asyncStreamOut_) //=> cleanup non-finalized output file { asyncStreamOut_->setWriteError(std::make_exception_ptr(ThreadStopRequest())); worker_.join(); try //see removeFilePlain() { accessFtpSession(login_, [&](FtpSession& session) //throw SysError { session.runSingleFtpCommand("DELE " + session.getServerPathInternal(filePath_), true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol }); } catch (const SysError& e) { logExtraError(replaceCpy(_("Cannot delete file %x."), L"%x", fmtPath(getCurlDisplayPath(login_, filePath_))) + L"\n\n" + e.toString()); } } } size_t getBlockSize() override { return FTP_BLOCK_SIZE_UPLOAD; } //throw (FileError) size_t tryWrite(const void* buffer, size_t bytesToWrite, const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X; may return short! CONTRACT: bytesToWrite > 0 { const size_t bytesWritten = asyncStreamOut_->tryWrite(buffer, bytesToWrite); //throw FileError reportBytesProcessed(notifyUnbufferedIO); //throw X return bytesWritten; } AFS::FinalizeResult finalize(const IoCallback& notifyUnbufferedIO /*throw X*/) override //throw FileError, X { if (!asyncStreamOut_) throw std::logic_error(std::string(__FILE__) + '[' + numberTo(__LINE__) + "] Contract violation!"); asyncStreamOut_->closeStream(); while (futUploadDone_.wait_for(std::chrono::milliseconds(25)) == std::future_status::timeout) reportBytesProcessed(notifyUnbufferedIO); //throw X reportBytesProcessed(notifyUnbufferedIO); //[!] once more, now that *all* bytes were written assert(isReady(futUploadDone_)); futUploadDone_.get(); //throw FileError //asyncStreamOut_->checkReadErrors(); //throw FileError -> not needed after *successful* upload asyncStreamOut_.reset(); //output finalized => no more exceptions from here on! //-------------------------------------------------------------------- AFS::FinalizeResult result; //result.filePrint = ... -> yet unknown at this point try { setModTimeIfAvailable(); //throw FileError, follows symlinks /* is setting modtime after closing the file handle a pessimization? FTP: no: could set modtime via CURLOPT_POSTQUOTE (but this would internally trigger an extra round-trip anyway!) */ } catch (const FileError& e) { result.errorModTime = e; /*might slice derived class?*/ } return result; } private: void reportBytesProcessed(const IoCallback& notifyUnbufferedIO /*throw X*/) //throw X { const int64_t bytesDelta = makeSigned(asyncStreamOut_->getTotalBytesRead()) - totalBytesReported_; totalBytesReported_ += bytesDelta; if (notifyUnbufferedIO) notifyUnbufferedIO(bytesDelta); //throw X } void setModTimeIfAvailable() const //throw FileError, follows symlinks { //assert(isReady(futUploadDone_)); => MUST NOT CALL *after* std::future<>::get()! if (modTime_) try { const std::string isoTime = utfTo(formatTime(Zstr("%Y%m%d%H%M%S"), getUtcTime(*modTime_))); //returns empty string on error if (isoTime.empty()) throw SysError(L"Invalid modification time (time_t: " + numberTo(*modTime_) + L')'); accessFtpSession(login_, [&](FtpSession& session) //throw SysError { if (!session.supportsMfmt()) //throw SysError throw SysError(L"Server does not support the MFMT command."); session.runSingleFtpCommand("MFMT " + isoTime + ' ' + session.getServerPathInternal(filePath_), true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol //not relevant for OutputStreamFtp, but: does MFMT follow symlinks? for Linux FTP server (using utime) it does }); } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot write modification time of %x."), L"%x", fmtPath(getCurlDisplayPath(login_, filePath_))), e.toString()); } } const FtpLogin login_; const AfsPath filePath_; const std::optional modTime_; int64_t totalBytesReported_ = 0; std::shared_ptr asyncStreamOut_ = std::make_shared(FTP_STREAM_BUFFER_SIZE); InterruptibleThread worker_; std::future futUploadDone_; }; //--------------------------------------------------------------------------------------------------------------------------- //=========================================================================================================================== class FtpFileSystem : public AbstractFileSystem { public: explicit FtpFileSystem(const FtpLogin& login) : login_(login) {} const FtpLogin& getLogin() const { return login_; } private: Zstring getInitPathPhrase(const AfsPath& itemPath) const override { return concatenateFtpFolderPathPhrase(login_, itemPath); } std::vector getPathPhraseAliases(const AfsPath& itemPath) const override { return {getInitPathPhrase(itemPath)}; } std::wstring getDisplayPath(const AfsPath& itemPath) const override { return getCurlDisplayPath(login_, itemPath); } bool isNullFileSystem() const override { return login_.server.empty(); } std::weak_ordering compareDeviceSameAfsType(const AbstractFileSystem& afsRhs) const override { const FtpLogin& lhs = login_; const FtpLogin& rhs = static_cast(afsRhs).login_; return FtpDeviceId(lhs) <=> FtpDeviceId(rhs); } //---------------------------------------------------------------------------------------------------------------- ItemType getItemType(const AfsPath& itemPath) const override //throw FileError { try { const std::optional parentPath = getParentPath(itemPath); if (!parentPath) //device root => quick access test { try { accessFtpSession(login_, [](FtpSession& session) { session.testConnection(); }); /*throw SysError*/ } catch (const SysError& e) { throw SysError(replaceCpy(_("Unable to connect to %x."), L"%x", fmtPath(login_.server)) + L'\n' + e.toString()); } return ItemType::folder; } const std::vector items = [&] { try { //don't use MLST: broken for Pure-FTPd: https://freefilesync.org/forum/viewtopic.php?t=4287 return FtpDirectoryReader::execute(login_, *parentPath); //throw SysError, SysErrorFtpProtocol } catch (const SysError& e) //add context: error might be folder-specific { throw SysError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtPath(parentPath->value.empty() ? Zstr("/") : getItemName(*parentPath))) + L'\n' + e.toString()); } }(); const Zstring itemName = getItemName(itemPath); assert(!itemName.empty()); //is the underlying file system case-sensitive? we don't know => assume "case-sensitive" //all path components (except the base folder part!) can be expected to have the right case anyway after directory traversal for (const FtpItem& item : items) if (item.itemName == itemName) return item.type; throw SysError(replaceCpy(_("%x does not exist."), L"%x", fmtPath(itemName))); } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getDisplayPath(itemPath))), e.toString()); } } std::optional getItemTypeIfExistsImpl(const AfsPath& itemPath) const //throw SysError { const std::optional parentPath = getParentPath(itemPath); if (!parentPath) //device root => quick access test { try { accessFtpSession(login_, [](FtpSession& session) { session.testConnection(); }); /*throw SysError*/ } catch (const SysError& e) { throw SysError(replaceCpy(_("Unable to connect to %x."), L"%x", fmtPath(login_.server)) + L'\n' + e.toString()); } return ItemType::folder; } std::optional lastFtpError; try { try { const Zstring itemName = getItemName(itemPath); assert(!itemName.empty()); for (const FtpItem& item : FtpDirectoryReader::execute(login_, *parentPath)) //throw SysError, SysErrorFtpProtocol if (item.itemName == itemName) //case-sensitive comparison! itemPath must be normalized! return item.type; return std::nullopt; } catch (const SysErrorFtpProtocol& e) { //let's dig deeper, but *only* for SysErrorFtpProtocol, not for general connection issues //+ check if FTP error code sounds like "not existing" if (e.ftpErrorCode == 550) //FTP 550 No such file or directory //501? "pathname that exists but is not a directory to a MLSD command generates a 501 reply": https://www.rfc-editor.org/rfc/rfc3659 //=> really? cannot reproduce, getting: "550 '/filename.txt' is not a directory" or "550 Can't check for file existence" lastFtpError = e; //-> get out of catch clause else throw; } } catch (const SysError& e) //add context: error might be folder-specific { throw SysError(replaceCpy(_("Cannot read directory %x."), L"%x", fmtPath(parentPath->value.empty() ? Zstr("/") : getItemName(*parentPath))) + L'\n' + e.toString()); } //---------------------------------------------------------------- if (const std::optional parentType = getItemTypeIfExistsImpl(*parentPath)) //throw SysError { if (*parentType == ItemType::file /*obscure, but possible*/) throw SysError(replaceCpy(_("The name %x is already used by another item."), L"%x", fmtPath(getItemName(*parentPath)))); throw* lastFtpError; //throw SysError; parent path existing, so traversal should not have failed! } else return std::nullopt; } std::optional getItemTypeIfExists(const AfsPath& itemPath) const override //throw FileError { try { return getItemTypeIfExistsImpl(itemPath); //throw SysError } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot read file attributes of %x."), L"%x", fmtPath(getDisplayPath(itemPath))), e.toString()); } } //---------------------------------------------------------------------------------------------------------------- //already existing: fail //=> FTP will (most likely) fail and give a clear error message: // freefilesync.org: "550 Can't create directory: File exists" // FileZilla Server: "550 Directory already exists" // Windows IIS: "550 Cannot create a file when that file already exists" void createFolderPlain(const AfsPath& folderPath) const override //throw FileError { try { accessFtpSession(login_, [&](FtpSession& session) //throw SysError { session.runSingleFtpCommand("MKD " + session.getServerPathInternal(folderPath), true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol }); } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot create directory %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } void removeFilePlain(const AfsPath& filePath) const override //throw FileError { try { accessFtpSession(login_, [&](FtpSession& session) //throw SysError { session.runSingleFtpCommand("DELE " + session.getServerPathInternal(filePath), true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol }); } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete file %x."), L"%x", fmtPath(getDisplayPath(filePath))), e.toString()); } } void removeSymlinkPlain(const AfsPath& linkPath) const override //throw FileError { try { accessFtpSession(login_, [&](FtpSession& session) //throw SysError { //works fine for Linux hosts, but what about Windows-hosted FTP??? Distinguish DELE/RMD? //Windows test, FileZilla Server and Windows IIS FTP: all symlinks are reported as regular folders session.runSingleFtpCommand("DELE " + session.getServerPathInternal(linkPath), true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol }); } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete symbolic link %x."), L"%x", fmtPath(getDisplayPath(linkPath))), e.toString()); } } void removeFolderPlain(const AfsPath& folderPath) const override //throw FileError { try { accessFtpSession(login_, [&](FtpSession& session) //throw SysError { //Windows server: FileZilla Server and Windows IIS FTP: all symlinks are reported as regular folders //Linux server (freefilesync.org): RMD will fail for symlinks! session.runSingleFtpCommand("RMD " + session.getServerPathInternal(folderPath), true /*requestUtf8*/); //throw SysError, SysErrorFtpProtocol }); } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot delete directory %x."), L"%x", fmtPath(getDisplayPath(folderPath))), e.toString()); } } void removeFolderIfExistsRecursion(const AfsPath& folderPath, //throw FileError const std::function& onBeforeFileDeletion /*throw X*/, const std::function& onBeforeSymlinkDeletion/*throw X*/, const std::function& onBeforeFolderDeletion /*throw X*/) const override { //default implementation: folder traversal AFS::removeFolderIfExistsRecursion(folderPath, onBeforeFileDeletion, onBeforeSymlinkDeletion, onBeforeFolderDeletion); //throw FileError, X } //---------------------------------------------------------------------------------------------------------------- AbstractPath getSymlinkResolvedPath(const AfsPath& linkPath) const override //throw FileError { throw FileError(replaceCpy(_("Cannot determine final path for %x."), L"%x", fmtPath(getDisplayPath(linkPath))), _("Operation not supported by device.")); } bool equalSymlinkContentForSameAfsType(const AfsPath& linkPathL, const AbstractPath& linkPathR) const override //throw FileError { throw FileError(replaceCpy(_("Cannot resolve symbolic link %x."), L"%x", fmtPath(getDisplayPath(linkPathL))), _("Operation not supported by device.")); } //---------------------------------------------------------------------------------------------------------------- //return value always bound: std::unique_ptr getInputStream(const AfsPath& filePath) const override //throw FileError, (ErrorFileLocked) { return std::make_unique(login_, filePath); } //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) //=> actual behavior: fail(+delete!)/overwrite/auto-rename std::unique_ptr getOutputStream(const AfsPath& filePath, //throw FileError std::optional streamSize, std::optional modTime) const override { /* most FTP servers overwrite, but some (e.g. IIS) can be configured to fail, others (pureFTP) can be configured to auto-rename: https://download.pureftpd.org/pub/pure-ftpd/doc/README '-r': Never overwrite existing files. Uploading a file whose name already exists causes an automatic rename. Files are called xyz, xyz.1, xyz.2, xyz.3, etc. */ //already existing: fail (+ delete!!!) return std::make_unique(login_, filePath, modTime); } //---------------------------------------------------------------------------------------------------------------- void traverseFolderRecursive(const TraverserWorkload& workload /*throw X*/, size_t parallelOps) const override { traverseFolderRecursiveFTP(login_, workload, parallelOps); //throw X } //---------------------------------------------------------------------------------------------------------------- //symlink handling: follow //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) FileCopyResult copyFileForSameAfsType(const AfsPath& sourcePath, const StreamAttributes& attrSource, //throw FileError, (ErrorFileLocked), X const AbstractPath& targetPath, bool copyFilePermissions, const IoCallback& notifyUnbufferedIO /*throw X*/) const override { //no native FTP file copy => use stream-based file copy: if (copyFilePermissions) throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); //already existing: undefined behavior! (e.g. fail/overwrite/auto-rename) return copyFileAsStream(sourcePath, attrSource, targetPath, notifyUnbufferedIO); //throw FileError, (ErrorFileLocked), X } //symlink handling: follow //already existing: fail void copyNewFolderForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override //throw FileError { //already existing: fail AFS::createFolderPlain(targetPath); //throw FileError if (copyFilePermissions) throw FileError(replaceCpy(_("Cannot write permissions of %x."), L"%x", fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); } //already existing: fail void copySymlinkForSameAfsType(const AfsPath& sourcePath, const AbstractPath& targetPath, bool copyFilePermissions) const override { throw FileError(replaceCpy(replaceCpy(_("Cannot copy symbolic link %x to %y."), L"%x", L'\n' + fmtPath(getDisplayPath(sourcePath))), L"%y", L'\n' + fmtPath(AFS::getDisplayPath(targetPath))), _("Operation not supported by device.")); } //already existing: undefined behavior! (e.g. fail/overwrite) //=> actual behavior: most linux-based FTP servers overwrite, Windows-based servers fail (but most can be configured to behave differently) // freefilesync.org: silent overwrite // Windows IIS: CURLE_QUOTE_ERROR: QUOT command failed with 550 Cannot create a file when that file already exists. // FileZilla Server: CURLE_QUOTE_ERROR: QUOT command failed with 553 file exists void moveAndRenameItemForSameAfsType(const AfsPath& pathFrom, const AbstractPath& pathTo) const override //throw FileError, ErrorMoveUnsupported { if (compareDeviceSameAfsType(pathTo.afsDevice.ref()) != std::weak_ordering::equivalent) throw ErrorMoveUnsupported(generateMoveErrorMsg(pathFrom, pathTo), _("Operation not supported between different devices.")); try { accessFtpSession(login_, [&](FtpSession& session) //throw SysError { curl_slist* quote = nullptr; ZEN_ON_SCOPE_EXIT(::curl_slist_free_all(quote)); quote = ::curl_slist_append(quote, ("RNFR " + session.getServerPathInternal(pathFrom )).c_str()); //throw SysError quote = ::curl_slist_append(quote, ("RNTO " + session.getServerPathInternal(pathTo.afsPath)).c_str()); // session.perform(AfsPath(), true /*isDir*/, CURLFTPMETHOD_NOCWD, //avoid needless CWDs { {CURLOPT_NOBODY, 1L}, {CURLOPT_QUOTE, quote}, }, true /*requestUtf8*/); //throw SysError, SysErrorPassword, SysErrorFtpProtocol }); } catch (const SysError& e) { throw FileError(generateMoveErrorMsg(pathFrom, pathTo), e.toString()); } } bool supportsPermissions(const AfsPath& folderPath) const override { return false; } //throw FileError //wait until there is real demand for copying from and to FTP with permissions => use stream-based file copy: //---------------------------------------------------------------------------------------------------------------- FileIconHolder getFileIcon (const AfsPath& filePath, int pixelSize) const override { return {}; } //throw FileError; optional return value ImageHolder getThumbnailImage(const AfsPath& filePath, int pixelSize) const override { return {}; } //throw FileError; optional return value void authenticateAccess(const RequestPasswordFun& requestPassword /*throw X*/) const override //throw FileError, X { auto connectServer = [&] //throw SysError, SysErrorPassword { accessFtpSession(login_, [](FtpSession& session) //connect with FTP server, *unless* already connected (in which case *nothing* is sent) { session.perform(AfsPath(), true /*isDir*/, CURLFTPMETHOD_NOCWD, {{CURLOPT_NOBODY, 1L}, {CURLOPT_SERVER_RESPONSE_TIMEOUT, 0}}, false /*requestUtf8*/); //caveat: connection phase only, so disable CURLOPT_SERVER_RESPONSE_TIMEOUT, or next access may fail with CURLE_OPERATION_TIMEDOUT! }); //throw SysError, SysErrorPassword, SysErrorFtpProtocol }; try { const std::shared_ptr mgr = globalFtpSessionManager.get(); if (!mgr) throw SysError(formatSystemError("getSessionPassword", L"", L"Function call not allowed during init/shutdown.")); mgr->setActiveConfig(login_); if (!login_.password) { try //1. test for connection error *before* bothering user to enter a password { connectServer(); //throw SysError, SysErrorPassword return; //got new FtpSession (connected in constructor) or already connected session from cache } catch (const SysErrorPassword& e) { if (!requestPassword) throw SysError(e.toString() + L'\n' + _("Password prompt not permitted by current settings.")); } std::wstring lastErrorMsg; for (;;) { //2. request (new) password std::wstring msg = replaceCpy(_("Please enter your password to connect to %x."), L"%x", fmtPath(getDisplayPath(AfsPath()))); if (lastErrorMsg.empty()) msg += L"\n" + _("The password will only be remembered until FreeFileSync is closed."); const Zstring password = requestPassword(msg, lastErrorMsg); //throw X mgr->setSessionPassword(login_, password); try //3. test access: { connectServer(); //throw SysError, SysErrorPassword return; } catch (const SysErrorPassword& e) { lastErrorMsg = e.toString(); } } } } catch (const SysError& e) { throw FileError(replaceCpy(_("Unable to connect to %x."), L"%x", fmtPath(getDisplayPath(AfsPath()))), e.toString()); } } bool hasNativeTransactionalCopy() const override { return false; } //---------------------------------------------------------------------------------------------------------------- int64_t getFreeDiskSpace(const AfsPath& folderPath) const override { return -1; } //throw FileError, returns < 0 if not available std::unique_ptr createRecyclerSession(const AfsPath& folderPath) const override //throw FileError, RecycleBinUnavailable { throw RecycleBinUnavailable(replaceCpy(_("The recycle bin is not available for %x."), L"%x", fmtPath(getDisplayPath(folderPath)))); } void moveToRecycleBin(const AfsPath& itemPath) const override //throw FileError, RecycleBinUnavailable { throw RecycleBinUnavailable(replaceCpy(_("The recycle bin is not available for %x."), L"%x", fmtPath(getDisplayPath(itemPath)))); } const FtpLogin login_; }; //=========================================================================================================================== //expects "clean" login data Zstring concatenateFtpFolderPathPhrase(const FtpLogin& login, const AfsPath& folderPath) //noexcept { Zstring username; if (!login.username.empty()) username = encodeFtpUsername(login.username) + Zstr("@"); Zstring server = login.server; if (parseIpv6Address(server) && login.portCfg > 0) server = Zstr('[') + server + Zstr(']'); //e.g. [::1]:80 Zstring port; if (login.portCfg > 0) port = Zstr(':') + numberTo(login.portCfg); Zstring relPath = getServerRelPath(folderPath); if (relPath == Zstr("/")) relPath.clear(); Zstring options; if (login.timeoutSec != FtpLogin().timeoutSec) options += Zstr("|timeout=") + numberTo(login.timeoutSec); if (login.useTls) options += Zstr("|ssl"); if (login.password) { if (!login.password->empty()) //password always last => visually truncated by folder input field options += Zstr("|pass64=") + encodePasswordBase64(*login.password); } else options += Zstr("|pwprompt"); return Zstring(ftpPrefix) + Zstr("//") + username + server + port + relPath + options; } } void fff::ftpInit() { assert(!globalFtpSessionManager.get()); globalFtpSessionManager.set(std::make_unique()); } void fff::ftpTeardown() { assert(globalFtpSessionManager.get()); globalFtpSessionManager.set(nullptr); } AfsPath fff::getFtpHomePath(const FtpLogin& login) //throw FileError { try { AfsPath homePath; accessFtpSession(login, [&](FtpSession& session) //throw SysError { homePath = session.getHomePath(); //throw SysError }); return homePath; } catch (const SysError& e) { throw FileError(replaceCpy(_("Cannot determine final path for %x."), L"%x", fmtPath(getCurlDisplayPath(login, AfsPath(Zstr("~"))))), e.toString()); } } AfsDevice fff::condenseToFtpDevice(const FtpLogin& login) //noexcept { //clean up input: FtpLogin loginTmp = login; trim(loginTmp.server); trim(loginTmp.username); loginTmp.timeoutSec = std::max(1, loginTmp.timeoutSec); if (startsWithAsciiNoCase(loginTmp.server, "http:" ) || startsWithAsciiNoCase(loginTmp.server, "https:") || startsWithAsciiNoCase(loginTmp.server, "ftp:" ) || startsWithAsciiNoCase(loginTmp.server, "ftps:" ) || startsWithAsciiNoCase(loginTmp.server, "sftp:" )) loginTmp.server = afterFirst(loginTmp.server, Zstr(':'), IfNotFoundReturn::none); trim(loginTmp.server, TrimSide::both, [](Zchar c) { return c == Zstr('/') || c == Zstr('\\'); }); if (std::optional> ip6AndPort = parseIpv6Address(loginTmp.server)) loginTmp.server = ip6AndPort->first; //remove leading/trailing brackets return makeSharedRef(loginTmp); } FtpLogin fff::extractFtpLogin(const AfsDevice& afsDevice) //noexcept { if (const auto ftpDevice = dynamic_cast(&afsDevice.ref())) return ftpDevice->getLogin(); assert(false); return {}; } bool fff::acceptsItemPathPhraseFtp(const Zstring& itemPathPhrase) //noexcept { Zstring path = expandMacros(itemPathPhrase); //expand before trimming! trim(path); return startsWithAsciiNoCase(path, ftpPrefix); //check for explicit FTP path } /* syntax: ftp://[[:]@][:port]/[|option_name=value] e.g. ftp://user001:secretpassword@private.example.com:222/mydirectory/ ftp://user001:secretpassword@[::1]:80/ipv6folder/ ftp://user001:secretpassword@::1/ipv6withoutPort/ ftp://user001@private.example.com/mydirectory|pass64=c2VjcmV0cGFzc3dvcmQ */ AbstractPath fff::createItemPathFtp(const Zstring& itemPathPhrase) //noexcept { Zstring pathPhrase = expandMacros(itemPathPhrase); //expand before trimming! trim(pathPhrase); if (startsWithAsciiNoCase(pathPhrase, ftpPrefix)) pathPhrase = pathPhrase.c_str() + strLength(ftpPrefix); trim(pathPhrase, TrimSide::left, [](Zchar c) { return c == Zstr('/') || c == Zstr('\\'); }); const ZstringView credentials = beforeFirst(pathPhrase, Zstr('@'), IfNotFoundReturn::none); const ZstringView fullPathOpt = afterFirst(pathPhrase, Zstr('@'), IfNotFoundReturn::all); FtpLogin login; login.username = decodeFtpUsername(Zstring(beforeFirst(credentials, Zstr(':'), IfNotFoundReturn::all))); //support standard FTP syntax, even though login.password = Zstring( afterFirst(credentials, Zstr(':'), IfNotFoundReturn::none)); //concatenateFtpFolderPathPhrase() uses "pass64" instead const ZstringView fullPath = beforeFirst(fullPathOpt, Zstr('|'), IfNotFoundReturn::all); const ZstringView options = afterFirst(fullPathOpt, Zstr('|'), IfNotFoundReturn::none); auto it = std::find_if(fullPath.begin(), fullPath.end(), [](Zchar c) { return c == '/' || c == '\\'; }); const ZstringView serverPort = makeStringView(fullPath.begin(), it); const AfsPath serverRelPath = sanitizeDeviceRelativePath({it, fullPath.end()}); if (std::optional> ip6AndPort = parseIpv6Address(serverPort)) //e.g. 2001:db8::ff00:42:8329 or [::1]:80 { login.server = ip6AndPort->first; login.portCfg = ip6AndPort->second; //0 if empty } else { login.server = Zstring(beforeLast(serverPort, Zstr(':'), IfNotFoundReturn::all)); const ZstringView port = afterLast(serverPort, Zstr(':'), IfNotFoundReturn::none); login.portCfg = stringTo(port); //0 if empty } split(options, Zstr('|'), [&](ZstringView optPhrase) { optPhrase = trimCpy(optPhrase); if (!optPhrase.empty()) { if (startsWith(optPhrase, Zstr("timeout="))) login.timeoutSec = stringTo(afterFirst(optPhrase, Zstr('='), IfNotFoundReturn::none)); else if (optPhrase == Zstr("ssl")) login.useTls = true; else if (startsWith(optPhrase, Zstr("pass64="))) login.password = decodePasswordBase64(afterFirst(optPhrase, Zstr('='), IfNotFoundReturn::none)); else if (optPhrase == Zstr("pwprompt")) login.password = std::nullopt; else assert(false); } }); return AbstractPath(makeSharedRef(login), serverRelPath); }