From c83b5fdd05c2750f9d10ae68f436b4d0ad3879aa Mon Sep 17 00:00:00 2001 From: Stenzek Date: Sun, 28 Jul 2024 21:58:59 +1000 Subject: [PATCH] FileSystem: Add AtomicRenamedFile --- src/common/file_system.cpp | 224 ++++++++++++++++++++++++++++++++++++- src/common/file_system.h | 27 +++++ 2 files changed, 250 insertions(+), 1 deletion(-) diff --git a/src/common/file_system.cpp b/src/common/file_system.cpp index a1bbb6008..f77ebaac4 100644 --- a/src/common/file_system.cpp +++ b/src/common/file_system.cpp @@ -7,6 +7,7 @@ #include "log.h" #include "path.h" #include "string_util.h" +#include "timer.h" #include #include @@ -1013,6 +1014,97 @@ std::FILE* FileSystem::OpenCFile(const char* filename, const char* mode, Error* #endif } +std::FILE* FileSystem::OpenExistingOrCreateCFile(const char* filename, s32 retry_ms, Error* error /*= nullptr*/) +{ +#ifdef _WIN32 + const std::wstring wfilename = GetWin32Path(filename); + if (wfilename.empty()) + { + Error::SetStringView(error, "Invalid path."); + return nullptr; + } + + HANDLE file = CreateFileW(wfilename.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, OPEN_EXISTING, 0, NULL); + + // if there's a sharing violation, keep retrying + if (file == INVALID_HANDLE_VALUE && GetLastError() == ERROR_SHARING_VIOLATION && retry_ms >= 0) + { + Common::Timer timer; + while (retry_ms == 0 || timer.GetTimeMilliseconds() <= retry_ms) + { + Sleep(1); + file = CreateFileW(wfilename.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, OPEN_EXISTING, 0, NULL); + if (file != INVALID_HANDLE_VALUE || GetLastError() != ERROR_SHARING_VIOLATION) + break; + } + } + + if (file == INVALID_HANDLE_VALUE && GetLastError() == ERROR_FILE_NOT_FOUND) + { + // try creating it + file = CreateFileW(wfilename.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, CREATE_NEW, 0, NULL); + if (file == INVALID_HANDLE_VALUE && GetLastError() == ERROR_FILE_EXISTS) + { + // someone else beat us in the race, try again with existing. + file = CreateFileW(wfilename.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, OPEN_EXISTING, 0, NULL); + } + } + + // done? + if (file == INVALID_HANDLE_VALUE) + { + Error::SetWin32(error, "CreateFile() failed: ", GetLastError()); + return nullptr; + } + + // convert to C FILE + const int fd = _open_osfhandle(reinterpret_cast(file), 0); + if (fd < 0) + { + Error::SetErrno(error, "_open_osfhandle() failed: ", errno); + CloseHandle(file); + return nullptr; + } + + // convert to a stream + std::FILE* cfile = _fdopen(fd, "r+b"); + if (!cfile) + { + Error::SetErrno(error, "_fdopen() failed: ", errno); + _close(fd); + } + + return cfile; +#else + std::FILE* fp = std::fopen(filename, "r+b"); + if (fp) + return fp; + + // don't try creating for any error other than "not exist" + if (errno != ENOENT) + { + Error::SetErrno(error, errno); + return nullptr; + } + + // try again, but create the file. mode "x" exists on all platforms. + fp = std::fopen(filename, "w+bx"); + if (fp) + return fp; + + // if it already exists, someone else beat us in the race. try again with existing. + if (errno == EEXIST) + fp = std::fopen(filename, "r+b"); + if (!fp) + { + Error::SetErrno(error, errno); + return nullptr; + } + + return fp; +#endif +} + int FileSystem::OpenFDFile(const char* filename, int flags, int mode, Error* error) { #ifdef _WIN32 @@ -1069,6 +1161,95 @@ std::FILE* FileSystem::OpenSharedCFile(const char* filename, const char* mode, F #endif } +FileSystem::AtomicRenamedFileDeleter::AtomicRenamedFileDeleter(std::string temp_filename, std::string final_filename) + : m_temp_filename(std::move(temp_filename)), m_final_filename(std::move(final_filename)) +{ +} + +FileSystem::AtomicRenamedFileDeleter::~AtomicRenamedFileDeleter() = default; + +void FileSystem::AtomicRenamedFileDeleter::operator()(std::FILE* fp) +{ + if (!fp) + return; + + Error error; + if (std::fclose(fp) != 0) + { + error.SetErrno(errno); + ERROR_LOG("Failed to close temporary file '{}', discarding.", Path::GetFileName(m_temp_filename)); + m_final_filename.clear(); + } + + // final filename empty => discarded. + if (m_final_filename.empty()) + { + if (!DeleteFile(m_temp_filename.c_str(), &error)) + ERROR_LOG("Failed to delete temporary file '{}': {}", Path::GetFileName(m_temp_filename), error.GetDescription()); + } + else + { + if (!RenamePath(m_temp_filename.c_str(), m_final_filename.c_str(), &error)) + ERROR_LOG("Failed to rename temporary file '{}': {}", Path::GetFileName(m_temp_filename), error.GetDescription()); + } +} + +void FileSystem::AtomicRenamedFileDeleter::discard() +{ + m_final_filename = {}; +} + +FileSystem::AtomicRenamedFile FileSystem::CreateAtomicRenamedFile(std::string filename, const char* mode, + Error* error /*= nullptr*/) +{ + std::string temp_filename; + std::FILE* fp = nullptr; + if (!filename.empty()) + { + // this is disgusting, but we need null termination, and std::string::data() does not guarantee it. + const size_t filename_length = filename.length(); + const size_t name_buf_size = filename_length + 8; + std::unique_ptr name_buf = std::make_unique(name_buf_size); + std::memcpy(name_buf.get(), filename.c_str(), filename_length); + StringUtil::Strlcpy(name_buf.get() + filename_length, ".XXXXXX", name_buf_size); + +#ifdef _WIN32 + _mktemp_s(name_buf.get(), name_buf_size); +#elif defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__) + mkstemp(name_buf.get()); +#else + mktemp(name_buf.get()); +#endif + + fp = OpenCFile(name_buf.get(), mode, error); + if (fp) + temp_filename.assign(name_buf.get(), name_buf_size - 1); + else + filename.clear(); + } + + return AtomicRenamedFile(fp, AtomicRenamedFileDeleter(std::move(temp_filename), std::move(filename))); +} + +bool FileSystem::WriteAtomicRenamedFile(std::string filename, const void* data, size_t data_length, + Error* error /*= nullptr*/) +{ + AtomicRenamedFile fp = CreateAtomicRenamedFile(std::move(filename), "wb", error); + if (data_length > 0 && std::fwrite(data, 1u, data_length, fp.get()) != data_length) [[unlikely]] + { + Error::SetErrno(error, "fwrite() failed: ", errno); + DiscardAtomicRenamedFile(fp); + return false; + } + + return true; +} + +void FileSystem::DiscardAtomicRenamedFile(AtomicRenamedFile& file) +{ + file.get_deleter().discard(); +} + #endif FileSystem::ManagedCFilePtr FileSystem::OpenManagedCFile(const char* filename, const char* mode, Error* error) @@ -1076,6 +1257,12 @@ FileSystem::ManagedCFilePtr FileSystem::OpenManagedCFile(const char* filename, c return ManagedCFilePtr(OpenCFile(filename, mode, error)); } +FileSystem::ManagedCFilePtr FileSystem::OpenExistingOrCreateManagedCFile(const char* filename, s32 retry_ms, + Error* error) +{ + return ManagedCFilePtr(OpenExistingOrCreateCFile(filename, retry_ms, error)); +} + FileSystem::ManagedCFilePtr FileSystem::OpenManagedSharedCFile(const char* filename, const char* mode, FileShareMode share_mode, Error* error) { @@ -1098,6 +1285,31 @@ int FileSystem::FSeek64(std::FILE* fp, s64 offset, int whence) #endif } +bool FileSystem::FSeek64(std::FILE* fp, s64 offset, int whence, Error* error) +{ +#ifdef _WIN32 + const int res = _fseeki64(fp, offset, whence); +#else + // Prevent truncation on platforms which don't have a 64-bit off_t. + if constexpr (sizeof(off_t) != sizeof(s64)) + { + if (offset < std::numeric_limits::min() || offset > std::numeric_limits::max()) + { + Error::SetStringView(error, "Invalid offset."); + return false; + } + } + + const int res = fseeko(fp, static_cast(offset), whence); +#endif + + if (res == 0) + return true; + + Error::SetErrno(error, errno); + return false; +} + s64 FileSystem::FTell64(std::FILE* fp) { #ifdef _WIN32 @@ -2479,7 +2691,17 @@ static bool SetLock(int fd, bool lock) return false; } - const bool res = (lockf(fd, lock ? F_LOCK : F_ULOCK, 0) == 0); + // bloody signals... + bool res; + for (;;) + { + res = (lockf(fd, lock ? F_LOCK : F_ULOCK, 0) == 0); + if (!res && errno == EINTR) + continue; + else + break; + } + if (lseek(fd, offs, SEEK_SET) < 0) Panic("Repositioning file descriptor after lock failed."); diff --git a/src/common/file_system.h b/src/common/file_system.h index ba80686e6..0a4ebb16d 100644 --- a/src/common/file_system.h +++ b/src/common/file_system.h @@ -108,7 +108,15 @@ struct FileDeleter using ManagedCFilePtr = std::unique_ptr; ManagedCFilePtr OpenManagedCFile(const char* filename, const char* mode, Error* error = nullptr); std::FILE* OpenCFile(const char* filename, const char* mode, Error* error = nullptr); + +/// Atomically opens a file in read/write mode, and if the file does not exist, creates it. +/// On Windows, if retry_ms is positive, this function will retry opening the file for this +/// number of milliseconds. NOTE: The file is opened in binary mode. +std::FILE* OpenExistingOrCreateCFile(const char* filename, s32 retry_ms = -1, Error* error = nullptr); +ManagedCFilePtr OpenExistingOrCreateManagedCFile(const char* filename, s32 retry_ms = -1, Error* error = nullptr); + int FSeek64(std::FILE* fp, s64 offset, int whence); +bool FSeek64(std::FILE* fp, s64 offset, int whence, Error* error); s64 FTell64(std::FILE* fp); s64 FSize64(std::FILE* fp, Error* error = nullptr); bool FTruncate64(std::FILE* fp, s64 size, Error* error = nullptr); @@ -130,6 +138,25 @@ ManagedCFilePtr OpenManagedSharedCFile(const char* filename, const char* mode, F Error* error = nullptr); std::FILE* OpenSharedCFile(const char* filename, const char* mode, FileShareMode share_mode, Error* error = nullptr); +/// Atomically-updated file creation. +class AtomicRenamedFileDeleter +{ +public: + AtomicRenamedFileDeleter(std::string temp_filename, std::string final_filename); + ~AtomicRenamedFileDeleter(); + + void operator()(std::FILE* fp); + void discard(); + +private: + std::string m_temp_filename; + std::string m_final_filename; +}; +using AtomicRenamedFile = std::unique_ptr; +AtomicRenamedFile CreateAtomicRenamedFile(std::string filename, const char* mode, Error* error = nullptr); +bool WriteAtomicRenamedFile(std::string filename, const void* data, size_t data_length, Error* error = nullptr); +void DiscardAtomicRenamedFile(AtomicRenamedFile& file); + /// Abstracts a POSIX file lock. #ifndef _WIN32 class POSIXLock