From af1625f676e21d2777aa4be70893bf01dc08328d Mon Sep 17 00:00:00 2001 From: Juan Jose Santamaria Flecha Date: Wed, 4 Sep 2019 17:17:37 -0400 Subject: [PATCH] WIP support for large files on Win32 . Workaround for Windows shortcomings, reimplement fstat()/stat() on top of GetFileInformationByHandle() and use 64 bits struct stat. --- configure | 6 + configure.in | 1 + src/include/port/win32_port.h | 40 +++++-- src/port/dirmod.c | 52 --------- src/port/win32_stat.c | 258 ++++++++++++++++++++++++++++++++++++++++++ src/tools/msvc/Mkvcbuild.pm | 2 +- 6 files changed, 294 insertions(+), 65 deletions(-) create mode 100644 src/port/win32_stat.c diff --git a/configure b/configure index f14709e..81271e5 100755 --- a/configure +++ b/configure @@ -16102,6 +16102,12 @@ fi esac case " $LIBOBJS " in + *" win32_stat.$ac_objext "* ) ;; + *) LIBOBJS="$LIBOBJS win32_stat.$ac_objext" + ;; +esac + + case " $LIBOBJS " in *" kill.$ac_objext "* ) ;; *) LIBOBJS="$LIBOBJS kill.$ac_objext" ;; diff --git a/configure.in b/configure.in index 805cf86..e260f2a 100644 --- a/configure.in +++ b/configure.in @@ -1770,6 +1770,7 @@ if test "$PORTNAME" = "win32"; then AC_CHECK_FUNCS(_configthreadlocale) AC_REPLACE_FUNCS(gettimeofday) AC_LIBOBJ(dirmod) + AC_LIBOBJ(win32_stat) AC_LIBOBJ(kill) AC_LIBOBJ(open) AC_LIBOBJ(system) diff --git a/src/include/port/win32_port.h b/src/include/port/win32_port.h index 1cf166a..2e317b7 100644 --- a/src/include/port/win32_port.h +++ b/src/include/port/win32_port.h @@ -52,7 +52,12 @@ #include #include /* for non-unicode version */ #undef near -#include /* needed before sys/stat hacking below */ +/* needed before sys/stat hacking below */ +#define fstat microsoft_native_fstat +#define stat microsoft_native_stat +#include +#undef fstat +#undef stat /* Must be here to avoid conflicting with prototype in windows.h */ #define mkdir(a,b) mkdir(a) @@ -249,20 +254,31 @@ typedef int pid_t; * Supplement to . * * We must pull in sys/stat.h before this part, else our overrides lose. - */ -#define lstat(path, sb) stat(path, sb) - -/* * stat() is not guaranteed to set the st_size field on win32, so we - * redefine it to our own implementation that is. + * redefine it to our own implementation. See src/port/win32_stat.c * - * Some frontends don't need the size from stat, so if UNSAFE_STAT_OK - * is defined we don't bother with this. + * The struct stat is 32 bit in MSVC so we redefine it as a copy of struct + * _stat64. This also fixes the struct size for MINGW builds. */ -#ifndef UNSAFE_STAT_OK -extern int pgwin32_safestat(const char *path, struct stat *buf); -#define stat(a,b) pgwin32_safestat(a,b) -#endif +struct stat /* It is actually _stat64 */ +{ + _dev_t st_dev; + _ino_t st_ino; + unsigned short st_mode; + short st_nlink; + short st_uid; + short st_gid; + _dev_t st_rdev; + __int64 st_size; + __time64_t st_atime; + __time64_t st_mtime; + __time64_t st_ctime; +}; +extern int _pgfstat64(int fileno, struct stat *buf); +extern int _pgstat64(char const *name, struct stat *buf); +#define fstat(fileno, sb) _pgfstat64(fileno, sb) +#define stat(path, sb) _pgstat64(path, sb) +#define lstat(path, sb) _pgstat64(path, sb) /* These macros are not provided by older MinGW, nor by MSVC */ #ifndef S_IRUSR diff --git a/src/port/dirmod.c b/src/port/dirmod.c index d793240..ec4b47f 100644 --- a/src/port/dirmod.c +++ b/src/port/dirmod.c @@ -353,55 +353,3 @@ pgwin32_is_junction(const char *path) return ((attr & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT); } #endif /* defined(WIN32) && !defined(__CYGWIN__) */ - - -#if defined(WIN32) && !defined(__CYGWIN__) - -#undef stat - -/* - * The stat() function in win32 is not guaranteed to update the st_size - * field when run. So we define our own version that uses the Win32 API - * to update this field. - */ -int -pgwin32_safestat(const char *path, struct stat *buf) -{ - int r; - WIN32_FILE_ATTRIBUTE_DATA attr; - - r = stat(path, buf); - if (r < 0) - { - if (GetLastError() == ERROR_DELETE_PENDING) - { - /* - * File has been deleted, but is not gone from the filesystem yet. - * This can happen when some process with FILE_SHARE_DELETE has it - * open and it will be fully removed once that handle is closed. - * Meanwhile, we can't open it, so indicate that the file just - * doesn't exist. - */ - errno = ENOENT; - return -1; - } - - return r; - } - - if (!GetFileAttributesEx(path, GetFileExInfoStandard, &attr)) - { - _dosmaperr(GetLastError()); - return -1; - } - - /* - * XXX no support for large files here, but we don't do that in general on - * Win32 yet. - */ - buf->st_size = attr.nFileSizeLow; - - return 0; -} - -#endif diff --git a/src/port/win32_stat.c b/src/port/win32_stat.c new file mode 100644 index 0000000..b2dcea4 --- /dev/null +++ b/src/port/win32_stat.c @@ -0,0 +1,258 @@ +/*------------------------------------------------------------------------- + * + * win32_stat.c + * Replacements for functions using GetFileInformationByHandle + * on Win32. + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/port/win32_stat.c + * + *------------------------------------------------------------------------- + */ + +#ifdef WIN32 + +#include "c.h" +#include +/* + * NtQueryInformationFile is used when the Windows version is older than + * Vista / Server 2008. We load it from the ntdll library. + */ +#if _WIN32_WINNT < 0x0600 +#include +#endif + +static const unsigned __int64 EpochShift = UINT64CONST(116444736000000000); + +/* + * Converts a FILETIME struct into a 64 bit time_t. + */ +static __time64_t +filetime_to_time(FILETIME const *ft) +{ + ULARGE_INTEGER unified_ft = { 0 }; + + unified_ft.LowPart = ft->dwLowDateTime; + unified_ft.HighPart = ft->dwHighDateTime; + + if (unified_ft.QuadPart < EpochShift) + return -1; + + unified_ft.QuadPart -= EpochShift; + unified_ft.QuadPart /= 10 * 1000 * 1000; + + return unified_ft.QuadPart; +} + +/* + * Converts WIN32 file attributes to unix mode. + * Only owner permissions are set. + */ +static unsigned short +fileattr_to_unixmode(int attr) +{ + unsigned short uxmode = 0; + + uxmode |= (unsigned short)((attr & FILE_ATTRIBUTE_DIRECTORY) ? + (_S_IFDIR) : (_S_IFREG)); + + uxmode |= (unsigned short)(attr & FILE_ATTRIBUTE_READONLY) ? + (_S_IREAD) : (_S_IREAD | _S_IWRITE); + + /* We could simulate _S_IEXEC with CMD's PATHEXT extensions, unnecessary */ + uxmode |= _S_IEXEC; + + return uxmode; +} + +/* + * Converts WIN32 file infomation to struct stat. + * GetFileInformationByHandle minimum supported version: Windows XP and + * Windows Server 2003. + */ +static int +fileinfo_to_stat(HANDLE hFile, struct stat *buf) +{ + BY_HANDLE_FILE_INFORMATION fiData; + + if (!GetFileInformationByHandle(hFile, &fiData)) + { + errno = ENOENT; + return -1; + } + + memset(buf, 0, sizeof(*buf)); + if (fiData.ftLastWriteTime.dwLowDateTime || fiData.ftLastWriteTime.dwHighDateTime) + buf->st_mtime = filetime_to_time(&fiData.ftLastWriteTime); + + if (fiData.ftLastAccessTime.dwLowDateTime || fiData.ftLastAccessTime.dwHighDateTime) + buf->st_atime = filetime_to_time(&fiData.ftLastAccessTime); + else + buf->st_atime = buf->st_mtime; + + if (fiData.ftCreationTime.dwLowDateTime || fiData.ftCreationTime.dwHighDateTime) + buf->st_ctime = filetime_to_time(&fiData.ftCreationTime); + else + buf->st_ctime = buf->st_mtime; + + buf->st_mode = fileattr_to_unixmode(fiData.dwFileAttributes); + buf->st_nlink = fiData.nNumberOfLinks; + + buf->st_size = ((__int64) fiData.nFileSizeHigh) << 32 | + (__int64)(fiData.nFileSizeLow); + return 0; +} + +#if _WIN32_WINNT < 0x0600 +#if !defined(__MINGW64_VERSION_MAJOR) +/* MinGW includes this in , but is missing in MSVC */ +typedef struct _FILE_STANDARD_INFORMATION +{ + LARGE_INTEGER AllocationSize; + LARGE_INTEGER EndOfFile; + ULONG NumberOfLinks; + BOOLEAN DeletePending; + BOOLEAN Directory; +} FILE_STANDARD_INFORMATION, *PFILE_STANDARD_INFORMATION; +#define FileStandardInformation 5 +#endif /* !defined(__MINGW64_VERSION_MAJOR) */ + +typedef NTSTATUS (NTAPI *PFN_NTQUERYINFORMATIONFILE) +( + IN HANDLE FileHandle, + OUT PIO_STATUS_BLOCK IoStatusBlock, + OUT PVOID FileInformation, + IN ULONG Length, + IN FILE_INFORMATION_CLASS FileInformationClass +); +static PFN_NTQUERYINFORMATIONFILE _NtQueryInformationFile = NULL; + +/* + * Load DLL file just once regardless of how many functions + * we load/call in it. + */ +static HMODULE ntdll = NULL; + +static void +LoadNtdll() +{ + if (ntdll != NULL) + return; + ntdll = LoadLibraryEx("ntdll.dll", NULL, 0); +} +#endif /* _WIN32_WINNT < 0x0600 */ + +/* + * We must use a handle so lstat() returns the information of the target file. + * As a reliable test for ERROR_DELETE_PENDING we use NtQueryInformationFile + * from Windows 2000 or GetFileInformationByHandleEx from Server 2008 / Vista + */ +int +_pgstat64(char const *name, struct stat *buf) +{ + SECURITY_ATTRIBUTES sa; + HANDLE hFile; + int ret; +#if _WIN32_WINNT < 0x0600 + IO_STATUS_BLOCK ioStatus; + FILE_STANDARD_INFORMATION standardInfo; +#else + FILE_STANDARD_INFO standardInfo; +#endif + + if (NULL == name || NULL == buf) + { + errno = EINVAL; + return -1; + } + + sa.nLength = sizeof(SECURITY_ATTRIBUTES); + sa.bInheritHandle = TRUE; + sa.lpSecurityDescriptor = NULL; + hFile = CreateFile(name, GENERIC_READ, (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE), + &sa, OPEN_EXISTING, + FILE_FLAG_NO_BUFFERING | FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, + NULL); + if (hFile == INVALID_HANDLE_VALUE) + { + CloseHandle(hFile); + errno = ENOENT; + return -1; + } + +#if _WIN32_WINNT < 0x0600 + if (_NtQueryInformationFile == NULL) + { + LoadNtdll(); + if(ntdll == NULL) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + /* Do not return ENOENT */ + errno = ENOMSG; + return -1; + } + + _NtQueryInformationFile = (PFN_NTQUERYINFORMATIONFILE) + GetProcAddress(ntdll, "NtQueryInformationFile"); + if (_NtQueryInformationFile == NULL) + { + _dosmaperr(GetLastError()); + CloseHandle(hFile); + errno = ENOMSG; + return -1; + } + } + + memset(&standardInfo, 0, sizeof(standardInfo)); + if (!NT_SUCCESS(_NtQueryInformationFile(hFile, &ioStatus, &standardInfo, sizeof(standardInfo), + FileStandardInformation))) +#else + if (!GetFileInformationByHandleEx(hFile, FileStandardInfo, &standardInfo, sizeof(standardInfo))) +#endif /* _WIN32_WINNT < 0x0600 */ + { + CloseHandle(hFile); + errno = ENOENT; + return -1; + } + if (standardInfo.DeletePending) + { + /* + * File has been deleted, but is not gone from the filesystem yet. + * This can happen when some process with FILE_SHARE_DELETE has it + * open and it will be fully removed once that handle is closed. + * Meanwhile, we can't open it, so indicate that the file just + * doesn't exist. + */ + DWORD error = ERROR_DELETE_PENDING; + _dosmaperr(error); + CloseHandle(hFile); + errno = ENOENT; + return -1; + } + + ret = fileinfo_to_stat(hFile, buf); + CloseHandle(hFile); + return ret; +} + +/* Shares _pgstat64() logic */ +int +_pgfstat64(int fileno, struct stat *buf) +{ + HANDLE hFile = (HANDLE)_get_osfhandle(fileno); + + if (INVALID_HANDLE_VALUE == hFile || NULL == buf) + { + errno = EINVAL; + return -1; + } + + return fileinfo_to_stat(hFile, buf); +} + +#endif /* WIN32 */ \ No newline at end of file diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 2eab635..b2bf321 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -101,7 +101,7 @@ sub mkvcbuild pg_strong_random.c pgcheckdir.c pgmkdirp.c pgsleep.c pgstrcasecmp.c pqsignal.c mkdtemp.c qsort.c qsort_arg.c quotes.c system.c sprompt.c strerror.c tar.c thread.c - win32env.c win32error.c win32security.c win32setlocale.c); + win32env.c win32error.c win32security.c win32setlocale.c win32_stat.c); push(@pgportfiles, 'rint.c') if ($vsVersion < '12.00'); -- 2.11.0