From eb16b62425f597a6f5ae82fa534651cd3c06f18d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 19:54:23 +0200 Subject: [PATCH 01/22] strbuf_readlink: don't call readlink twice if hint is the exact link size strbuf_readlink() calls readlink() twice if the hint argument specifies the exact size of the link target (e.g. by passing stat.st_size as returned by lstat()). This is necessary because 'readlink(..., hint) == hint' could mean that the buffer was too small. Use hint + 1 as buffer size to prevent this. Signed-off-by: Karsten Blees --- strbuf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/strbuf.c b/strbuf.c index e3397cc4c7..8dc283cd67 100644 --- a/strbuf.c +++ b/strbuf.c @@ -568,12 +568,12 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) while (hint < STRBUF_MAXLINK) { ssize_t len; - strbuf_grow(sb, hint); - len = readlink(path, sb->buf, hint); + strbuf_grow(sb, hint + 1); + len = readlink(path, sb->buf, hint + 1); if (len < 0) { if (errno != ERANGE) break; - } else if (len < hint) { + } else if (len <= hint) { strbuf_setlen(sb, len); return 0; } From b97606f4570fa17ed950f1f7b7e44f41b25776a0 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 22:15:40 +0200 Subject: [PATCH 02/22] strbuf_readlink: support link targets that exceed PATH_MAX strbuf_readlink() refuses to read link targets that exceed PATH_MAX (even if a sufficient size was specified by the caller). As some platforms support longer paths, remove this restriction (similar to strbuf_getcwd()). Signed-off-by: Karsten Blees --- strbuf.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/strbuf.c b/strbuf.c index 8dc283cd67..954461715d 100644 --- a/strbuf.c +++ b/strbuf.c @@ -556,8 +556,6 @@ ssize_t strbuf_write(struct strbuf *sb, FILE *f) return sb->len ? fwrite(sb->buf, 1, sb->len, f) : 0; } -#define STRBUF_MAXLINK (2*PATH_MAX) - int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) { size_t oldalloc = sb->alloc; @@ -565,7 +563,7 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) if (hint < 32) hint = 32; - while (hint < STRBUF_MAXLINK) { + for (;;) { ssize_t len; strbuf_grow(sb, hint + 1); From 1a0c0b5f104745c3918cad59e2f01ea9d4420008 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 19:58:14 +0200 Subject: [PATCH 03/22] lockfile.c: use is_dir_sep() instead of hardcoded '/' checks Signed-off-by: Karsten Blees --- lockfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lockfile.c b/lockfile.c index cc9a4b8428..62e4c9a14f 100644 --- a/lockfile.c +++ b/lockfile.c @@ -17,14 +17,14 @@ static void trim_last_path_component(struct strbuf *path) int i = path->len; /* back up past trailing slashes, if any */ - while (i && path->buf[i - 1] == '/') + while (i && is_dir_sep(path->buf[i - 1])) i--; /* * then go backwards until a slash, or the beginning of the * string */ - while (i && path->buf[i - 1] != '/') + while (i && !is_dir_sep(path->buf[i - 1])) i--; strbuf_setlen(path, i); From 697182a1eeb183d8ee3240538794ab207b12c733 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 12 May 2015 11:09:01 +0200 Subject: [PATCH 04/22] Win32: don't call GetFileAttributes twice in mingw_lstat() GetFileAttributes cannot handle paths with trailing dir separator. The current [l]stat implementation calls GetFileAttributes twice if the path has trailing slashes (first with the original path passed to [l]stat, and and a second time with a path copy with trailing '/' removed). With Unicode conversion, we get the length of the path for free and also have a (wide char) buffer that can be modified. Remove trailing directory separators before calling the Win32 API. Signed-off-by: Karsten Blees --- compat/mingw.c | 48 ++++++++++++------------------------------------ 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0f874a5024..b0c1816b45 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -771,9 +771,18 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; wchar_t wfilename[MAX_LONG_PATH]; - if (xutftowcs_long_path(wfilename, file_name) < 0) + int wlen = xutftowcs_long_path(wfilename, file_name); + if (wlen < 0) return -1; + /* strip trailing '/', or GetFileAttributes will fail */ + while (wlen && is_dir_sep(wfilename[wlen - 1])) + wfilename[--wlen] = 0; + if (!wlen) { + errno = ENOENT; + return -1; + } + if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { buf->st_ino = 0; buf->st_gid = 0; @@ -833,39 +842,6 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) return -1; } -/* We provide our own lstat/fstat functions, since the provided - * lstat/fstat functions are so slow. These stat functions are - * tailored for Git's usage (read: fast), and are not meant to be - * complete. Note that Git stat()s are redirected to mingw_lstat() - * too, since Windows doesn't really handle symlinks that well. - */ -static int do_stat_internal(int follow, const char *file_name, struct stat *buf) -{ - int namelen; - char alt_name[MAX_LONG_PATH]; - - if (!do_lstat(follow, file_name, buf)) - return 0; - - /* if file_name ended in a '/', Windows returned ENOENT; - * try again without trailing slashes - */ - if (errno != ENOENT) - return -1; - - namelen = strlen(file_name); - if (namelen && file_name[namelen-1] != '/') - return -1; - while (namelen && file_name[namelen-1] == '/') - --namelen; - if (!namelen || namelen >= MAX_LONG_PATH) - return -1; - - memcpy(alt_name, file_name, namelen); - alt_name[namelen] = 0; - return do_lstat(follow, alt_name, buf); -} - int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) @@ -893,11 +869,11 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) int mingw_lstat(const char *file_name, struct stat *buf) { - return do_stat_internal(0, file_name, buf); + return do_lstat(0, file_name, buf); } int mingw_stat(const char *file_name, struct stat *buf) { - return do_stat_internal(1, file_name, buf); + return do_lstat(1, file_name, buf); } int mingw_fstat(int fd, struct stat *buf) From 034a82a50e94b5e434b93ec84467c14142173388 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 01:18:14 +0200 Subject: [PATCH 05/22] Win32: implement stat() with symlink support With respect to symlinks, the current stat() implementation is almost the same as lstat(): except for the file type (st_mode & S_IFMT), it returns information about the link rather than the target. Implement stat by opening the file with as little permissions as possible and calling GetFileInformationByHandle on it. This way, all link resoltion is handled by the Windows file system layer. If symlinks are disabled, use lstat() as before, but fail with ELOOP if a symlink would have to be resolved. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index b0c1816b45..6ec7cfac3f 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -871,9 +871,26 @@ int mingw_lstat(const char *file_name, struct stat *buf) { return do_lstat(0, file_name, buf); } + int mingw_stat(const char *file_name, struct stat *buf) { - return do_lstat(1, file_name, buf); + wchar_t wfile_name[MAX_LONG_PATH]; + HANDLE hnd; + int result; + + /* open the file and let Windows resolve the links */ + if (xutftowcs_long_path(wfile_name, file_name) < 0) + return -1; + hnd = CreateFileW(wfile_name, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + result = get_file_info_by_handle(hnd, buf); + CloseHandle(hnd); + return result; } int mingw_fstat(int fd, struct stat *buf) From ab8d99d799da7febfa58072a49b4de12e8fc1a12 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 12 May 2015 00:58:39 +0200 Subject: [PATCH 06/22] Win32: remove separate do_lstat() function With the new mingw_stat() implementation, do_lstat() is only called from mingw_lstat() (with follow == 0). Remove the extra function and the old mingw_stat()-specific (follow == 1) logic. Signed-off-by: Karsten Blees --- compat/mingw.c | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 6ec7cfac3f..4af643fd1f 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -760,14 +760,7 @@ static int has_valid_directory_prefix(wchar_t *wfilename) return 1; } -/* We keep the do_lstat code in a separate function to avoid recursion. - * When a path ends with a slash, the stat will fail with ENOENT. In - * this case, we strip the trailing slashes and stat again. - * - * If follow is true then act like stat() and report on the link - * target. Otherwise report on the link itself. - */ -static int do_lstat(int follow, const char *file_name, struct stat *buf) +int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; wchar_t wfilename[MAX_LONG_PATH]; @@ -801,13 +794,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) if (handle != INVALID_HANDLE_VALUE) { if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { - if (follow) { - char buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; - buf->st_size = readlink(file_name, buffer, MAXIMUM_REPARSE_DATA_BUFFER_SIZE); - } else { - buf->st_mode = S_IFLNK; - } - buf->st_mode |= S_IREAD; + buf->st_mode = S_IFLNK | S_IREAD; if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) buf->st_mode |= S_IWRITE; } @@ -867,11 +854,6 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) return 0; } -int mingw_lstat(const char *file_name, struct stat *buf) -{ - return do_lstat(0, file_name, buf); -} - int mingw_stat(const char *file_name, struct stat *buf) { wchar_t wfile_name[MAX_LONG_PATH]; From 5c498593ec844a8ef7f0859d1f1a13d33a8a26a3 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 00:17:56 +0200 Subject: [PATCH 07/22] Win32: let mingw_lstat() error early upon problems with reparse points When obtaining lstat information for reparse points, we need to call FindFirstFile() in addition to GetFileInformationEx() to obtain the type of the reparse point (symlink, mount point etc.). However, currently there is no error handling whatsoever if FindFirstFile() fails. Call FindFirstFile() before modifying the stat *buf output parameter and error out if the call fails. Note: The FindFirstFile() return value includes all the data that we get from GetFileAttributesEx(), so we could replace GetFileAttributesEx() with FindFirstFile(). We don't do that because GetFileAttributesEx() is about twice as fast for single files. I.e. we only pay the extra cost of calling FindFirstFile() in the rare case that we encounter a reparse point. Note: The indentation of the remaining reparse point code will be fixed in the next patch. Signed-off-by: Karsten Blees --- compat/mingw.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 4af643fd1f..74ed47428e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -763,6 +763,7 @@ static int has_valid_directory_prefix(wchar_t *wfilename) int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; + WIN32_FIND_DATAW findbuf = { 0 }; wchar_t wfilename[MAX_LONG_PATH]; int wlen = xutftowcs_long_path(wfilename, file_name); if (wlen < 0) @@ -777,6 +778,13 @@ int mingw_lstat(const char *file_name, struct stat *buf) } if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { + /* for reparse points, use FindFirstFile to get the reparse tag */ + if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + HANDLE handle = FindFirstFileW(wfilename, &findbuf); + if (handle == INVALID_HANDLE_VALUE) + goto error; + FindClose(handle); + } buf->st_ino = 0; buf->st_gid = 0; buf->st_uid = 0; @@ -789,20 +797,16 @@ int mingw_lstat(const char *file_name, struct stat *buf) filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - WIN32_FIND_DATAW findbuf; - HANDLE handle = FindFirstFileW(wfilename, &findbuf); - if (handle != INVALID_HANDLE_VALUE) { if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { buf->st_mode = S_IFLNK | S_IREAD; if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) buf->st_mode |= S_IWRITE; } - FindClose(handle); - } } return 0; } +error: switch (GetLastError()) { case ERROR_ACCESS_DENIED: case ERROR_SHARING_VIOLATION: From 1957c149fd4bd6cfa4f6a4561b6f77a6b0bf6f15 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 10 Jan 2017 23:21:56 +0100 Subject: [PATCH 08/22] mingw: teach fscache and dirent about symlinks Move S_IFLNK detection to file_attr_to_st_mode() and reuse it in fscache. Implement DT_LNK detection in dirent.c and the fscache readdir version. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 13 +++---------- compat/win32.h | 6 ++++-- compat/win32/dirent.c | 5 ++++- compat/win32/fscache.c | 11 +++++++---- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 74ed47428e..6a0c228f4c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -789,21 +789,14 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, + findbuf.dwReserved0); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); - if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && - (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { - buf->st_mode = S_IFLNK | S_IREAD; - if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) - buf->st_mode |= S_IWRITE; - } - } return 0; } error: @@ -848,7 +841,7 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ diff --git a/compat/win32.h b/compat/win32.h index a97e880757..671bcc81f9 100644 --- a/compat/win32.h +++ b/compat/win32.h @@ -6,10 +6,12 @@ #include #endif -static inline int file_attr_to_st_mode (DWORD attr) +static inline int file_attr_to_st_mode (DWORD attr, DWORD tag) { int fMode = S_IREAD; - if (attr & FILE_ATTRIBUTE_DIRECTORY) + if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && tag == IO_REPARSE_TAG_SYMLINK) + fMode |= S_IFLNK; + else if (attr & FILE_ATTRIBUTE_DIRECTORY) fMode |= S_IFDIR; else fMode |= S_IFREG; diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index c693886b42..9fc8bfb645 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -15,7 +15,10 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ - if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + if ((fdata->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + && fdata->dwReserved0 == IO_REPARSE_TAG_SYMLINK) + ent->d_type = DT_LNK; + else if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ent->d_type = DT_DIR; else ent->d_type = DT_REG; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index a7fadd32dd..19eee39a08 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -194,10 +194,13 @@ static struct fsentry *fseentry_create_entry(struct fscache *cache, fdata->FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT ? fdata->EaSize : 0; - fse->st_mode = file_attr_to_st_mode(fdata->FileAttributes); - fse->dirent.d_type = S_ISDIR(fse->st_mode) ? DT_DIR : DT_REG; - fse->u.s.st_size = fdata->EndOfFile.LowPart | - (((off_t)fdata->EndOfFile.HighPart) << 32); + fse->st_mode = file_attr_to_st_mode(fdata->FileAttributes, + fdata->EaSize); + fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG : + S_ISDIR(fse->st_mode) ? DT_DIR : DT_LNK; + fse->u.s.st_size = S_ISLNK(fse->st_mode) ? MAX_LONG_PATH : + fdata->EndOfFile.LowPart | + (((off_t)fdata->EndOfFile.HighPart) << 32); filetime_to_timespec((FILETIME *)&(fdata->LastAccessTime), &(fse->u.s.st_atim)); filetime_to_timespec((FILETIME *)&(fdata->LastWriteTime), From 3beac6b4cb7780915d39f7f3cb43dce9f8360e72 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 01:11:37 +0200 Subject: [PATCH 09/22] Win32: lstat(): return adequate stat.st_size for symlinks Git typically doesn't trust the stat.st_size member of symlinks (e.g. see strbuf_readlink()). However, some functions take shortcuts if st_size is 0 (e.g. diff_populate_filespec()). In mingw_lstat() and fscache_lstat(), make sure to return an adequate size. The extra overhead of opening and reading the reparse point to calculate the exact size is not necessary, as git doesn't rely on the value anyway. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 6a0c228f4c..3b105e0bd7 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -791,8 +791,8 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_nlink = 1; buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, findbuf.dwReserved0); - buf->st_size = fdata.nFileSizeLow | - (((off_t)fdata.nFileSizeHigh)<<32); + buf->st_size = S_ISLNK(buf->st_mode) ? MAX_LONG_PATH : + fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); From 3ec4e76f35ddb34d2ae29f74ef024e59dee7c069 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 19 May 2015 21:48:55 +0200 Subject: [PATCH 10/22] Win32: factor out retry logic The retry pattern is duplicated in three places. It also seems to be too hard to use: mingw_unlink() and mingw_rmdir() duplicate the code to retry, and both of them do so incompletely. They also do not restore errno if the user answers 'no'. Introduce a retry_ask_yes_no() helper function that handles retry with small delay, asking the user, and restoring errno. mingw_unlink: include _wchmod in the retry loop (which may fail if the file is locked exclusively). mingw_rmdir: include special error handling in the retry loop. Signed-off-by: Karsten Blees --- compat/mingw.c | 98 ++++++++++++++++++++++---------------------------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3b105e0bd7..bb19104918 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -12,8 +12,6 @@ #define HCAST(type, handle) ((type)(intptr_t)handle) -static const int delay[] = { 0, 1, 10, 20, 40 }; - void open_in_gdb(void) { static struct child_process cp = CHILD_PROCESS_INIT; @@ -190,15 +188,12 @@ static int read_yes_no_answer(void) return -1; } -static int ask_yes_no_if_possible(const char *format, ...) +static int ask_yes_no_if_possible(const char *format, va_list args) { char question[4096]; const char *retry_hook[] = { NULL, NULL, NULL }; - va_list args; - va_start(args, format); vsnprintf(question, sizeof(question), format, args); - va_end(args); if ((retry_hook[0] = mingw_getenv("GIT_ASK_YESNO"))) { retry_hook[1] = question; @@ -220,6 +215,31 @@ static int ask_yes_no_if_possible(const char *format, ...) } } +static int retry_ask_yes_no(int *tries, const char *format, ...) +{ + static const int delay[] = { 0, 1, 10, 20, 40 }; + va_list args; + int result, saved_errno = errno; + + if ((*tries) < ARRAY_SIZE(delay)) { + /* + * We assume that some other process had the file open at the wrong + * moment and retry. In order to give the other process a higher + * chance to complete its operation, we give up our time slice now. + * If we have to retry again, we do sleep a bit. + */ + Sleep(delay[*tries]); + (*tries)++; + return 1; + } + + va_start(args, format); + result = ask_yes_no_if_possible(format, args); + va_end(args); + errno = saved_errno; + return result; +} + /* Windows only */ enum hide_dotfiles_type { HIDE_DOTFILES_FALSE = 0, @@ -298,7 +318,7 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf) int mingw_unlink(const char *pathname) { - int ret, tries = 0; + int tries = 0; wchar_t wpathname[MAX_LONG_PATH]; if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; @@ -306,26 +326,16 @@ int mingw_unlink(const char *pathname) if (DeleteFileW(wpathname)) return 0; - /* read-only files cannot be removed */ - _wchmod(wpathname, 0666); - while ((ret = _wunlink(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { + do { + /* read-only files cannot be removed */ + _wchmod(wpathname, 0666); + if (!_wunlink(wpathname)) + return 0; if (!is_file_in_use_error(GetLastError())) break; - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - } - while (ret == -1 && is_file_in_use_error(GetLastError()) && - ask_yes_no_if_possible("Unlink of file '%s' failed. " - "Should I try again?", pathname)) - ret = _wunlink(wpathname); - return ret; + } while (retry_ask_yes_no(&tries, "Unlink of file '%s' failed. " + "Should I try again?", pathname)); + return -1; } static int is_dir_empty(const wchar_t *wpath) @@ -352,12 +362,14 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { - int ret, tries = 0; + int tries = 0; wchar_t wpathname[MAX_LONG_PATH]; if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; - while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { + do { + if (!_wrmdir(wpathname)) + return 0; if (!is_file_in_use_error(GetLastError())) errno = err_win_to_posix(GetLastError()); if (errno != EACCES) @@ -366,21 +378,9 @@ int mingw_rmdir(const char *pathname) errno = ENOTEMPTY; break; } - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - } - while (ret == -1 && errno == EACCES && is_file_in_use_error(GetLastError()) && - ask_yes_no_if_possible("Deletion of directory '%s' failed. " - "Should I try again?", pathname)) - ret = _wrmdir(wpathname); - return ret; + } while (retry_ask_yes_no(&tries, "Deletion of directory '%s' failed. " + "Should I try again?", pathname)); + return -1; } static inline int needs_hiding(const char *path) @@ -2310,20 +2310,8 @@ repeat: SetFileAttributesW(wpnew, attrs); } } - if (tries < ARRAY_SIZE(delay) && gle == ERROR_ACCESS_DENIED) { - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - goto repeat; - } if (gle == ERROR_ACCESS_DENIED && - ask_yes_no_if_possible("Rename from '%s' to '%s' failed. " + retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " "Should I try again?", pold, pnew)) goto repeat; From aa6377fba4706c0bb185d9ad06e2a958568e471e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:55:05 +0200 Subject: [PATCH 11/22] Win32: change default of 'core.symlinks' to false Symlinks on Windows don't work the same way as on Unix systems. E.g. there are different types of symlinks for directories and files, creating symlinks requires administrative privileges etc. By default, disable symlink support on Windows. I.e. users explicitly have to enable it with 'git config [--system|--global] core.symlinks true'. The test suite ignores system / global config files. Allow testing *with* symlink support by checking if native symlinks are enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Reminder: This would need to be changed if / when we find a way to run the test suite in a non-MSys-based shell (e.g. dash). Signed-off-by: Karsten Blees --- compat/mingw.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index bb19104918..94ff844081 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2885,6 +2885,15 @@ static void setup_windows_environment(void) if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) setenv("LC_CTYPE", "C.UTF-8", 1); + + /* + * Change 'core.symlinks' default to false, unless native symlinks are + * enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Thus we can + * run the test suite (which doesn't obey config files) with or without + * symlink support. + */ + if (!(tmp = getenv("MSYS")) || !strstr(tmp, "winsymlinks:nativestrict")) + has_symlinks = 0; } int is_valid_win32_path(const char *path, int allow_literal_nul) From 6b21f220dd09232271de60691803693272b81c5f Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 00:32:03 +0200 Subject: [PATCH 12/22] Win32: add symlink-specific error codes Signed-off-by: Karsten Blees --- compat/mingw.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 94ff844081..10857bbfae 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -87,6 +87,7 @@ int err_win_to_posix(DWORD winerr) case ERROR_INVALID_PARAMETER: error = EINVAL; break; case ERROR_INVALID_PASSWORD: error = EPERM; break; case ERROR_INVALID_PRIMARY_GROUP: error = EINVAL; break; + case ERROR_INVALID_REPARSE_DATA: error = EINVAL; break; case ERROR_INVALID_SIGNAL_NUMBER: error = EINVAL; break; case ERROR_INVALID_TARGET_HANDLE: error = EIO; break; case ERROR_INVALID_WORKSTATION: error = EACCES; break; @@ -101,6 +102,7 @@ int err_win_to_posix(DWORD winerr) case ERROR_NEGATIVE_SEEK: error = ESPIPE; break; case ERROR_NOACCESS: error = EFAULT; break; case ERROR_NONE_MAPPED: error = EINVAL; break; + case ERROR_NOT_A_REPARSE_POINT: error = EINVAL; break; case ERROR_NOT_ENOUGH_MEMORY: error = ENOMEM; break; case ERROR_NOT_READY: error = EAGAIN; break; case ERROR_NOT_SAME_DEVICE: error = EXDEV; break; @@ -121,6 +123,9 @@ int err_win_to_posix(DWORD winerr) case ERROR_PIPE_NOT_CONNECTED: error = EPIPE; break; case ERROR_PRIVILEGE_NOT_HELD: error = EACCES; break; case ERROR_READ_FAULT: error = EIO; break; + case ERROR_REPARSE_ATTRIBUTE_CONFLICT: error = EINVAL; break; + case ERROR_REPARSE_TAG_INVALID: error = EINVAL; break; + case ERROR_REPARSE_TAG_MISMATCH: error = EINVAL; break; case ERROR_SEEK: error = EIO; break; case ERROR_SEEK_ON_DEVICE: error = ESPIPE; break; case ERROR_SHARING_BUFFER_EXCEEDED: error = ENFILE; break; From b10d749259526daa09a82a02de0a2cd39ac9a0f0 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:06:10 +0200 Subject: [PATCH 13/22] Win32: mingw_unlink: support symlinks to directories _wunlink() / DeleteFileW() refuses to delete symlinks to directories. If _wunlink() fails with ERROR_ACCESS_DENIED, try _wrmdir() as well. Signed-off-by: Karsten Blees --- compat/mingw.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 10857bbfae..31612d628c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -338,6 +338,13 @@ int mingw_unlink(const char *pathname) return 0; if (!is_file_in_use_error(GetLastError())) break; + /* + * _wunlink() / DeleteFileW() for directory symlinks fails with + * ERROR_ACCESS_DENIED (EACCES), so try _wrmdir() as well. This is the + * same error we get if a file is in use (already checked above). + */ + if (!_wrmdir(wpathname)) + return 0; } while (retry_ask_yes_no(&tries, "Unlink of file '%s' failed. " "Should I try again?", pathname)); return -1; From 69f3b00f64091d8d59ed19cf8959e584d2abb1a3 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 19 May 2015 22:42:48 +0200 Subject: [PATCH 14/22] Win32: mingw_rename: support renaming symlinks MSVCRT's _wrename() cannot rename symlinks over existing files: it returns success without doing anything. Newer MSVCR*.dll versions probably do not have this problem: according to CRT sources, they just call MoveFileEx() with the MOVEFILE_COPY_ALLOWED flag. Get rid of _wrename() and call MoveFileEx() with proper error handling. Signed-off-by: Karsten Blees --- compat/mingw.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 31612d628c..2e6f0dffd1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2282,27 +2282,29 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) #undef rename int mingw_rename(const char *pold, const char *pnew) { - DWORD attrs, gle; + DWORD attrs = INVALID_FILE_ATTRIBUTES, gle; int tries = 0; wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; if (xutftowcs_long_path(wpold, pold) < 0 || xutftowcs_long_path(wpnew, pnew) < 0) return -1; - /* - * Try native rename() first to get errno right. - * It is based on MoveFile(), which cannot overwrite existing files. - */ - if (!_wrename(wpold, wpnew)) - return 0; - if (errno != EEXIST) - return -1; repeat: - if (MoveFileExW(wpold, wpnew, MOVEFILE_REPLACE_EXISTING)) + if (MoveFileExW(wpold, wpnew, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED)) return 0; - /* TODO: translate more errors */ gle = GetLastError(); - if (gle == ERROR_ACCESS_DENIED && + + /* revert file attributes on failure */ + if (attrs != INVALID_FILE_ATTRIBUTES) + SetFileAttributesW(wpnew, attrs); + + if (!is_file_in_use_error(gle)) { + errno = err_win_to_posix(gle); + return -1; + } + + if (attrs == INVALID_FILE_ATTRIBUTES && (attrs = GetFileAttributesW(wpnew)) != INVALID_FILE_ATTRIBUTES) { if (attrs & FILE_ATTRIBUTE_DIRECTORY) { DWORD attrsold = GetFileAttributesW(wpold); @@ -2314,16 +2316,10 @@ repeat: return -1; } if ((attrs & FILE_ATTRIBUTE_READONLY) && - SetFileAttributesW(wpnew, attrs & ~FILE_ATTRIBUTE_READONLY)) { - if (MoveFileExW(wpold, wpnew, MOVEFILE_REPLACE_EXISTING)) - return 0; - gle = GetLastError(); - /* revert file attributes on failure */ - SetFileAttributesW(wpnew, attrs); - } + SetFileAttributesW(wpnew, attrs & ~FILE_ATTRIBUTE_READONLY)) + goto repeat; } - if (gle == ERROR_ACCESS_DENIED && - retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " + if (retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " "Should I try again?", pold, pnew)) goto repeat; From cbf0d09859524921b7165c53e6de1f294457e4b5 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:17:31 +0200 Subject: [PATCH 15/22] Win32: mingw_chdir: change to symlink-resolved directory If symlinks are enabled, resolve all symlinks when changing directories, as required by POSIX. Note: Git's real_path() function bases its link resolution algorithm on this property of chdir(). Unfortunately, the current directory on Windows is limited to only MAX_PATH (260) characters. Therefore using symlinks and long paths in combination may be problematic. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 2e6f0dffd1..46b97e4212 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -726,7 +726,24 @@ int mingw_chdir(const char *dirname) wchar_t wdirname[MAX_LONG_PATH]; if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; - result = _wchdir(wdirname); + + if (has_symlinks) { + HANDLE hnd = CreateFileW(wdirname, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + if (!GetFinalPathNameByHandleW(hnd, wdirname, ARRAY_SIZE(wdirname), 0)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(hnd); + return -1; + } + CloseHandle(hnd); + } + + result = _wchdir(normalize_ntpath(wdirname)); current_directory_len = GetCurrentDirectoryW(0, NULL); return result; } From 5dcbd476c4756563c0edba81bfc636d52591f6c0 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:24:41 +0200 Subject: [PATCH 16/22] Win32: implement readlink() Implement readlink() by reading NTFS reparse points. Works for symlinks and directory junctions. If symlinks are disabled, fail with ENOSYS. Signed-off-by: Karsten Blees --- compat/mingw.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 3 +- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 46b97e4212..3a1c1ae036 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2,6 +2,7 @@ #include "win32.h" #include #include +#include #include "../strbuf.h" #include "../run-command.h" #include "../cache.h" @@ -2613,6 +2614,103 @@ int link(const char *oldpath, const char *newpath) return 0; } +#ifndef _WINNT_H +/* + * The REPARSE_DATA_BUFFER structure is defined in the Windows DDK (in + * ntifs.h) and in MSYS1's winnt.h (which defines _WINNT_H). So define + * it ourselves if we are on MSYS2 (whose winnt.h defines _WINNT_). + */ +typedef struct _REPARSE_DATA_BUFFER { + DWORD ReparseTag; + WORD ReparseDataLength; + WORD Reserved; +#ifndef _MSC_VER + _ANONYMOUS_UNION +#endif + union { + struct { + WORD SubstituteNameOffset; + WORD SubstituteNameLength; + WORD PrintNameOffset; + WORD PrintNameLength; + ULONG Flags; + WCHAR PathBuffer[1]; + } SymbolicLinkReparseBuffer; + struct { + WORD SubstituteNameOffset; + WORD SubstituteNameLength; + WORD PrintNameOffset; + WORD PrintNameLength; + WCHAR PathBuffer[1]; + } MountPointReparseBuffer; + struct { + BYTE DataBuffer[1]; + } GenericReparseBuffer; + } DUMMYUNIONNAME; +} REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER; +#endif + +int readlink(const char *path, char *buf, size_t bufsiz) +{ + HANDLE handle; + WCHAR wpath[MAX_LONG_PATH], *wbuf; + REPARSE_DATA_BUFFER *b = alloca(MAXIMUM_REPARSE_DATA_BUFFER_SIZE); + DWORD dummy; + char tmpbuf[MAX_LONG_PATH]; + int len; + + if (xutftowcs_long_path(wpath, path) < 0) + return -1; + + /* read reparse point data */ + handle = CreateFileW(wpath, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (handle == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + if (!DeviceIoControl(handle, FSCTL_GET_REPARSE_POINT, NULL, 0, b, + MAXIMUM_REPARSE_DATA_BUFFER_SIZE, &dummy, NULL)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(handle); + return -1; + } + CloseHandle(handle); + + /* get target path for symlinks or mount points (aka 'junctions') */ + switch (b->ReparseTag) { + case IO_REPARSE_TAG_SYMLINK: + wbuf = (WCHAR*) (((char*) b->SymbolicLinkReparseBuffer.PathBuffer) + + b->SymbolicLinkReparseBuffer.SubstituteNameOffset); + *(WCHAR*) (((char*) wbuf) + + b->SymbolicLinkReparseBuffer.SubstituteNameLength) = 0; + break; + case IO_REPARSE_TAG_MOUNT_POINT: + wbuf = (WCHAR*) (((char*) b->MountPointReparseBuffer.PathBuffer) + + b->MountPointReparseBuffer.SubstituteNameOffset); + *(WCHAR*) (((char*) wbuf) + + b->MountPointReparseBuffer.SubstituteNameLength) = 0; + break; + default: + errno = EINVAL; + return -1; + } + + /* + * Adapt to strange readlink() API: Copy up to bufsiz *bytes*, potentially + * cutting off a UTF-8 sequence. Insufficient bufsize is *not* a failure + * condition. There is no conversion function that produces invalid UTF-8, + * so convert to a (hopefully large enough) temporary buffer, then memcpy + * the requested number of bytes (including '\0' for robustness). + */ + if ((len = xwcstoutf(tmpbuf, normalize_ntpath(wbuf), MAX_LONG_PATH)) < 0) + return -1; + memcpy(buf, tmpbuf, min(bufsiz, len + 1)); + return min(bufsiz, len); +} + pid_t waitpid(pid_t pid, int *status, int options) { HANDLE h = OpenProcess(SYNCHRONIZE | PROCESS_QUERY_INFORMATION, diff --git a/compat/mingw.h b/compat/mingw.h index f414bc28bb..903a210b69 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -123,8 +123,6 @@ struct utsname { * trivial stubs */ -static inline int readlink(const char *path, char *buf, size_t bufsiz) -{ errno = ENOSYS; return -1; } static inline int symlink(const char *oldpath, const char *newpath) { errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) @@ -217,6 +215,7 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); +int readlink(const char *path, char *buf, size_t bufsiz); /* * replacements of existing functions From d4363f6cde61b718f1b8adaf24f4a122d8dcfedd Mon Sep 17 00:00:00 2001 From: Bill Zissimopoulos Date: Thu, 28 May 2020 16:35:57 -0700 Subject: [PATCH 17/22] mingw: lstat: compute correct size for symlinks This commit fixes mingw_lstat by computing the proper size for symlinks according to POSIX. POSIX specifies that upon successful return from lstat: "the value of the st_size member shall be set to the length of the pathname contained in the symbolic link not including any terminating null byte". Prior to this commit the mingw_lstat function returned a fixed size of 4096. This caused problems in git repositories that were accessed by git for Cygwin or git for WSL. For example, doing `git reset --hard` using git for Windows would update the size of symlinks in the index to be 4096; at a later time git for Cygwin or git for WSL would find that symlinks have changed size during `git status`. Vice versa doing `git reset --hard` in git for Cygwin or git for WSL would update the size of symlinks in the index with the correct value, only for git for Windows to find incorrectly at a later time that the size had changed. Signed-off-by: Bill Zissimopoulos Signed-off-by: Johannes Schindelin --- compat/mingw.c | 65 ++++++++++++++++++++++++++++-------------- compat/win32/fscache.c | 12 ++++++++ 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3a1c1ae036..9779fa531a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -790,10 +790,14 @@ static int has_valid_directory_prefix(wchar_t *wfilename) return 1; } +static int readlink_1(const WCHAR *wpath, BOOL fail_on_unknown_tag, + char *tmpbuf, int *plen, DWORD *ptag); + int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - WIN32_FIND_DATAW findbuf = { 0 }; + DWORD reparse_tag = 0; + int link_len = 0; wchar_t wfilename[MAX_LONG_PATH]; int wlen = xutftowcs_long_path(wfilename, file_name); if (wlen < 0) @@ -808,20 +812,21 @@ int mingw_lstat(const char *file_name, struct stat *buf) } if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { - /* for reparse points, use FindFirstFile to get the reparse tag */ + /* for reparse points, get the link tag and length */ if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - HANDLE handle = FindFirstFileW(wfilename, &findbuf); - if (handle == INVALID_HANDLE_VALUE) - goto error; - FindClose(handle); + char tmpbuf[MAX_LONG_PATH]; + + if (readlink_1(wfilename, FALSE, tmpbuf, &link_len, + &reparse_tag) < 0) + return -1; } buf->st_ino = 0; buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, - findbuf.dwReserved0); - buf->st_size = S_ISLNK(buf->st_mode) ? MAX_LONG_PATH : + reparse_tag); + buf->st_size = S_ISLNK(buf->st_mode) ? link_len : fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); @@ -829,7 +834,7 @@ int mingw_lstat(const char *file_name, struct stat *buf) filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); return 0; } -error: + switch (GetLastError()) { case ERROR_ACCESS_DENIED: case ERROR_SHARING_VIOLATION: @@ -2650,17 +2655,13 @@ typedef struct _REPARSE_DATA_BUFFER { } REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER; #endif -int readlink(const char *path, char *buf, size_t bufsiz) +static int readlink_1(const WCHAR *wpath, BOOL fail_on_unknown_tag, + char *tmpbuf, int *plen, DWORD *ptag) { HANDLE handle; - WCHAR wpath[MAX_LONG_PATH], *wbuf; + WCHAR *wbuf; REPARSE_DATA_BUFFER *b = alloca(MAXIMUM_REPARSE_DATA_BUFFER_SIZE); DWORD dummy; - char tmpbuf[MAX_LONG_PATH]; - int len; - - if (xutftowcs_long_path(wpath, path) < 0) - return -1; /* read reparse point data */ handle = CreateFileW(wpath, 0, @@ -2680,7 +2681,7 @@ int readlink(const char *path, char *buf, size_t bufsiz) CloseHandle(handle); /* get target path for symlinks or mount points (aka 'junctions') */ - switch (b->ReparseTag) { + switch ((*ptag = b->ReparseTag)) { case IO_REPARSE_TAG_SYMLINK: wbuf = (WCHAR*) (((char*) b->SymbolicLinkReparseBuffer.PathBuffer) + b->SymbolicLinkReparseBuffer.SubstituteNameOffset); @@ -2694,10 +2695,34 @@ int readlink(const char *path, char *buf, size_t bufsiz) + b->MountPointReparseBuffer.SubstituteNameLength) = 0; break; default: - errno = EINVAL; - return -1; + if (fail_on_unknown_tag) { + errno = EINVAL; + return -1; + } else { + *plen = MAX_LONG_PATH; + return 0; + } } + if ((*plen = + xwcstoutf(tmpbuf, normalize_ntpath(wbuf), MAX_LONG_PATH)) < 0) + return -1; + return 0; +} + +int readlink(const char *path, char *buf, size_t bufsiz) +{ + WCHAR wpath[MAX_LONG_PATH]; + char tmpbuf[MAX_LONG_PATH]; + int len; + DWORD tag; + + if (xutftowcs_long_path(wpath, path) < 0) + return -1; + + if (readlink_1(wpath, TRUE, tmpbuf, &len, &tag) < 0) + return -1; + /* * Adapt to strange readlink() API: Copy up to bufsiz *bytes*, potentially * cutting off a UTF-8 sequence. Insufficient bufsize is *not* a failure @@ -2705,8 +2730,6 @@ int readlink(const char *path, char *buf, size_t bufsiz) * so convert to a (hopefully large enough) temporary buffer, then memcpy * the requested number of bytes (including '\0' for robustness). */ - if ((len = xwcstoutf(tmpbuf, normalize_ntpath(wbuf), MAX_LONG_PATH)) < 0) - return -1; memcpy(buf, tmpbuf, min(bufsiz, len + 1)); return min(bufsiz, len); } diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 19eee39a08..932c818ed4 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -574,6 +574,18 @@ int fscache_lstat(const char *filename, struct stat *st) if (!fse) return -1; + /* + * Special case symbolic links: FindFirstFile()/FindNextFile() did not + * provide us with the length of the target path. + */ + if (fse->u.s.st_size == MAX_LONG_PATH && S_ISLNK(fse->st_mode)) { + char buf[MAX_LONG_PATH]; + int len = readlink(filename, buf, sizeof(buf) - 1); + + if (len > 0) + fse->u.s.st_size = len; + } + /* copy stat data */ st->st_ino = 0; st->st_gid = 0; From d4f0ee562e4a2ac5bb0793e074324b22ac1c42ca Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:32:03 +0200 Subject: [PATCH 18/22] Win32: implement basic symlink() functionality (file symlinks only) Implement symlink() that always creates file symlinks. Fails with ENOSYS if symlinks are disabled or unsupported. Note: CreateSymbolicLinkW() was introduced with symlink support in Windows Vista. For compatibility with Windows XP, we need to load it dynamically and fail gracefully if it isnt's available. Signed-off-by: Karsten Blees --- compat/mingw.c | 28 ++++++++++++++++++++++++++++ compat/mingw.h | 3 +-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 9779fa531a..b6c8b63f54 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2619,6 +2619,34 @@ int link(const char *oldpath, const char *newpath) return 0; } +int symlink(const char *target, const char *link) +{ + wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; + int len; + + /* fail if symlinks are disabled or API is not supported (WinXP) */ + if (!has_symlinks) { + errno = ENOSYS; + return -1; + } + + if ((len = xutftowcs_long_path(wtarget, target)) < 0 + || xutftowcs_long_path(wlink, link) < 0) + return -1; + + /* convert target dir separators to backslashes */ + while (len--) + if (wtarget[len] == '/') + wtarget[len] = '\\'; + + /* create file symlink */ + if (!CreateSymbolicLinkW(wlink, wtarget, 0)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + return 0; +} + #ifndef _WINNT_H /* * The REPARSE_DATA_BUFFER structure is defined in the Windows DDK (in diff --git a/compat/mingw.h b/compat/mingw.h index 903a210b69..41214cb1d7 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -123,8 +123,6 @@ struct utsname { * trivial stubs */ -static inline int symlink(const char *oldpath, const char *newpath) -{ errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) { errno = ENOSYS; return -1; } #ifndef __MINGW64_VERSION_MAJOR @@ -215,6 +213,7 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); +int symlink(const char *target, const char *link); int readlink(const char *path, char *buf, size_t bufsiz); /* From 76312bc6947ef25deda4133c8820b4449508bd66 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:48:35 +0200 Subject: [PATCH 19/22] Win32: symlink: add support for symlinks to directories Symlinks on Windows have a flag that indicates whether the target is a file or a directory. Symlinks of wrong type simply don't work. This even affects core Win32 APIs (e.g. DeleteFile() refuses to delete directory symlinks). However, CreateFile() with FILE_FLAG_BACKUP_SEMANTICS doesn't seem to care. Check the target type by first creating a tentative file symlink, opening it, and checking the type of the resulting handle. If it is a directory, recreate the symlink with the directory flag set. It is possible to create symlinks before the target exists (or in case of symlinks to symlinks: before the target type is known). If this happens, create a tentative file symlink and postpone the directory decision: keep a list of phantom symlinks to be processed whenever a new directory is created in mingw_mkdir(). Limitations: This algorithm may fail if a link target changes from file to directory or vice versa, or if the target directory is created in another process. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 164 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index b6c8b63f54..6d6f0b95e6 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -297,6 +297,131 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } +enum phantom_symlink_result { + PHANTOM_SYMLINK_RETRY, + PHANTOM_SYMLINK_DONE, + PHANTOM_SYMLINK_DIRECTORY +}; + +static inline int is_wdir_sep(wchar_t wchar) +{ + return wchar == L'/' || wchar == L'\\'; +} + +static const wchar_t *make_relative_to(const wchar_t *path, + const wchar_t *relative_to, wchar_t *out, + size_t size) +{ + size_t i = wcslen(relative_to), len; + + /* Is `path` already absolute? */ + if (is_wdir_sep(path[0]) || + (iswalpha(path[0]) && path[1] == L':' && is_wdir_sep(path[2]))) + return path; + + while (i > 0 && !is_wdir_sep(relative_to[i - 1])) + i--; + + /* Is `relative_to` in the current directory? */ + if (!i) + return path; + + len = wcslen(path); + if (i + len + 1 > size) { + error("Could not make '%S' relative to '%S' (too large)", + path, relative_to); + return NULL; + } + + memcpy(out, relative_to, i * sizeof(wchar_t)); + wcscpy(out + i, path); + return out; +} + +/* + * Changes a file symlink to a directory symlink if the target exists and is a + * directory. + */ +static enum phantom_symlink_result +process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink) +{ + HANDLE hnd; + BY_HANDLE_FILE_INFORMATION fdata; + wchar_t relative[MAX_LONG_PATH]; + const wchar_t *rel; + + /* check that wlink is still a file symlink */ + if ((GetFileAttributesW(wlink) + & (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)) + != FILE_ATTRIBUTE_REPARSE_POINT) + return PHANTOM_SYMLINK_DONE; + + /* make it relative, if necessary */ + rel = make_relative_to(wtarget, wlink, relative, ARRAY_SIZE(relative)); + if (!rel) + return PHANTOM_SYMLINK_DONE; + + /* let Windows resolve the link by opening it */ + hnd = CreateFileW(rel, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return PHANTOM_SYMLINK_RETRY; + } + + if (!GetFileInformationByHandle(hnd, &fdata)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(hnd); + return PHANTOM_SYMLINK_RETRY; + } + CloseHandle(hnd); + + /* if target exists and is a file, we're done */ + if (!(fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + return PHANTOM_SYMLINK_DONE; + + /* otherwise recreate the symlink with directory flag */ + if (DeleteFileW(wlink) && CreateSymbolicLinkW(wlink, wtarget, 1)) + return PHANTOM_SYMLINK_DIRECTORY; + + errno = err_win_to_posix(GetLastError()); + return PHANTOM_SYMLINK_RETRY; +} + +/* keep track of newly created symlinks to non-existing targets */ +struct phantom_symlink_info { + struct phantom_symlink_info *next; + wchar_t *wlink; + wchar_t *wtarget; +}; + +static struct phantom_symlink_info *phantom_symlinks = NULL; +static CRITICAL_SECTION phantom_symlinks_cs; + +static void process_phantom_symlinks(void) +{ + struct phantom_symlink_info *current, **psi; + EnterCriticalSection(&phantom_symlinks_cs); + /* process phantom symlinks list */ + psi = &phantom_symlinks; + while ((current = *psi)) { + enum phantom_symlink_result result = process_phantom_symlink( + current->wtarget, current->wlink); + if (result == PHANTOM_SYMLINK_RETRY) { + psi = ¤t->next; + } else { + /* symlink was processed, remove from list */ + *psi = current->next; + free(current); + /* if symlink was a directory, start over */ + if (result == PHANTOM_SYMLINK_DIRECTORY) + psi = &phantom_symlinks; + } + } + LeaveCriticalSection(&phantom_symlinks_cs); +} + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -457,6 +582,8 @@ int mingw_mkdir(const char *path, int mode) return -1; ret = _wmkdir(wpath); + if (!ret) + process_phantom_symlinks(); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); return ret; @@ -2644,6 +2771,42 @@ int symlink(const char *target, const char *link) errno = err_win_to_posix(GetLastError()); return -1; } + + /* convert to directory symlink if target exists */ + switch (process_phantom_symlink(wtarget, wlink)) { + case PHANTOM_SYMLINK_RETRY: { + /* if target doesn't exist, add to phantom symlinks list */ + wchar_t wfullpath[MAX_LONG_PATH]; + struct phantom_symlink_info *psi; + + /* convert to absolute path to be independent of cwd */ + len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); + if (!len || len >= MAX_LONG_PATH) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* over-allocate and fill phantom_symlink_info structure */ + psi = xmalloc(sizeof(struct phantom_symlink_info) + + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); + psi->wlink = (wchar_t *)(psi + 1); + wcscpy(psi->wlink, wfullpath); + psi->wtarget = psi->wlink + len + 1; + wcscpy(psi->wtarget, wtarget); + + EnterCriticalSection(&phantom_symlinks_cs); + psi->next = phantom_symlinks; + phantom_symlinks = psi; + LeaveCriticalSection(&phantom_symlinks_cs); + break; + } + case PHANTOM_SYMLINK_DIRECTORY: + /* if we created a dir symlink, process other phantom symlinks */ + process_phantom_symlinks(); + break; + default: + break; + } return 0; } @@ -3404,6 +3567,7 @@ int wmain(int argc, const wchar_t **wargv) /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); + InitializeCriticalSection(&phantom_symlinks_cs); /* initialize critical section for fscache */ InitializeCriticalSection(&fscache_cs); From 932215ad73519a9dfea365582c003647f31e6785 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 30 May 2017 21:50:57 +0200 Subject: [PATCH 20/22] mingw: try to create symlinks without elevated permissions With Windows 10 Build 14972 in Developer Mode, a new flag is supported by CreateSymbolicLink() to create symbolic links even when running outside of an elevated session (which was previously required). This new flag is called SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE and has the numeric value 0x02. Previous Windows 10 versions will not understand that flag and return an ERROR_INVALID_PARAMETER, therefore we have to be careful to try passing that flag only when the build number indicates that it is supported. For more information about the new flag, see this blog post: https://blogs.windows.com/buildingapps/2016/12/02/symlinks-windows-10/ This patch is loosely based on the patch submitted by Samuel D. Leslie as https://github.com/git-for-windows/git/pull/1184. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 6d6f0b95e6..e17b49728c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -297,6 +297,8 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } +static DWORD symlink_file_flags = 0, symlink_directory_flags = 1; + enum phantom_symlink_result { PHANTOM_SYMLINK_RETRY, PHANTOM_SYMLINK_DONE, @@ -382,7 +384,8 @@ process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink) return PHANTOM_SYMLINK_DONE; /* otherwise recreate the symlink with directory flag */ - if (DeleteFileW(wlink) && CreateSymbolicLinkW(wlink, wtarget, 1)) + if (DeleteFileW(wlink) && + CreateSymbolicLinkW(wlink, wtarget, symlink_directory_flags)) return PHANTOM_SYMLINK_DIRECTORY; errno = err_win_to_posix(GetLastError()); @@ -2767,7 +2770,7 @@ int symlink(const char *target, const char *link) wtarget[len] = '\\'; /* create file symlink */ - if (!CreateSymbolicLinkW(wlink, wtarget, 0)) { + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { errno = err_win_to_posix(GetLastError()); return -1; } @@ -3503,6 +3506,24 @@ static void maybe_redirect_std_handles(void) GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); } +static void adjust_symlink_flags(void) +{ + /* + * Starting with Windows 10 Build 14972, symbolic links can be created + * using CreateSymbolicLink() without elevation by passing the flag + * SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE (0x02) as last + * parameter, provided the Developer Mode has been enabled. Some + * earlier Windows versions complain about this flag with an + * ERROR_INVALID_PARAMETER, hence we have to test the build number + * specifically. + */ + if (GetVersion() >= 14972 << 16) { + symlink_file_flags |= 2; + symlink_directory_flags |= 2; + } + +} + #ifdef _MSC_VER #ifdef _DEBUG #include @@ -3537,6 +3558,7 @@ int wmain(int argc, const wchar_t **wargv) #endif maybe_redirect_std_handles(); + adjust_symlink_flags(); fsync_object_files = 1; /* determine size of argv and environ conversion buffer */ From e42505f20e8c3f1946c993becd669f0f76414908 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 2 Mar 2020 21:54:29 +0100 Subject: [PATCH 21/22] mingw: emulate stat() a little more faithfully When creating directories via `safe_create_leading_directories()`, we might encounter an already-existing directory which is not readable by the current user. To handle that situation, Git's code calls `stat()` to determine whether we're looking at a directory. In such a case, `CreateFile()` will fail, though, no matter what, and consequently `mingw_stat()` will fail, too. But POSIX semantics seem to still allow `stat()` to go forward. So let's call `mingw_lstat()` for the rescue if we fail to get a file handle due to denied permission in `mingw_stat()`, and fill the stat info that way. We need to be careful to not allow this to go forward in case that we're looking at a symbolic link: to resolve the link, we would still have to create a file handle, and we just found out that we cannot. Therefore, `stat()` still needs to fail with `EACCES` in that case. This fixes https://github.com/git-for-windows/git/issues/2531. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index e17b49728c..83e4a272c8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1029,7 +1029,19 @@ int mingw_stat(const char *file_name, struct stat *buf) FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); if (hnd == INVALID_HANDLE_VALUE) { - errno = err_win_to_posix(GetLastError()); + DWORD err = GetLastError(); + + if (err == ERROR_ACCESS_DENIED && + !mingw_lstat(file_name, buf) && + !S_ISLNK(buf->st_mode)) + /* + * POSIX semantics state to still try to fill + * information, even if permission is denied to create + * a file handle. + */ + return 0; + + errno = err_win_to_posix(err); return -1; } result = get_file_info_by_handle(hnd, buf); From 2f330d7cb17f6e8b81c98b0b23508a8fe415cddb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 4 Jun 2020 23:16:07 +0200 Subject: [PATCH 22/22] mingw: special-case index entries for symlinks with buggy size In https://github.com/git-for-windows/git/pull/2637, we fixed a bug where symbolic links' target path sizes were recorded incorrectly in the index. The downside of this fix was that every user with tracked symbolic links in their checkouts would see them as modified in `git status`, but not in `git diff`, and only a `git add ` (or `git add -u`) would "fix" this. Let's do better than that: we can detect that situation and simply pretend that a symbolic link with a known bad size (or a size that just happens to be that bad size, a _very_ unlikely scenario because it would overflow our buffers due to the trailing NUL byte) means that it needs to be re-checked as if we had just checked it out. Signed-off-by: Johannes Schindelin --- read-cache.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/read-cache.c b/read-cache.c index b21813a647..c5ba8d239f 100644 --- a/read-cache.c +++ b/read-cache.c @@ -453,6 +453,17 @@ int ie_modified(struct index_state *istate, * then we know it is. */ if ((changed & DATA_CHANGED) && +#ifdef GIT_WINDOWS_NATIVE + /* + * Work around Git for Windows v2.27.0 fixing a bug where symlinks' + * target path lengths were not read at all, and instead recorded + * as 4096: now, all symlinks would appear as modified. + * + * So let's just special-case symlinks with a target path length + * (i.e. `sd_size`) of 4096 and force them to be re-checked. + */ + (!S_ISLNK(st->st_mode) || ce->ce_stat_data.sd_size != MAX_LONG_PATH) && +#endif (S_ISGITLINK(ce->ce_mode) || ce->ce_stat_data.sd_size != 0)) return changed;