From 157887c27b0d5aa59cca435416e3fe1be121bb6c Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 6 Jul 2013 02:09:35 +0200 Subject: [PATCH 01/44] Win32: make FILETIME conversion functions public We will use them in the upcoming "FSCache" patches (to accelerate sequential lstat() calls). Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 18 ------------------ compat/mingw.h | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index fae3cb6925..f35e33346d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -651,24 +651,6 @@ int mingw_chmod(const char *filename, int mode) return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) -{ - long long hnsec = filetime_to_hnsec(ft); - ts->tv_sec = (time_t)(hnsec / 10000000); - ts->tv_nsec = (hnsec % 10000000) * 100; -} - /** * Verifies that safe_create_leading_directories() would succeed. */ diff --git a/compat/mingw.h b/compat/mingw.h index 5f52b13698..7bff435f1e 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -345,6 +345,17 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows, * including our own struct stat with 64 bit st_size and nanosecond-precision @@ -361,6 +372,13 @@ struct timespec { #endif #endif +static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) +{ + long long hnsec = filetime_to_hnsec(ft); + ts->tv_sec = (time_t)(hnsec / 10000000); + ts->tv_nsec = (hnsec % 10000000) * 100; +} + struct mingw_stat { _dev_t st_dev; _ino_t st_ino; From 6cce2b2966bb3c891ca70691b9bf5a22df1c3dc9 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:17:31 +0200 Subject: [PATCH 02/44] Win32: dirent.c: Move opendir down Move opendir down in preparation for the next patch. Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 52420ec7d4..2603a0fa39 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -18,40 +18,6 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - struct dirent *readdir(DIR *dir) { if (!dir) { @@ -90,3 +56,37 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *opendir(const char *name) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + DIR *dir; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((len = xutftowcs_path(pattern, name)) < 0) + return NULL; + + /* append optional '/' and wildcard '*' */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '/'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(DIR)); + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return dir; +} From b772d0bd1a29496c95824ccdba4f63d638ce2b9a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:18:40 +0200 Subject: [PATCH 03/44] Win32: Make the dirent implementation pluggable Emulating the POSIX dirent API on Windows via FindFirstFile/FindNextFile is pretty staightforward, however, most of the information provided in the WIN32_FIND_DATA structure is thrown away in the process. A more sophisticated implementation may cache this data, e.g. for later reuse in calls to lstat. Make the dirent implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Define a base DIR structure with pointers to readdir/closedir that match the opendir implementation (i.e. similar to vtable pointers in OOP). Define readdir/closedir so that they call the function pointers in the DIR structure. This allows to choose the opendir implementation on a call-by-call basis. Move the fixed sized dirent.d_name buffer to the dirent-specific DIR structure, as d_name may be implementation specific (e.g. a caching implementation may just set d_name to point into the cache instead of copying the entire file name string). Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 27 +++++++++++++++++---------- compat/win32/dirent.h | 26 +++++++++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 2603a0fa39..6b87042182 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,19 @@ #include "../../git-compat-util.h" -struct DIR { +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ struct dirent dd_dir; /* includes d_type */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + char dd_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ +} dirent_DIR; + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) @@ -18,7 +22,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -45,7 +49,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -57,13 +61,13 @@ int closedir(DIR *dir) return 0; } -DIR *opendir(const char *name) +DIR *dirent_opendir(const char *name) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; HANDLE h; int len; - DIR *dir; + dirent_DIR *dir; /* convert name to UTF-16 and check length < MAX_PATH */ if ((len = xutftowcs_path(pattern, name)) < 0) @@ -84,9 +88,12 @@ DIR *opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); + dir = xmalloc(sizeof(dirent_DIR)); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; + dir->dd_dir.d_name = dir->dd_name; dir->dd_handle = h; dir->dd_stat = 0; finddata2dirent(&dir->dd_dir, &fdata); - return dir; + return (DIR*) dir; } diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bf..6b3ddee51b 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,32 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char *d_name; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ From c9df979aed84f316638c35a7b6f728fb29e24bb7 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:21:30 +0200 Subject: [PATCH 04/44] Win32: make the lstat implementation pluggable Emulating the POSIX lstat API on Windows via GetFileAttributes[Ex] is quite slow. Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. A caching implementation may improve performance by bulk-reading entire directories or reusing data obtained via opendir / readdir. Make the lstat implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 2 ++ compat/mingw.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index f35e33346d..7136131f21 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -790,6 +790,8 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return do_lstat(follow, alt_name, buf); } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) { BY_HANDLE_FILE_INFORMATION fdata; diff --git a/compat/mingw.h b/compat/mingw.h index 7bff435f1e..cd39838fed 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -411,7 +411,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); int mingw_utime(const char *file_name, const struct utimbuf *times); From 2919cb6778d90d973530ea336d563f94ae96561c Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:23:27 +0200 Subject: [PATCH 05/44] add infrastructure for read-only file system level caches Add a macro to mark code sections that only read from the file system, along with a config option and documentation. This facilitates implementation of relatively simple file system level caches without the need to synchronize with the file system. Enable read-only sections for 'git status' and preload_index. Signed-off-by: Karsten Blees --- Documentation/config/core.txt | 6 ++++++ builtin/commit.c | 1 + compat/mingw.c | 6 ++++++ compat/mingw.h | 2 ++ git-compat-util.h | 15 +++++++++++++++ preload-index.c | 2 ++ 6 files changed, 32 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 7e9b6c8f4c..f12ebc8db0 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -551,6 +551,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/builtin/commit.c b/builtin/commit.c index 0c6e97edcd..a24914f5e5 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1371,6 +1371,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(1); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) progress_flag = REFRESH_PROGRESS; diff --git a/compat/mingw.c b/compat/mingw.c index 7136131f21..c7945f8710 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -227,6 +227,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; +int core_fscache; int mingw_core_config(const char *var, const char *value, void *cb) { @@ -238,6 +239,11 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); diff --git a/compat/mingw.h b/compat/mingw.h index cd39838fed..085440f8e2 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -11,6 +11,8 @@ typedef _sigset_t sigset_t; #undef _POSIX_THREAD_SAFE_FUNCTIONS #endif +extern int core_fscache; + extern int mingw_core_config(const char *var, const char *value, void *cb); #define platform_core_config mingw_core_config diff --git a/git-compat-util.h b/git-compat-util.h index 17bf1ebd81..e7d3334cb0 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1271,6 +1271,21 @@ static inline int is_missing_file_error(int errno_) return (errno_ == ENOENT || errno_ == ENOTDIR); } +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + extern int cmd_main(int, const char **); /* diff --git a/preload-index.c b/preload-index.c index e73600ee78..5e8791c43e 100644 --- a/preload-index.c +++ b/preload-index.c @@ -120,6 +120,7 @@ void preload_index(struct index_state *index, pthread_mutex_init(&pd.mutex, NULL); } + enable_fscache(1); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; int err; @@ -145,6 +146,7 @@ void preload_index(struct index_state *index, stop_progress(&pd.progress); trace_performance_leave("preload index"); + enable_fscache(0); } int repo_read_index_preload(struct repository *repo, From f41d747b80b84751eb18475a11c1bb3ac07899c4 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 1 Oct 2013 12:51:54 +0200 Subject: [PATCH 06/44] Win32: add a cache below mingw's lstat and dirent implementations Checking the work tree status is quite slow on Windows, due to slow lstat emulation (git calls lstat once for each file in the index). Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. Add an lstat implementation that uses a cache for lstat data. Cache misses read the entire parent directory and add it to the cache. Subsequent lstat calls for the same directory are served directly from the cache. Also implement opendir / readdir / closedir so that they create and use directory listings in the cache. The cache doesn't track file system changes and doesn't plug into any modifying file APIs, so it has to be explicitly enabled for git functions that don't modify the working copy. Note: in an earlier version of this patch, the cache was always active and tracked file system changes via ReadDirectoryChangesW. However, this was much more complex and had negative impact on the performance of modifying git commands such as 'git checkout'. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 444 +++++++++++++++++++++++++++++++++++++++++ compat/win32/fscache.h | 10 + config.mak.uname | 4 +- git-compat-util.h | 2 + 4 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 compat/win32/fscache.c create mode 100644 compat/win32/fscache.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 0000000000..36d03b1f17 --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,444 @@ +#include "../../cache.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" + +static int initialized; +static volatile long enabled; +static struct hashmap map; +static CRITICAL_SECTION mutex; + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + const char *name; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + struct timespec st_atim; + struct timespec st_mtim; + struct timespec st_ctim; + } s; + } u; +}; + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(void *unused_cmp_data, + const struct fsentry *fse1, const struct fsentry *fse2, + void *unused_keydata) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && + (res = fsentry_cmp(NULL, fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2, NULL))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return strnicmp(fse1->name, fse2->name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + fse->name = name; + fse->len = len; + hashmap_entry_init(fse, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + char *nm = ((char*) fse) + sizeof(struct fsentry); + memcpy(nm, name, len); + nm[len] = 0; + /* init the rest of the structure */ + fsentry_init(fse, list, nm, len); + fse->next = NULL; + fse->u.refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->u.refcnt)); +} + +/* + * Release the reference to an fsentry, frees the memory if its the last ref. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + if (InterlockedDecrement(&(fse->u.refcnt))) + return; + + while (fse) { + struct fsentry *next = fse->next; + free(fse); + fse = next; + } +} + +/* + * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + */ +static struct fsentry *fseentry_create_entry(struct fsentry *list, + const WIN32_FIND_DATAW *fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + fse = fsentry_alloc(list, buf, len); + + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->u.s.st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) + | fdata->nFileSizeLow; + filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->u.s.st_atim)); + filetime_to_timespec(&(fdata->ftLastWriteTime), &(fse->u.s.st_mtim)); + filetime_to_timespec(&(fdata->ftCreationTime), &(fse->u.s.st_ctim)); + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(const struct fsentry *dir) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { + if (errno == ERANGE) + errno = ENAMETOOLONG; + return NULL; + } + + /* append optional '/' and wildcard '*' */ + if (wlen) + pattern[wlen++] = '/'; + pattern[wlen++] = '*'; + pattern[wlen] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(NULL, dir->name, dir->len); + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + do { + *phead = fseentry_create_entry(list, &fdata); + phead = &(*phead)->next; + } while (FindNextFileW(h, &fdata)); + + /* remember result of last FindNextFile, then close find handle */ + err = GetLastError(); + FindClose(h); + + /* return the list if we've got all the files */ + if (err == ERROR_NO_MORE_FILES) + return list; + + /* otherwise free the list and return error */ + fsentry_release(list); + errno = err_win_to_posix(err); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&map, fse); +} + +/* + * Clears the cache. + */ +static void fscache_clear(void) +{ + hashmap_free(&map, 1); + hashmap_init(&map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); +} + +/* + * Checks if the cache is enabled for the given path. + */ +static inline int fscache_enabled(const char *path) +{ + return enabled > 0 && !is_absolute_path(path); +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fsentry *key) +{ + struct fsentry *fse; + + EnterCriticalSection(&mutex); + /* check if entry is in cache */ + fse = hashmap_get(&map, key, NULL); + if (fse) { + fsentry_addref(fse); + LeaveCriticalSection(&mutex); + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get(&map, key->list, NULL); + if (fse) { + LeaveCriticalSection(&mutex); + /* dir entry without file entry -> file doesn't exist */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing (outside mutex!) */ + LeaveCriticalSection(&mutex); + fse = fsentry_create_list(key->list ? key->list : key); + if (!fse) + return NULL; + + EnterCriticalSection(&mutex); + /* add directory listing if it hasn't been added by some other thread */ + if (!hashmap_get(&map, key, NULL)) + fscache_add(fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get(&map, key, NULL); + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + LeaveCriticalSection(&mutex); + return fse; +} + +/* + * Enables or disables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(int enable) +{ + int result; + + if (!initialized) { + /* allow the cache to be disabled entirely */ + if (!core_fscache) + return 0; + + InitializeCriticalSection(&mutex); + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, 0); + initialized = 1; + } + + result = enable ? InterlockedIncrement(&enabled) + : InterlockedDecrement(&enabled); + + if (enable && result == 1) { + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } else if (!enable && !result) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } + return result; +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; + struct fsentry key[2], *fse; + + if (!fscache_enabled(filename)) + return mingw_lstat(filename, st); + + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(key, NULL, filename, dirlen); + fsentry_init(key + 1, key, filename + base, len - base); + fse = fscache_get(key + 1); + if (!fse) + return -1; + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->u.s.st_size; + st->st_atim = fse->u.s.st_atim; + st->st_mtim = fse->u.s.st_mtim; + st->st_ctim = fse->u.s.st_ctim; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_name = (char*) next->name; + return &(dir->dirent); +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct fsentry key, *list; + fscache_DIR *dir; + int len; + + if (!fscache_enabled(dirname)) + return dirent_opendir(dirname); + + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key, NULL, dirname, len); + list = fscache_get(&key); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 0000000000..ed518b422d --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,10 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +int fscache_enable(int enable); +#define enable_fscache(x) fscache_enable(x) + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); + +#endif diff --git a/config.mak.uname b/config.mak.uname index a5bbb1b078..9caea922af 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -430,7 +430,7 @@ ifeq ($(uname_S),Windows) COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -ENTRY:wmainCRTStartup -SUBSYSTEM:CONSOLE # invalidcontinue.obj allows Git's source code to close the same file @@ -609,7 +609,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DWIN32 -DPROTECT_NTFS_DEFAULT=1 EXTLIBS += -lws2_32 GITLIBS += git.res diff --git a/git-compat-util.h b/git-compat-util.h index e7d3334cb0..49c76f3eca 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -208,9 +208,11 @@ /* pull in Windows compatibility stuff */ #include "compat/win32/path-utils.h" #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/win32/path-utils.h" #include "compat/msvc.h" +#include "compat/win32/fscache.h" #else #include #include From 942ef0de47870460a7a1a938f2d106c7fcfc63be Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 24 Jun 2014 13:22:35 +0200 Subject: [PATCH 07/44] fscache: load directories only once If multiple threads access a directory that is not yet in the cache, the directory will be loaded by each thread. Only one of the results is added to the cache, all others are leaked. This wastes performance and memory. On cache miss, add a future object to the cache to indicate that the directory is currently being loaded. Subsequent threads register themselves with the future object and wait. When the first thread has loaded the directory, it replaces the future object with the result and notifies waiting threads. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 67 +++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 36d03b1f17..b3b6e3146c 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -33,6 +33,8 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; + /* Handle to wait on the loading thread. */ + HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -245,16 +247,43 @@ static inline int fscache_enabled(const char *path) return enabled > 0 && !is_absolute_path(path); } +/* + * Looks up a cache entry, waits if its being loaded by another thread. + * The mutex must be owned by the calling thread. + */ +static struct fsentry *fscache_get_wait(struct fsentry *key) +{ + struct fsentry *fse = hashmap_get(&map, key, NULL); + + /* return if its a 'real' entry (future entries have refcnt == 0) */ + if (!fse || fse->list || fse->u.refcnt) + return fse; + + /* create an event and link our key to the future entry */ + key->u.hwait = CreateEvent(NULL, TRUE, FALSE, NULL); + key->next = fse->next; + fse->next = key; + + /* wait for the loading thread to signal us */ + LeaveCriticalSection(&mutex); + WaitForSingleObject(key->u.hwait, INFINITE); + CloseHandle(key->u.hwait); + EnterCriticalSection(&mutex); + + /* repeat cache lookup */ + return hashmap_get(&map, key, NULL); +} + /* * Looks up or creates a cache entry for the specified key. */ static struct fsentry *fscache_get(struct fsentry *key) { - struct fsentry *fse; + struct fsentry *fse, *future, *waiter; EnterCriticalSection(&mutex); /* check if entry is in cache */ - fse = hashmap_get(&map, key, NULL); + fse = fscache_get_wait(key); if (fse) { fsentry_addref(fse); LeaveCriticalSection(&mutex); @@ -262,7 +291,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = hashmap_get(&map, key->list, NULL); + fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); /* dir entry without file entry -> file doesn't exist */ @@ -271,16 +300,34 @@ static struct fsentry *fscache_get(struct fsentry *key) } } + /* add future entry to indicate that we're loading it */ + future = key->list ? key->list : key; + future->next = NULL; + future->u.refcnt = 0; + hashmap_add(&map, future); + /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(key->list ? key->list : key); - if (!fse) - return NULL; - + fse = fsentry_create_list(future); EnterCriticalSection(&mutex); - /* add directory listing if it hasn't been added by some other thread */ - if (!hashmap_get(&map, key, NULL)) - fscache_add(fse); + + /* remove future entry and signal waiting threads */ + hashmap_remove(&map, future, NULL); + waiter = future->next; + while (waiter) { + HANDLE h = waiter->u.hwait; + waiter = waiter->next; + SetEvent(h); + } + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + LeaveCriticalSection(&mutex); + return NULL; + } + + /* add directory listing to the cache */ + fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) From b27332257bea721fb83f54fe10efadb603fb28f5 Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 08/44] pack-objects (mingw): demonstrate a segmentation fault with large deltas There is a problem in the way 9ac3f0e5b3e4 (pack-objects: fix performance issues on packing large deltas, 2018-07-22) initializes that mutex in the `packing_data` struct. The problem manifests in a segmentation fault on Windows, when a mutex (AKA critical section) is accessed without being initialized. (With pthreads, you apparently do not really have to initialize them?) This was reported in https://github.com/git-for-windows/git/issues/1839. Signed-off-by: Johannes Schindelin --- t/t7419-submodule-long-path.sh | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 t/t7419-submodule-long-path.sh diff --git a/t/t7419-submodule-long-path.sh b/t/t7419-submodule-long-path.sh new file mode 100755 index 0000000000..9f9d2ea446 --- /dev/null +++ b/t/t7419-submodule-long-path.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +longpath=${longpath:0:90}; export longpath + +test_expect_failure 'submodule with a long path' ' + git init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin master + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout master && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + git init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git submodule add ../parent foo && + git submodule update --init --recursive && + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin master + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From 749f5100f571650eb5d6b92f96d253e3949b49ff Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 28 Jul 2015 21:07:41 +0200 Subject: [PATCH 09/44] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts, added support for chdir(), etc] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- Documentation/config/core.txt | 7 ++ compat/mingw.c | 144 ++++++++++++++++++++++++++------- compat/mingw.h | 75 +++++++++++++++-- compat/win32/dirent.c | 14 ++-- compat/win32/fscache.c | 17 ++-- t/t2031-checkout-long-paths.sh | 102 +++++++++++++++++++++++ t/t7419-submodule-long-path.sh | 24 +++--- 7 files changed, 326 insertions(+), 57 deletions(-) create mode 100755 t/t2031-checkout-long-paths.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index f12ebc8db0..71dce85e76 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -557,6 +557,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/compat/mingw.c b/compat/mingw.c index c7945f8710..66d6d36536 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -228,6 +228,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; int core_fscache; +int core_long_paths; int mingw_core_config(const char *var, const char *value, void *cb) { @@ -244,6 +245,11 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); @@ -281,8 +287,8 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; /* read-only files cannot be removed */ @@ -311,7 +317,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -332,8 +338,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -408,9 +414,12 @@ static int set_hidden_flag(const wchar_t *path, int set) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; - if (xutftowcs_path(wpath, path) < 0) + wchar_t wpath[MAX_LONG_PATH]; + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); @@ -483,7 +492,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; open_fn_t open_fn; va_start(args, oflags); @@ -498,7 +507,7 @@ int mingw_open (const char *filename, int oflags, ...) else open_fn = _wopen; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = open_fn(wfilename, oflags, mode); @@ -555,10 +564,10 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -577,10 +586,10 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -634,25 +643,31 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { - wchar_t wdirname[MAX_PATH]; - if (xutftowcs_path(wdirname, dirname) < 0) + int result; + wchar_t wdirname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -700,8 +715,8 @@ static int has_valid_directory_prefix(wchar_t *wfilename) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -872,8 +887,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -934,6 +949,7 @@ char *mingw_mktemp(char *template) wchar_t wtemplate[MAX_PATH]; int offset = 0; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; @@ -1457,6 +1473,10 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen if (*argv && !strcmp(cmd, *argv)) wcmd[0] = L'\0'; + /* + * Paths to executables and to the current directory do not support + * long paths, therefore we cannot use xutftowcs_long_path() here. + */ else if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -1855,8 +1875,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -2165,9 +2186,9 @@ int mingw_raise(int sig) int link(const char *oldpath, const char *newpath) { - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!CreateHardLinkW(wnewpath, woldpath, NULL)) { @@ -2367,6 +2388,68 @@ static void setup_windows_environment(void) } } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + #if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from @@ -2523,6 +2606,9 @@ int wmain(int argc, const wchar_t **wargv) /* initialize Unicode console */ winansi_init(); + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + /* invoke the real main() using our utf8 version of argv. */ exit_status = main(argc, argv); diff --git a/compat/mingw.h b/compat/mingw.h index 085440f8e2..ec24214464 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -12,6 +12,7 @@ typedef _sigset_t sigset_t; #endif extern int core_fscache; +extern int core_long_paths; extern int mingw_core_config(const char *var, const char *value, void *cb); #define platform_core_config mingw_core_config @@ -475,6 +476,42 @@ extern const char *program_data_config(void); #include #endif +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -532,17 +569,45 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) return xutftowcsn(wcs, utf, wcslen, -1); } +/** + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). + */ +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) +{ + int result = xutftowcsn(wcs, utf, wcslen, utflen); + if (result < 0 && errno == ERANGE) + errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); + return result; +} + /** * Simplified file system specific variant of xutftowcsn, assumes output * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... */ static inline int xutftowcs_path(wchar_t *wcs, const char *utf) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); - if (result < 0 && errno == ERANGE) - errno = ENAMETOOLONG; - return result; + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); } /** diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 6b87042182..b3bd8d7af7 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -63,19 +63,23 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, core_long_paths)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index b3b6e3146c..a015d4d8cb 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -166,23 +166,24 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, */ static struct fsentry *fsentry_create_list(const struct fsentry *dir) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int wlen; struct fsentry *list, **phead; DWORD err; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, + dir->len, MAX_PATH - 2, core_long_paths)) < 0) return NULL; - } - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (wlen) - pattern[wlen++] = '/'; + pattern[wlen++] = '\\'; pattern[wlen++] = '*'; pattern[wlen] = 0; diff --git a/t/t2031-checkout-long-paths.sh b/t/t2031-checkout-long-paths.sh new file mode 100755 index 0000000000..f30f8920ca --- /dev/null +++ b/t/t2031-checkout-long-paths.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test ! -d longpa* +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa*/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >>$(ls longpa*/longtestfile) && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + if test -z "$debug" + then + rm -rf longpa~1 + fi +' + +# check that the template used in the test won't be too long: +abspath="$(pwd)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7419-submodule-long-path.sh b/t/t7419-submodule-long-path.sh index 9f9d2ea446..2ca9794ca5 100755 --- a/t/t7419-submodule-long-path.sh +++ b/t/t7419-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -longpath=${longpath:0:90}; export longpath +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' git init --bare remote && test_create_repo bundle1 && ( @@ -56,7 +61,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' git init --bare super && test_create_repo child && ( @@ -96,6 +101,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From 012db54cea2817c871ca107ac9e583f2a77610b9 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 5 Jul 2014 00:00:36 +0200 Subject: [PATCH 10/44] Win32: fix 'lstat("dir/")' with long paths Use a suffciently large buffer to strip the trailing slash. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 66d6d36536..ba76ac3ff2 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -787,7 +787,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) static int do_stat_internal(int follow, const char *file_name, struct stat *buf) { int namelen; - char alt_name[PATH_MAX]; + char alt_name[MAX_LONG_PATH]; if (!do_lstat(follow, file_name, buf)) return 0; @@ -803,7 +803,7 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return -1; while (namelen && file_name[namelen-1] == '/') --namelen; - if (!namelen || namelen >= PATH_MAX) + if (!namelen || namelen >= MAX_LONG_PATH) return -1; memcpy(alt_name, file_name, namelen); From 2ea8987f48b084b838dddb91cc00537dbb5818f8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 09:52:07 +0000 Subject: [PATCH 11/44] Help debugging with MSys2 by optionally executing bash with strace MSys2's strace facility is very useful for debugging... With this patch, the bash will be executed through strace if the environment variable GIT_STRACE_COMMANDS is set, which comes in real handy when investigating issues in the test suite. Also support passing a path to a log file via GIT_STRACE_COMMANDS to force Git to call strace.exe with the `-o ` argument, i.e. to log into a file rather than print the log directly. That comes in handy when the output would otherwise misinterpreted by a calling process as part of Git's output. Note: the values "1", "yes" or "true" are *not* specifying paths, but tell Git to let strace.exe log directly to the console. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index ba76ac3ff2..772ea064d2 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1437,6 +1437,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen HANDLE cons; const char *(*quote_arg)(const char *arg) = is_msys2_sh(*argv) ? quote_arg_msys2 : quote_arg_msvc; + const char *strace_env; do_unset_environment_variables(); @@ -1499,6 +1500,31 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen free(quoted); } + strace_env = getenv("GIT_STRACE_COMMANDS"); + if (strace_env) { + char *p = path_lookup("strace.exe", 1); + if (!p) + return error("strace not found!"); + if (xutftowcs_path(wcmd, p) < 0) { + free(p); + return -1; + } + free(p); + if (!strcmp("1", strace_env) || + !strcasecmp("yes", strace_env) || + !strcasecmp("true", strace_env)) + strbuf_insert(&args, 0, "strace ", 7); + else { + const char *quoted = quote_arg(strace_env); + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "strace -o %s ", quoted); + if (quoted != strace_env) + free((char *)quoted); + strbuf_insert(&args, 0, buf.buf, buf.len); + strbuf_release(&buf); + } + } + ALLOC_ARRAY(wargs, st_add(st_mult(2, args.len), 1)); xutftowcs(wargs, args.buf, 2 * args.len + 1); strbuf_release(&args); From 5f67d55da0f4ad5a5ff28da3f18efda1cdd0da37 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 16 Feb 2015 14:06:59 +0100 Subject: [PATCH 12/44] Build Python stuff with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 9caea922af..c875ff1218 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -664,6 +664,7 @@ else NO_LIBPCRE1_JIT = UnfortunatelyYes NO_CURL = USE_NED_ALLOCATOR = YesPlease + NO_PYTHON = else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease From ed1c7218d1a2eea8a37945f0c014fde83652cf56 Mon Sep 17 00:00:00 2001 From: nalla Date: Thu, 16 Apr 2015 11:45:05 +0100 Subject: [PATCH 13/44] mingw: explicitly `fflush` stdout For performance reasons `stdout` is not unbuffered by default. That leads to problems if after printing to `stdout` a read on `stdin` is performed. For that reason interactive commands like `git clean -i` do not function properly anymore if the `stdout` is not flushed by `fflush(stdout)` before trying to read from `stdin`. In the case of `git clean -i` all reads on `stdin` were preceded by a `fflush(stdout)` call. Signed-off-by: nalla --- builtin/clean.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index 7be689f480..35437dfcc0 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -579,6 +579,7 @@ static int *list_and_choose(struct menu_opts *opts, struct menu_stuff *stuff) clean_get_color(CLEAN_COLOR_RESET)); } + fflush(stdout); if (strbuf_getline_lf(&choice, stdin) != EOF) { strbuf_trim(&choice); } else { @@ -661,6 +662,7 @@ static int filter_by_patterns_cmd(void) clean_print_color(CLEAN_COLOR_PROMPT); printf(_("Input ignore patterns>> ")); clean_print_color(CLEAN_COLOR_RESET); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) strbuf_trim(&confirm); else @@ -759,6 +761,7 @@ static int ask_each_cmd(void) qname = quote_path_relative(item->string, NULL, &buf); /* TRANSLATORS: Make sure to keep [y/N] as is */ printf(_("Remove %s [y/N]? "), qname); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) { strbuf_trim(&confirm); } else { From c9f80b6649bd816074f58e275010b507c8a57c77 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 19 Mar 2015 16:33:44 +0100 Subject: [PATCH 14/44] mingw: Support `git_terminal_prompt` with more terminals The `git_terminal_prompt()` function expects the terminal window to be attached to a Win32 Console. However, this is not the case with terminal windows other than `cmd.exe`'s, e.g. with MSys2's own `mintty`. Non-cmd terminals such as `mintty` still have to have a Win32 Console to be proper console programs, but have to hide the Win32 Console to be able to provide more flexibility (such as being resizeable not only vertically but also horizontally). By writing to that Win32 Console, `git_terminal_prompt()` manages only to send the prompt to nowhere and to wait for input from a Console to which the user has no access. This commit introduces a function specifically to support `mintty` -- or other terminals that are compatible with MSys2's `/dev/tty` emulation. We use the `TERM` environment variable as an indicator for that: if the value starts with "xterm" (such as `mintty`'s "xterm_256color"), we prefer to let `xterm_prompt()` handle the user interaction. The most prominent user of `git_terminal_prompt()` is certainly `git-remote-https.exe`. It is an interesting use case because both `stdin` and `stdout` are redirected when Git calls said executable, yet it still wants to access the terminal. When running inside a `mintty`, the terminal is not accessible to the `git-remote-https.exe` program, though, because it is a MinGW program and the `mintty` terminal is not backed by a Win32 console. To solve that problem, we simply call out to the shell -- which is an *MSys2* program and can therefore access `/dev/tty`. Helped-by: nalla Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/terminal.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/compat/terminal.c b/compat/terminal.c index cb08d7a7fc..ae9b5fb90d 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -1,4 +1,4 @@ -#include "git-compat-util.h" +#include "cache.h" #include "compat/terminal.h" #include "sigchain.h" #include "strbuf.h" @@ -194,6 +194,54 @@ static int mingw_getchar(void) } #define getchar mingw_getchar +static char *shell_prompt(const char *prompt, int echo) +{ + const char *read_input[] = { + /* Note: call 'bash' explicitly, as 'read -s' is bash-specific */ + "bash", "-c", echo ? + "cat >/dev/tty && read -r line /dev/tty && read -r -s line /dev/tty", + NULL + }; + struct child_process child = CHILD_PROCESS_INIT; + static struct strbuf buffer = STRBUF_INIT; + int prompt_len = strlen(prompt), len = -1, code; + + child.argv = read_input; + child.in = -1; + child.out = -1; + + if (start_command(&child)) + return NULL; + + if (write_in_full(child.in, prompt, prompt_len) != prompt_len) { + error("could not write to prompt script"); + close(child.in); + goto ret; + } + close(child.in); + + strbuf_reset(&buffer); + len = strbuf_read(&buffer, child.out, 1024); + if (len < 0) { + error("could not read from prompt script"); + goto ret; + } + + strbuf_strip_suffix(&buffer, "\n"); + strbuf_strip_suffix(&buffer, "\r"); + +ret: + close(child.out); + code = finish_command(&child); + if (code) { + error("failed to execute prompt script (exit code %d)", code); + return NULL; + } + + return len < 0 ? NULL : buffer.buf; +} + #endif #ifndef FORCE_TEXT @@ -205,6 +253,12 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; +#ifdef GIT_WINDOWS_NATIVE + const char *term = getenv("TERM"); + + if (term && starts_with(term, "xterm")) + return shell_prompt(prompt, echo); +#endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); if (!input_fh) From a4cc521c90078b2cf3e17e70bb9d443c64b80610 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 9 May 2015 02:11:48 +0200 Subject: [PATCH 15/44] compat/terminal.c: only use the Windows console if bash 'read -r' fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accessing the Windows console through the special CONIN$ / CONOUT$ devices doesn't work properly for non-ASCII usernames an passwords. It also doesn't work for terminal emulators that hide the native console window (such as mintty), and 'TERM=xterm*' is not necessarily a reliable indicator for such terminals. The new shell_prompt() function, on the other hand, works fine for both MSys1 and MSys2, in native console windows as well as mintty, and properly supports Unicode. It just needs bash on the path (for 'read -s', which is bash-specific). On Windows, try to use the shell to read from the terminal. If that fails with ENOENT (i.e. bash was not found), use CONIN/OUT as fallback. Note: To test this, create a UTF-8 credential file with non-ASCII chars, e.g. in git-bash: 'echo url=http://täst.com > cred.txt'. Then in git-cmd, 'git credential fill --- compat/terminal.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/compat/terminal.c b/compat/terminal.c index ae9b5fb90d..4fe0c1a960 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -210,6 +210,7 @@ static char *shell_prompt(const char *prompt, int echo) child.argv = read_input; child.in = -1; child.out = -1; + child.silent_exec_failure = 1; if (start_command(&child)) return NULL; @@ -253,11 +254,14 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; -#ifdef GIT_WINDOWS_NATIVE - const char *term = getenv("TERM"); - if (term && starts_with(term, "xterm")) - return shell_prompt(prompt, echo); +#ifdef GIT_WINDOWS_NATIVE + + /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ + char *result = shell_prompt(prompt, echo); + if (result || errno != ENOENT) + return result; + #endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); From 81c5a308f57a115ab20fd0e7a28a92bb479948cc Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:14:20 +0100 Subject: [PATCH 16/44] winansi: simplify loading the GetCurrentConsoleFontEx() function We introduced helper macros to simplify loading functions dynamically. Might just as well use them. Signed-off-by: Johannes Schindelin --- compat/winansi.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/compat/winansi.c b/compat/winansi.c index 05c3ee7de5..1df8722c9c 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -7,6 +7,7 @@ #include #include #include "win32.h" +#include "win32/lazyload.h" static int fd_is_interactive[3] = { 0, 0, 0 }; #define FD_CONSOLE 0x1 @@ -41,26 +42,21 @@ typedef struct _CONSOLE_FONT_INFOEX { #endif #endif -typedef BOOL (WINAPI *PGETCURRENTCONSOLEFONTEX)(HANDLE, BOOL, - PCONSOLE_FONT_INFOEX); - static void warn_if_raster_font(void) { DWORD fontFamily = 0; - PGETCURRENTCONSOLEFONTEX pGetCurrentConsoleFontEx; + DECLARE_PROC_ADDR(kernel32.dll, BOOL, GetCurrentConsoleFontEx, + HANDLE, BOOL, PCONSOLE_FONT_INFOEX); /* don't bother if output was ascii only */ if (!non_ascii_used) return; /* GetCurrentConsoleFontEx is available since Vista */ - pGetCurrentConsoleFontEx = (PGETCURRENTCONSOLEFONTEX) GetProcAddress( - GetModuleHandle("kernel32.dll"), - "GetCurrentConsoleFontEx"); - if (pGetCurrentConsoleFontEx) { + if (INIT_PROC_ADDR(GetCurrentConsoleFontEx)) { CONSOLE_FONT_INFOEX cfi; cfi.cbSize = sizeof(cfi); - if (pGetCurrentConsoleFontEx(console, 0, &cfi)) + if (GetCurrentConsoleFontEx(console, 0, &cfi)) fontFamily = cfi.FontFamily; } else { /* pre-Vista: check default console font in registry */ From 7c7e0a70594f6af2a9136a8a7686f8aea57e7a36 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 6 Sep 2016 09:50:33 +0200 Subject: [PATCH 17/44] Unbreak interactive GPG prompt upon signing With the recent update in efee955 (gpg-interface: check gpg signature creation status, 2016-06-17), we ask GPG to send all status updates to stderr, and then catch the stderr in an strbuf. But GPG might fail, and send error messages to stderr. And we simply do not show them to the user. Even worse: this swallows any interactive prompt for a passphrase. And detaches stderr from the tty so that the passphrase cannot be read. So while the first problem could be fixed (by printing the captured stderr upon error), the second problem cannot be easily fixed, and presents a major regression. So let's just revert commit efee9553a4f97b2ecd8f49be19606dd4cf7d9c28. This fixes https://github.com/git-for-windows/git/issues/871 Cc: Michael J Gruber Signed-off-by: Johannes Schindelin --- gpg-interface.c | 8 ++------ t/t7004-tag.sh | 13 ------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/gpg-interface.c b/gpg-interface.c index 8ed274533f..24348691f8 100644 --- a/gpg-interface.c +++ b/gpg-interface.c @@ -293,11 +293,9 @@ int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *sig struct child_process gpg = CHILD_PROCESS_INIT; int ret; size_t i, j, bottom; - struct strbuf gpg_status = STRBUF_INIT; argv_array_pushl(&gpg.args, use_format->program, - "--status-fd=2", "-bsau", signing_key, NULL); @@ -309,12 +307,10 @@ int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *sig */ sigchain_push(SIGPIPE, SIG_IGN); ret = pipe_command(&gpg, buffer->buf, buffer->len, - signature, 1024, &gpg_status, 0); + signature, 1024, NULL, 0); sigchain_pop(SIGPIPE); - ret |= !strstr(gpg_status.buf, "\n[GNUPG:] SIG_CREATED "); - strbuf_release(&gpg_status); - if (ret) + if (ret || signature->len == bottom) return error(_("gpg failed to sign the data")); /* Strip CR from the line endings, in case we are on Windows. */ diff --git a/t/t7004-tag.sh b/t/t7004-tag.sh index 0b01862c23..a05df0d7b6 100755 --- a/t/t7004-tag.sh +++ b/t/t7004-tag.sh @@ -1345,12 +1345,6 @@ test_expect_success GPG \ 'test_config user.signingkey BobTheMouse && test_must_fail git tag -s -m tail tag-gpg-failure' -# try to produce invalid signature -test_expect_success GPG \ - 'git tag -s fails if gpg is misconfigured (bad signature format)' \ - 'test_config gpg.program echo && - test_must_fail git tag -s -m tail tag-gpg-failure' - # try to sign with bad user.signingkey test_expect_success GPGSM \ 'git tag -s fails if gpgsm is misconfigured (bad key)' \ @@ -1358,13 +1352,6 @@ test_expect_success GPGSM \ test_config gpg.format x509 && test_must_fail git tag -s -m tail tag-gpg-failure' -# try to produce invalid signature -test_expect_success GPGSM \ - 'git tag -s fails if gpgsm is misconfigured (bad signature format)' \ - 'test_config gpg.x509.program echo && - test_config gpg.format x509 && - test_must_fail git tag -s -m tail tag-gpg-failure' - # try to verify without gpg: rm -rf gpghome From 47689ca1f79ffb0b18810a09c81c43c193a9c156 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 23 Feb 2015 15:55:47 +0000 Subject: [PATCH 18/44] mingw: disable t9020 POSIX-to-Windows path mangling would make it fail. Symptoms: ++ init_git ++ rm -fr .git ++ git init Initialized empty Git repository in [...] ++ git remote add svnsim testsvn::sim:///usr/src/git/wip5/t/t9154/svn.dump ++ git remote add svnfile testsvn::file:///usr/src/git/wip5/t/t9154/svn.dump ++ git fetch svnsim progress Imported commit 1. fatal: Write to frontend failed: Bad file descriptor fast-import: dumping crash report to .git/fast_import_crash_23356 fatal: error while running fast-import fatal: unexpected end of fast-import feedback error: last command exited with $?=128 not ok 1 - simple fetch Since the remote-svn project seems to be dormant at the moment (and not complete enough to be used, which is a pity), let's just skip this test on Windows. Signed-off-by: Johannes Schindelin --- t/t9020-remote-svn.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/t/t9020-remote-svn.sh b/t/t9020-remote-svn.sh index 6fca08e5e3..76d9be2e1d 100755 --- a/t/t9020-remote-svn.sh +++ b/t/t9020-remote-svn.sh @@ -12,6 +12,12 @@ then test_done fi +if test_have_prereq MINGW +then + skip_all='skipping remote-svn tests for lack of POSIX' + test_done +fi + # Override svnrdump with our simulator PATH="$HOME:$PATH" export PATH PYTHON_PATH GIT_BUILD_DIR From bea148bb4d46b27141a2d470fcc59f12237c2635 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 Feb 2017 13:28:58 +0100 Subject: [PATCH 19/44] mingw: ensure valid CTYPE A change between versions 2.4.1 and 2.6.0 of the MSYS2 runtime modified how Cygwin's runtime (and hence Git for Windows' MSYS2 runtime derivative) handles locales: d16a56306d (Consolidate wctomb/mbtowc calls for POSIX-1.2008, 2016-07-20). An unintended side-effect is that "cold-calling" into the POSIX emulation will start with a locale based on the current code page, something that Git for Windows is very ill-prepared for, as it expects to be able to pass a command-line containing non-ASCII characters to the shell without having those characters munged. One symptom of this behavior: when `git clone` or `git fetch` shell out to call `git-upload-pack` with a path that contains non-ASCII characters, the shell tried to interpret the entire command-line (including command-line parameters) as executable path, which obviously must fail. This fixes https://github.com/git-for-windows/git/issues/1036 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 772ea064d2..0c2b30fe3c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2412,6 +2412,9 @@ static void setup_windows_environment(void) if (!tmp && (tmp = getenv("USERPROFILE"))) setenv("HOME", tmp, 1); } + + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) + setenv("LC_CTYPE", "C", 1); } int handle_long_path(wchar_t *path, int len, int max_path, int expand) From 1c07f1b9d5d751ae49f3665bb881d9fd2872de32 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 11 Jan 2017 21:08:15 +0100 Subject: [PATCH 20/44] t9001, t9116: avoid pipes When grepping through the output of a command in the test suite, there is always a chance that something goes wrong, in which case there would not be anything useful to debug. Let's redirect the output into a file instead, and grep that file, so that the log can be inspected easily if the grep fails. Signed-off-by: Johannes Schindelin --- t/t9001-send-email.sh | 4 ++-- t/t9116-git-svn-log.sh | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index ee1efcc59d..748e263169 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -1180,8 +1180,8 @@ test_expect_success $PREREQ 'in-reply-to but no threading' ' --to=nobody@example.com \ --in-reply-to="" \ --no-thread \ - $patches | - grep "In-Reply-To: " + $patches >out && + grep "In-Reply-To: " out ' test_expect_success $PREREQ 'no in-reply-to and no threading' ' diff --git a/t/t9116-git-svn-log.sh b/t/t9116-git-svn-log.sh index 45773ee560..0a9f1ef366 100755 --- a/t/t9116-git-svn-log.sh +++ b/t/t9116-git-svn-log.sh @@ -43,14 +43,18 @@ test_expect_success 'setup repository and import' ' test_expect_success 'run log' " git reset --hard origin/a && - git svn log -r2 origin/trunk | grep ^r2 && - git svn log -r4 origin/trunk | grep ^r4 && - git svn log -r3 | grep ^r3 + git svn log -r2 origin/trunk >out && + grep ^r2 out && + git svn log -r4 origin/trunk >out && + grep ^r4 out && + git svn log -r3 >out && + grep ^r3 out " test_expect_success 'run log against a from trunk' " git reset --hard origin/trunk && - git svn log -r3 origin/a | grep ^r3 + git svn log -r3 origin/a >out && + grep ^r3 out " printf 'r1 \nr2 \nr4 \n' > expected-range-r1-r2-r4 From f2fc56998bbc717a1fa15adff0289da701b454d4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 Feb 2017 20:34:38 +0100 Subject: [PATCH 21/44] mingw: make is_hidden tests in t0001/t5611 more robust We should not actually expect the first `attrib.exe` in the PATH to be the one we are looking for. Or that it is in the PATH, for that matter. Signed-off-by: Johannes Schindelin --- t/t0001-init.sh | 2 +- t/t5611-clone-config.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 4d04e6a863..a6c5fe9e14 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -412,7 +412,7 @@ test_expect_success SYMLINKS 're-init to move gitdir symlink' ' # Tests for the hidden file attribute on windows is_hidden () { # Use the output of `attrib`, ignore the absolute path - case "$(attrib "$1")" in *H*?:*) return 0;; esac + case "$("$SYSTEMROOT"/system32/attrib "$1")" in *H*?:*) return 0;; esac return 1 } diff --git a/t/t5611-clone-config.sh b/t/t5611-clone-config.sh index 60c1ba951b..87b8073cd7 100755 --- a/t/t5611-clone-config.sh +++ b/t/t5611-clone-config.sh @@ -95,7 +95,7 @@ test_expect_success 'clone -c remote..fetch= --origin=' ' # Tests for the hidden file attribute on windows is_hidden () { # Use the output of `attrib`, ignore the absolute path - case "$(attrib "$1")" in *H*?:*) return 0;; esac + case "$("$SYSTEMROOT"/system32/attrib "$1")" in *H*?:*) return 0;; esac return 1 } From dc0cc6a0cb0ff6c6ac3ca8bc4f32b4b15908ab81 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 19:54:23 +0200 Subject: [PATCH 22/44] strbuf_readlink: don't call readlink twice if hint is the exact link size strbuf_readlink() calls readlink() twice if the hint argument specifies the exact size of the link target (e.g. by passing stat.st_size as returned by lstat()). This is necessary because 'readlink(..., hint) == hint' could mean that the buffer was too small. Use hint + 1 as buffer size to prevent this. Signed-off-by: Karsten Blees --- strbuf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/strbuf.c b/strbuf.c index 87ecf7f975..d74e193463 100644 --- a/strbuf.c +++ b/strbuf.c @@ -531,12 +531,12 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) while (hint < STRBUF_MAXLINK) { ssize_t len; - strbuf_grow(sb, hint); - len = readlink(path, sb->buf, hint); + strbuf_grow(sb, hint + 1); + len = readlink(path, sb->buf, hint + 1); if (len < 0) { if (errno != ERANGE) break; - } else if (len < hint) { + } else if (len <= hint) { strbuf_setlen(sb, len); return 0; } From 41f0299186ddbe164e4d0489e13b1d49a8db3cc7 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 22:15:40 +0200 Subject: [PATCH 23/44] strbuf_readlink: support link targets that exceed PATH_MAX strbuf_readlink() refuses to read link targets that exceed PATH_MAX (even if a sufficient size was specified by the caller). As some platforms support longer paths, remove this restriction (similar to strbuf_getcwd()). Signed-off-by: Karsten Blees --- strbuf.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/strbuf.c b/strbuf.c index d74e193463..a931e55f4a 100644 --- a/strbuf.c +++ b/strbuf.c @@ -519,8 +519,6 @@ ssize_t strbuf_write(struct strbuf *sb, FILE *f) } -#define STRBUF_MAXLINK (2*PATH_MAX) - int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) { size_t oldalloc = sb->alloc; @@ -528,7 +526,7 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) if (hint < 32) hint = 32; - while (hint < STRBUF_MAXLINK) { + for (;;) { ssize_t len; strbuf_grow(sb, hint + 1); From 3fd09dc2fd5b59e52b6b57f9b5eb44a54c9a7e58 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 19:58:14 +0200 Subject: [PATCH 24/44] lockfile.c: use is_dir_sep() instead of hardcoded '/' checks Signed-off-by: Karsten Blees --- lockfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lockfile.c b/lockfile.c index 8e8ab4f29f..3704a603f6 100644 --- a/lockfile.c +++ b/lockfile.c @@ -17,14 +17,14 @@ static void trim_last_path_component(struct strbuf *path) int i = path->len; /* back up past trailing slashes, if any */ - while (i && path->buf[i - 1] == '/') + while (i && is_dir_sep(path->buf[i - 1])) i--; /* * then go backwards until a slash, or the beginning of the * string */ - while (i && path->buf[i - 1] != '/') + while (i && !is_dir_sep(path->buf[i - 1])) i--; strbuf_setlen(path, i); From 1bea90cd2aafba476d483e955121ffb6432adde7 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 12 May 2015 11:09:01 +0200 Subject: [PATCH 25/44] Win32: don't call GetFileAttributes twice in mingw_lstat() GetFileAttributes cannot handle paths with trailing dir separator. The current [l]stat implementation calls GetFileAttributes twice if the path has trailing slashes (first with the original path passed to [l]stat, and and a second time with a path copy with trailing '/' removed). With Unicode conversion, we get the length of the path for free and also have a (wide char) buffer that can be modified. Remove trailing directory separators before calling the Win32 API. Signed-off-by: Karsten Blees --- compat/mingw.c | 48 ++++++++++++------------------------------------ 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c2b30fe3c..945b999c02 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -716,9 +716,18 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; wchar_t wfilename[MAX_LONG_PATH]; - if (xutftowcs_long_path(wfilename, file_name) < 0) + int wlen = xutftowcs_long_path(wfilename, file_name); + if (wlen < 0) return -1; + /* strip trailing '/', or GetFileAttributes will fail */ + while (wlen && is_dir_sep(wfilename[wlen - 1])) + wfilename[--wlen] = 0; + if (!wlen) { + errno = ENOENT; + return -1; + } + if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { buf->st_ino = 0; buf->st_gid = 0; @@ -778,39 +787,6 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) return -1; } -/* We provide our own lstat/fstat functions, since the provided - * lstat/fstat functions are so slow. These stat functions are - * tailored for Git's usage (read: fast), and are not meant to be - * complete. Note that Git stat()s are redirected to mingw_lstat() - * too, since Windows doesn't really handle symlinks that well. - */ -static int do_stat_internal(int follow, const char *file_name, struct stat *buf) -{ - int namelen; - char alt_name[MAX_LONG_PATH]; - - if (!do_lstat(follow, file_name, buf)) - return 0; - - /* if file_name ended in a '/', Windows returned ENOENT; - * try again without trailing slashes - */ - if (errno != ENOENT) - return -1; - - namelen = strlen(file_name); - if (namelen && file_name[namelen-1] != '/') - return -1; - while (namelen && file_name[namelen-1] == '/') - --namelen; - if (!namelen || namelen >= MAX_LONG_PATH) - return -1; - - memcpy(alt_name, file_name, namelen); - alt_name[namelen] = 0; - return do_lstat(follow, alt_name, buf); -} - int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) @@ -838,11 +814,11 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) int mingw_lstat(const char *file_name, struct stat *buf) { - return do_stat_internal(0, file_name, buf); + return do_lstat(0, file_name, buf); } int mingw_stat(const char *file_name, struct stat *buf) { - return do_stat_internal(1, file_name, buf); + return do_lstat(1, file_name, buf); } int mingw_fstat(int fd, struct stat *buf) From d5389c45646c7e4777ecbc4706ecf7f217bd2de9 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 01:18:14 +0200 Subject: [PATCH 26/44] Win32: implement stat() with symlink support With respect to symlinks, the current stat() implementation is almost the same as lstat(): except for the file type (st_mode & S_IFMT), it returns information about the link rather than the target. Implement stat by opening the file with as little permissions as possible and calling GetFileInformationByHandle on it. This way, all link resoltion is handled by the Windows file system layer. If symlinks are disabled, use lstat() as before, but fail with ELOOP if a symlink would have to be resolved. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 945b999c02..094de91870 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -816,9 +816,26 @@ int mingw_lstat(const char *file_name, struct stat *buf) { return do_lstat(0, file_name, buf); } + int mingw_stat(const char *file_name, struct stat *buf) { - return do_lstat(1, file_name, buf); + wchar_t wfile_name[MAX_LONG_PATH]; + HANDLE hnd; + int result; + + /* open the file and let Windows resolve the links */ + if (xutftowcs_long_path(wfile_name, file_name) < 0) + return -1; + hnd = CreateFileW(wfile_name, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + result = get_file_info_by_handle(hnd, buf); + CloseHandle(hnd); + return result; } int mingw_fstat(int fd, struct stat *buf) From e59ef4175f5fa42333c542a90458d842f6dc5695 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 12 May 2015 00:58:39 +0200 Subject: [PATCH 27/44] Win32: remove separate do_lstat() function With the new mingw_stat() implementation, do_lstat() is only called from mingw_lstat() (with follow == 0). Remove the extra function and the old mingw_stat()-specific (follow == 1) logic. Signed-off-by: Karsten Blees --- compat/mingw.c | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 094de91870..27b1d38124 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -705,14 +705,7 @@ static int has_valid_directory_prefix(wchar_t *wfilename) return 1; } -/* We keep the do_lstat code in a separate function to avoid recursion. - * When a path ends with a slash, the stat will fail with ENOENT. In - * this case, we strip the trailing slashes and stat again. - * - * If follow is true then act like stat() and report on the link - * target. Otherwise report on the link itself. - */ -static int do_lstat(int follow, const char *file_name, struct stat *buf) +int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; wchar_t wfilename[MAX_LONG_PATH]; @@ -746,13 +739,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) if (handle != INVALID_HANDLE_VALUE) { if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { - if (follow) { - char buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; - buf->st_size = readlink(file_name, buffer, MAXIMUM_REPARSE_DATA_BUFFER_SIZE); - } else { - buf->st_mode = S_IFLNK; - } - buf->st_mode |= S_IREAD; + buf->st_mode = S_IFLNK | S_IREAD; if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) buf->st_mode |= S_IWRITE; } @@ -812,11 +799,6 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) return 0; } -int mingw_lstat(const char *file_name, struct stat *buf) -{ - return do_lstat(0, file_name, buf); -} - int mingw_stat(const char *file_name, struct stat *buf) { wchar_t wfile_name[MAX_LONG_PATH]; From bf2324f4feac7c33ee0d7aa83a71f39c832f970f Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 00:17:56 +0200 Subject: [PATCH 28/44] Win32: let mingw_lstat() error early upon problems with reparse points When obtaining lstat information for reparse points, we need to call FindFirstFile() in addition to GetFileInformationEx() to obtain the type of the reparse point (symlink, mount point etc.). However, currently there is no error handling whatsoever if FindFirstFile() fails. Call FindFirstFile() before modifying the stat *buf output parameter and error out if the call fails. Note: The FindFirstFile() return value includes all the data that we get from GetFileAttributesEx(), so we could replace GetFileAttributesEx() with FindFirstFile(). We don't do that because GetFileAttributesEx() is about twice as fast for single files. I.e. we only pay the extra cost of calling FindFirstFile() in the rare case that we encounter a reparse point. Note: The indentation of the remaining reparse point code will be fixed in the next patch. Signed-off-by: Karsten Blees --- compat/mingw.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 27b1d38124..6e33acf30d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -708,6 +708,7 @@ static int has_valid_directory_prefix(wchar_t *wfilename) int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; + WIN32_FIND_DATAW findbuf = { 0 }; wchar_t wfilename[MAX_LONG_PATH]; int wlen = xutftowcs_long_path(wfilename, file_name); if (wlen < 0) @@ -722,6 +723,13 @@ int mingw_lstat(const char *file_name, struct stat *buf) } if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { + /* for reparse points, use FindFirstFile to get the reparse tag */ + if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + HANDLE handle = FindFirstFileW(wfilename, &findbuf); + if (handle == INVALID_HANDLE_VALUE) + goto error; + FindClose(handle); + } buf->st_ino = 0; buf->st_gid = 0; buf->st_uid = 0; @@ -734,20 +742,16 @@ int mingw_lstat(const char *file_name, struct stat *buf) filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - WIN32_FIND_DATAW findbuf; - HANDLE handle = FindFirstFileW(wfilename, &findbuf); - if (handle != INVALID_HANDLE_VALUE) { if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { buf->st_mode = S_IFLNK | S_IREAD; if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) buf->st_mode |= S_IWRITE; } - FindClose(handle); - } } return 0; } +error: switch (GetLastError()) { case ERROR_ACCESS_DENIED: case ERROR_SHARING_VIOLATION: From d82a6db2481b57fe70833e3738818e328698a328 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 10 Jan 2017 23:21:56 +0100 Subject: [PATCH 29/44] Win32: teach fscache and dirent about symlinks Move S_IFLNK detection to file_attr_to_st_mode() and reuse it in fscache. Implement DT_LNK detection in dirent.c and the fscache readdir version. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 13 +++---------- compat/win32.h | 6 ++++-- compat/win32/dirent.c | 5 ++++- compat/win32/fscache.c | 6 ++++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 6e33acf30d..e4c3e099d5 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -734,21 +734,14 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, + findbuf.dwReserved0); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); - if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && - (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { - buf->st_mode = S_IFLNK | S_IREAD; - if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) - buf->st_mode |= S_IWRITE; - } - } return 0; } error: @@ -793,7 +786,7 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ diff --git a/compat/win32.h b/compat/win32.h index a97e880757..671bcc81f9 100644 --- a/compat/win32.h +++ b/compat/win32.h @@ -6,10 +6,12 @@ #include #endif -static inline int file_attr_to_st_mode (DWORD attr) +static inline int file_attr_to_st_mode (DWORD attr, DWORD tag) { int fMode = S_IREAD; - if (attr & FILE_ATTRIBUTE_DIRECTORY) + if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && tag == IO_REPARSE_TAG_SYMLINK) + fMode |= S_IFLNK; + else if (attr & FILE_ATTRIBUTE_DIRECTORY) fMode |= S_IFDIR; else fMode |= S_IFREG; diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index b3bd8d7af7..8c654d722b 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -16,7 +16,10 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ - if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + if ((fdata->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + && fdata->dwReserved0 == IO_REPARSE_TAG_SYMLINK) + ent->d_type = DT_LNK; + else if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ent->d_type = DT_DIR; else ent->d_type = DT_REG; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index a015d4d8cb..4386c4362e 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -149,7 +149,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, fse = fsentry_alloc(list, buf, len); - fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, + fdata->dwReserved0); fse->u.s.st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) | fdata->nFileSizeLow; filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->u.s.st_atim)); @@ -442,7 +443,8 @@ static struct dirent *fscache_readdir(DIR *base_dir) if (!next) return NULL; dir->pfsentry = next; - dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_type = S_ISREG(next->st_mode) ? DT_REG : + S_ISDIR(next->st_mode) ? DT_DIR : DT_LNK; dir->dirent.d_name = (char*) next->name; return &(dir->dirent); } From b843247d78f900a5021e360f2dba7cd50432c1af Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 01:11:37 +0200 Subject: [PATCH 30/44] Win32: lstat(): return adequate stat.st_size for symlinks Git typically doesn't trust the stat.st_size member of symlinks (e.g. see strbuf_readlink()). However, some functions take shortcuts if st_size is 0 (e.g. diff_populate_filespec()). In mingw_lstat() and fscache_lstat(), make sure to return an adequate size. The extra overhead of opening and reading the reparse point to calculate the exact size is not necessary, as git doesn't rely on the value anyway. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 4 ++-- compat/win32/fscache.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index e4c3e099d5..e7665e5eaf 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -736,8 +736,8 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_nlink = 1; buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, findbuf.dwReserved0); - buf->st_size = fdata.nFileSizeLow | - (((off_t)fdata.nFileSizeHigh)<<32); + buf->st_size = S_ISLNK(buf->st_mode) ? MAX_LONG_PATH : + fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 4386c4362e..cc2c22b0c5 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -151,8 +151,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, fdata->dwReserved0); - fse->u.s.st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) - | fdata->nFileSizeLow; + fse->u.s.st_size = S_ISLNK(fse->st_mode) ? MAX_LONG_PATH : + fdata->nFileSizeLow | (((off_t) fdata->nFileSizeHigh) << 32); filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->u.s.st_atim)); filetime_to_timespec(&(fdata->ftLastWriteTime), &(fse->u.s.st_mtim)); filetime_to_timespec(&(fdata->ftCreationTime), &(fse->u.s.st_ctim)); From 489176b043fa4d8e8ff1529a556132fdb85a6b94 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 19 May 2015 21:48:55 +0200 Subject: [PATCH 31/44] Win32: factor out retry logic The retry pattern is duplicated in three places. It also seems to be too hard to use: mingw_unlink() and mingw_rmdir() duplicate the code to retry, and both of them do so incompletely. They also do not restore errno if the user answers 'no'. Introduce a retry_ask_yes_no() helper function that handles retry with small delay, asking the user, and restoring errno. mingw_unlink: include _wchmod in the retry loop (which may fail if the file is locked exclusively). mingw_rmdir: include special error handling in the retry loop. Signed-off-by: Karsten Blees --- compat/mingw.c | 98 ++++++++++++++++++++++---------------------------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index e7665e5eaf..bafa95473b 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -11,8 +11,6 @@ #define HCAST(type, handle) ((type)(intptr_t)handle) -static const int delay[] = { 0, 1, 10, 20, 40 }; - void open_in_gdb(void) { static struct child_process cp = CHILD_PROCESS_INIT; @@ -188,15 +186,12 @@ static int read_yes_no_answer(void) return -1; } -static int ask_yes_no_if_possible(const char *format, ...) +static int ask_yes_no_if_possible(const char *format, va_list args) { char question[4096]; const char *retry_hook[] = { NULL, NULL, NULL }; - va_list args; - va_start(args, format); vsnprintf(question, sizeof(question), format, args); - va_end(args); if ((retry_hook[0] = mingw_getenv("GIT_ASK_YESNO"))) { retry_hook[1] = question; @@ -218,6 +213,31 @@ static int ask_yes_no_if_possible(const char *format, ...) } } +static int retry_ask_yes_no(int *tries, const char *format, ...) +{ + static const int delay[] = { 0, 1, 10, 20, 40 }; + va_list args; + int result, saved_errno = errno; + + if ((*tries) < ARRAY_SIZE(delay)) { + /* + * We assume that some other process had the file open at the wrong + * moment and retry. In order to give the other process a higher + * chance to complete its operation, we give up our time slice now. + * If we have to retry again, we do sleep a bit. + */ + Sleep(delay[*tries]); + (*tries)++; + return 1; + } + + va_start(args, format); + result = ask_yes_no_if_possible(format, args); + va_end(args); + errno = saved_errno; + return result; +} + /* Windows only */ enum hide_dotfiles_type { HIDE_DOTFILES_FALSE = 0, @@ -286,31 +306,21 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf) int mingw_unlink(const char *pathname) { - int ret, tries = 0; + int tries = 0; wchar_t wpathname[MAX_LONG_PATH]; if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; - /* read-only files cannot be removed */ - _wchmod(wpathname, 0666); - while ((ret = _wunlink(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { + do { + /* read-only files cannot be removed */ + _wchmod(wpathname, 0666); + if (!_wunlink(wpathname)) + return 0; if (!is_file_in_use_error(GetLastError())) break; - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - } - while (ret == -1 && is_file_in_use_error(GetLastError()) && - ask_yes_no_if_possible("Unlink of file '%s' failed. " - "Should I try again?", pathname)) - ret = _wunlink(wpathname); - return ret; + } while (retry_ask_yes_no(&tries, "Unlink of file '%s' failed. " + "Should I try again?", pathname)); + return -1; } static int is_dir_empty(const wchar_t *wpath) @@ -337,12 +347,14 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { - int ret, tries = 0; + int tries = 0; wchar_t wpathname[MAX_LONG_PATH]; if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; - while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { + do { + if (!_wrmdir(wpathname)) + return 0; if (!is_file_in_use_error(GetLastError())) errno = err_win_to_posix(GetLastError()); if (errno != EACCES) @@ -351,21 +363,9 @@ int mingw_rmdir(const char *pathname) errno = ENOTEMPTY; break; } - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - } - while (ret == -1 && errno == EACCES && is_file_in_use_error(GetLastError()) && - ask_yes_no_if_possible("Deletion of directory '%s' failed. " - "Should I try again?", pathname)) - ret = _wrmdir(wpathname); - return ret; + } while (retry_ask_yes_no(&tries, "Deletion of directory '%s' failed. " + "Should I try again?", pathname)); + return -1; } static inline int needs_hiding(const char *path) @@ -1911,20 +1911,8 @@ repeat: SetFileAttributesW(wpnew, attrs); } } - if (tries < ARRAY_SIZE(delay) && gle == ERROR_ACCESS_DENIED) { - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - goto repeat; - } if (gle == ERROR_ACCESS_DENIED && - ask_yes_no_if_possible("Rename from '%s' to '%s' failed. " + retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " "Should I try again?", pold, pnew)) goto repeat; From cfffdbde8b4e36d8dfc3aaef9255d0545d78aa7e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:55:05 +0200 Subject: [PATCH 32/44] Win32: change default of 'core.symlinks' to false Symlinks on Windows don't work the same way as on Unix systems. E.g. there are different types of symlinks for directories and files, creating symlinks requires administrative privileges etc. By default, disable symlink support on Windows. I.e. users explicitly have to enable it with 'git config [--system|--global] core.symlinks true'. The test suite ignores system / global config files. Allow testing *with* symlink support by checking if native symlinks are enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Reminder: This would need to be changed if / when we find a way to run the test suite in a non-MSys-based shell (e.g. dash). Signed-off-by: Karsten Blees --- compat/mingw.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index bafa95473b..4ae1a0cad0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2373,6 +2373,15 @@ static void setup_windows_environment(void) setenv("HOME", tmp, 1); } + /* + * Change 'core.symlinks' default to false, unless native symlinks are + * enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Thus we can + * run the test suite (which doesn't obey config files) with or without + * symlink support. + */ + if (!(tmp = getenv("MSYS")) || !strstr(tmp, "winsymlinks:nativestrict")) + has_symlinks = 0; + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) setenv("LC_CTYPE", "C", 1); } From 6cc95843725ee095c5e836a7e8fb519819c0df8a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 00:32:03 +0200 Subject: [PATCH 33/44] Win32: add symlink-specific error codes Signed-off-by: Karsten Blees --- compat/mingw.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 4ae1a0cad0..de8f4cfae8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -86,6 +86,7 @@ int err_win_to_posix(DWORD winerr) case ERROR_INVALID_PARAMETER: error = EINVAL; break; case ERROR_INVALID_PASSWORD: error = EPERM; break; case ERROR_INVALID_PRIMARY_GROUP: error = EINVAL; break; + case ERROR_INVALID_REPARSE_DATA: error = EINVAL; break; case ERROR_INVALID_SIGNAL_NUMBER: error = EINVAL; break; case ERROR_INVALID_TARGET_HANDLE: error = EIO; break; case ERROR_INVALID_WORKSTATION: error = EACCES; break; @@ -100,6 +101,7 @@ int err_win_to_posix(DWORD winerr) case ERROR_NEGATIVE_SEEK: error = ESPIPE; break; case ERROR_NOACCESS: error = EFAULT; break; case ERROR_NONE_MAPPED: error = EINVAL; break; + case ERROR_NOT_A_REPARSE_POINT: error = EINVAL; break; case ERROR_NOT_ENOUGH_MEMORY: error = ENOMEM; break; case ERROR_NOT_READY: error = EAGAIN; break; case ERROR_NOT_SAME_DEVICE: error = EXDEV; break; @@ -120,6 +122,9 @@ int err_win_to_posix(DWORD winerr) case ERROR_PIPE_NOT_CONNECTED: error = EPIPE; break; case ERROR_PRIVILEGE_NOT_HELD: error = EACCES; break; case ERROR_READ_FAULT: error = EIO; break; + case ERROR_REPARSE_ATTRIBUTE_CONFLICT: error = EINVAL; break; + case ERROR_REPARSE_TAG_INVALID: error = EINVAL; break; + case ERROR_REPARSE_TAG_MISMATCH: error = EINVAL; break; case ERROR_SEEK: error = EIO; break; case ERROR_SEEK_ON_DEVICE: error = ESPIPE; break; case ERROR_SHARING_BUFFER_EXCEEDED: error = ENFILE; break; From a6e9c81af1c9194087fbda8a259abf8b36d96093 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:06:10 +0200 Subject: [PATCH 34/44] Win32: mingw_unlink: support symlinks to directories _wunlink() / DeleteFileW() refuses to delete symlinks to directories. If _wunlink() fails with ERROR_ACCESS_DENIED, try _wrmdir() as well. Signed-off-by: Karsten Blees --- compat/mingw.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index de8f4cfae8..fb417fa0f8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -323,6 +323,13 @@ int mingw_unlink(const char *pathname) return 0; if (!is_file_in_use_error(GetLastError())) break; + /* + * _wunlink() / DeleteFileW() for directory symlinks fails with + * ERROR_ACCESS_DENIED (EACCES), so try _wrmdir() as well. This is the + * same error we get if a file is in use (already checked above). + */ + if (!_wrmdir(wpathname)) + return 0; } while (retry_ask_yes_no(&tries, "Unlink of file '%s' failed. " "Should I try again?", pathname)); return -1; From 2d7219082eecd86308e4f41f83f9a12d4f2a61be Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 19 May 2015 22:42:48 +0200 Subject: [PATCH 35/44] Win32: mingw_rename: support renaming symlinks MSVCRT's _wrename() cannot rename symlinks over existing files: it returns success without doing anything. Newer MSVCR*.dll versions probably do not have this problem: according to CRT sources, they just call MoveFileEx() with the MOVEFILE_COPY_ALLOWED flag. Get rid of _wrename() and call MoveFileEx() with proper error handling. Signed-off-by: Karsten Blees --- compat/mingw.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index fb417fa0f8..f8632812b2 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1883,27 +1883,29 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) #undef rename int mingw_rename(const char *pold, const char *pnew) { - DWORD attrs, gle; + DWORD attrs = INVALID_FILE_ATTRIBUTES, gle; int tries = 0; wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; if (xutftowcs_long_path(wpold, pold) < 0 || xutftowcs_long_path(wpnew, pnew) < 0) return -1; - /* - * Try native rename() first to get errno right. - * It is based on MoveFile(), which cannot overwrite existing files. - */ - if (!_wrename(wpold, wpnew)) - return 0; - if (errno != EEXIST) - return -1; repeat: - if (MoveFileExW(wpold, wpnew, MOVEFILE_REPLACE_EXISTING)) + if (MoveFileExW(wpold, wpnew, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED)) return 0; - /* TODO: translate more errors */ gle = GetLastError(); - if (gle == ERROR_ACCESS_DENIED && + + /* revert file attributes on failure */ + if (attrs != INVALID_FILE_ATTRIBUTES) + SetFileAttributesW(wpnew, attrs); + + if (!is_file_in_use_error(gle)) { + errno = err_win_to_posix(gle); + return -1; + } + + if (attrs == INVALID_FILE_ATTRIBUTES && (attrs = GetFileAttributesW(wpnew)) != INVALID_FILE_ATTRIBUTES) { if (attrs & FILE_ATTRIBUTE_DIRECTORY) { DWORD attrsold = GetFileAttributesW(wpold); @@ -1915,16 +1917,10 @@ repeat: return -1; } if ((attrs & FILE_ATTRIBUTE_READONLY) && - SetFileAttributesW(wpnew, attrs & ~FILE_ATTRIBUTE_READONLY)) { - if (MoveFileExW(wpold, wpnew, MOVEFILE_REPLACE_EXISTING)) - return 0; - gle = GetLastError(); - /* revert file attributes on failure */ - SetFileAttributesW(wpnew, attrs); - } + SetFileAttributesW(wpnew, attrs & ~FILE_ATTRIBUTE_READONLY)) + goto repeat; } - if (gle == ERROR_ACCESS_DENIED && - retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " + if (retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " "Should I try again?", pold, pnew)) goto repeat; From cc0ab95826a4f55c8da866c5719aeb74f2fdb2c3 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:17:31 +0200 Subject: [PATCH 36/44] Win32: mingw_chdir: change to symlink-resolved directory If symlinks are enabled, resolve all symlinks when changing directories, as required by POSIX. Note: Git's real_path() function bases its link resolution algorithm on this property of chdir(). Unfortunately, the current directory on Windows is limited to only MAX_PATH (260) characters. Therefore using symlinks and long paths in combination may be problematic. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index f8632812b2..7e336fcced 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -671,7 +671,24 @@ int mingw_chdir(const char *dirname) wchar_t wdirname[MAX_LONG_PATH]; if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; - result = _wchdir(wdirname); + + if (has_symlinks) { + HANDLE hnd = CreateFileW(wdirname, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + if (!GetFinalPathNameByHandleW(hnd, wdirname, ARRAY_SIZE(wdirname), 0)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(hnd); + return -1; + } + CloseHandle(hnd); + } + + result = _wchdir(normalize_ntpath(wdirname)); current_directory_len = GetCurrentDirectoryW(0, NULL); return result; } From 20443e8990c8e45228b6e4a1a896f66ff36c50dd Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:24:41 +0200 Subject: [PATCH 37/44] Win32: implement readlink() Implement readlink() by reading NTFS reparse points. Works for symlinks and directory junctions. If symlinks are disabled, fail with ENOSYS. Signed-off-by: Karsten Blees --- compat/mingw.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 3 +- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 7e336fcced..3b9584c7d6 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2,6 +2,7 @@ #include "win32.h" #include #include +#include #include "../strbuf.h" #include "../run-command.h" #include "../cache.h" @@ -2209,6 +2210,103 @@ int link(const char *oldpath, const char *newpath) return 0; } +#ifndef _WINNT_H +/* + * The REPARSE_DATA_BUFFER structure is defined in the Windows DDK (in + * ntifs.h) and in MSYS1's winnt.h (which defines _WINNT_H). So define + * it ourselves if we are on MSYS2 (whose winnt.h defines _WINNT_). + */ +typedef struct _REPARSE_DATA_BUFFER { + DWORD ReparseTag; + WORD ReparseDataLength; + WORD Reserved; +#ifndef _MSC_VER + _ANONYMOUS_UNION +#endif + union { + struct { + WORD SubstituteNameOffset; + WORD SubstituteNameLength; + WORD PrintNameOffset; + WORD PrintNameLength; + ULONG Flags; + WCHAR PathBuffer[1]; + } SymbolicLinkReparseBuffer; + struct { + WORD SubstituteNameOffset; + WORD SubstituteNameLength; + WORD PrintNameOffset; + WORD PrintNameLength; + WCHAR PathBuffer[1]; + } MountPointReparseBuffer; + struct { + BYTE DataBuffer[1]; + } GenericReparseBuffer; + } DUMMYUNIONNAME; +} REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER; +#endif + +int readlink(const char *path, char *buf, size_t bufsiz) +{ + HANDLE handle; + WCHAR wpath[MAX_LONG_PATH], *wbuf; + REPARSE_DATA_BUFFER *b = alloca(MAXIMUM_REPARSE_DATA_BUFFER_SIZE); + DWORD dummy; + char tmpbuf[MAX_LONG_PATH]; + int len; + + if (xutftowcs_long_path(wpath, path) < 0) + return -1; + + /* read reparse point data */ + handle = CreateFileW(wpath, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (handle == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + if (!DeviceIoControl(handle, FSCTL_GET_REPARSE_POINT, NULL, 0, b, + MAXIMUM_REPARSE_DATA_BUFFER_SIZE, &dummy, NULL)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(handle); + return -1; + } + CloseHandle(handle); + + /* get target path for symlinks or mount points (aka 'junctions') */ + switch (b->ReparseTag) { + case IO_REPARSE_TAG_SYMLINK: + wbuf = (WCHAR*) (((char*) b->SymbolicLinkReparseBuffer.PathBuffer) + + b->SymbolicLinkReparseBuffer.SubstituteNameOffset); + *(WCHAR*) (((char*) wbuf) + + b->SymbolicLinkReparseBuffer.SubstituteNameLength) = 0; + break; + case IO_REPARSE_TAG_MOUNT_POINT: + wbuf = (WCHAR*) (((char*) b->MountPointReparseBuffer.PathBuffer) + + b->MountPointReparseBuffer.SubstituteNameOffset); + *(WCHAR*) (((char*) wbuf) + + b->MountPointReparseBuffer.SubstituteNameLength) = 0; + break; + default: + errno = EINVAL; + return -1; + } + + /* + * Adapt to strange readlink() API: Copy up to bufsiz *bytes*, potentially + * cutting off a UTF-8 sequence. Insufficient bufsize is *not* a failure + * condition. There is no conversion function that produces invalid UTF-8, + * so convert to a (hopefully large enough) temporary buffer, then memcpy + * the requested number of bytes (including '\0' for robustness). + */ + if ((len = xwcstoutf(tmpbuf, normalize_ntpath(wbuf), MAX_LONG_PATH)) < 0) + return -1; + memcpy(buf, tmpbuf, min(bufsiz, len + 1)); + return min(bufsiz, len); +} + pid_t waitpid(pid_t pid, int *status, int options) { HANDLE h = OpenProcess(SYNCHRONIZE | PROCESS_QUERY_INFORMATION, diff --git a/compat/mingw.h b/compat/mingw.h index ec24214464..ab0800bafb 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -123,8 +123,6 @@ struct utsname { * trivial stubs */ -static inline int readlink(const char *path, char *buf, size_t bufsiz) -{ errno = ENOSYS; return -1; } static inline int symlink(const char *oldpath, const char *newpath) { errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) @@ -218,6 +216,7 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); +int readlink(const char *path, char *buf, size_t bufsiz); /* * replacements of existing functions From 8906d15a84c09533265eb78b3678be4f413fb88a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:32:03 +0200 Subject: [PATCH 38/44] Win32: implement basic symlink() functionality (file symlinks only) Implement symlink() that always creates file symlinks. Fails with ENOSYS if symlinks are disabled or unsupported. Note: CreateSymbolicLinkW() was introduced with symlink support in Windows Vista. For compatibility with Windows XP, we need to load it dynamically and fail gracefully if it isnt's available. Signed-off-by: Karsten Blees --- compat/mingw.c | 28 ++++++++++++++++++++++++++++ compat/mingw.h | 3 +-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3b9584c7d6..ff67499ff1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2210,6 +2210,34 @@ int link(const char *oldpath, const char *newpath) return 0; } +int symlink(const char *target, const char *link) +{ + wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; + int len; + + /* fail if symlinks are disabled or API is not supported (WinXP) */ + if (!has_symlinks) { + errno = ENOSYS; + return -1; + } + + if ((len = xutftowcs_long_path(wtarget, target)) < 0 + || xutftowcs_long_path(wlink, link) < 0) + return -1; + + /* convert target dir separators to backslashes */ + while (len--) + if (wtarget[len] == '/') + wtarget[len] = '\\'; + + /* create file symlink */ + if (!CreateSymbolicLinkW(wlink, wtarget, 0)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + return 0; +} + #ifndef _WINNT_H /* * The REPARSE_DATA_BUFFER structure is defined in the Windows DDK (in diff --git a/compat/mingw.h b/compat/mingw.h index ab0800bafb..17d4746f55 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -123,8 +123,6 @@ struct utsname { * trivial stubs */ -static inline int symlink(const char *oldpath, const char *newpath) -{ errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) { errno = ENOSYS; return -1; } #ifndef __MINGW64_VERSION_MAJOR @@ -216,6 +214,7 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); +int symlink(const char *target, const char *link); int readlink(const char *path, char *buf, size_t bufsiz); /* From 8f26b7767d876861288d540b04a549623a1f4d79 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:48:35 +0200 Subject: [PATCH 39/44] Win32: symlink: add support for symlinks to directories Symlinks on Windows have a flag that indicates whether the target is a file or a directory. Symlinks of wrong type simply don't work. This even affects core Win32 APIs (e.g. DeleteFile() refuses to delete directory symlinks). However, CreateFile() with FILE_FLAG_BACKUP_SEMANTICS doesn't seem to care. Check the target type by first creating a tentative file symlink, opening it, and checking the type of the resulting handle. If it is a directory, recreate the symlink with the directory flag set. It is possible to create symlinks before the target exists (or in case of symlinks to symlinks: before the target type is known). If this happens, create a tentative file symlink and postpone the directory decision: keep a list of phantom symlinks to be processed whenever a new directory is created in mingw_mkdir(). Limitations: This algorithm may fail if a link target changes from file to directory or vice versa, or if the target directory is created in another process. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 164 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index ff67499ff1..d7ee911b1e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -285,6 +285,131 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } +enum phantom_symlink_result { + PHANTOM_SYMLINK_RETRY, + PHANTOM_SYMLINK_DONE, + PHANTOM_SYMLINK_DIRECTORY +}; + +static inline int is_wdir_sep(wchar_t wchar) +{ + return wchar == L'/' || wchar == L'\\'; +} + +static const wchar_t *make_relative_to(const wchar_t *path, + const wchar_t *relative_to, wchar_t *out, + size_t size) +{ + size_t i = wcslen(relative_to), len; + + /* Is `path` already absolute? */ + if (is_wdir_sep(path[0]) || + (iswalpha(path[0]) && path[1] == L':' && is_wdir_sep(path[2]))) + return path; + + while (i > 0 && !is_wdir_sep(relative_to[i - 1])) + i--; + + /* Is `relative_to` in the current directory? */ + if (!i) + return path; + + len = wcslen(path); + if (i + len + 1 > size) { + error("Could not make '%S' relative to '%S' (too large)", + path, relative_to); + return NULL; + } + + memcpy(out, relative_to, i * sizeof(wchar_t)); + wcscpy(out + i, path); + return out; +} + +/* + * Changes a file symlink to a directory symlink if the target exists and is a + * directory. + */ +static enum phantom_symlink_result +process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink) +{ + HANDLE hnd; + BY_HANDLE_FILE_INFORMATION fdata; + wchar_t relative[MAX_LONG_PATH]; + const wchar_t *rel; + + /* check that wlink is still a file symlink */ + if ((GetFileAttributesW(wlink) + & (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)) + != FILE_ATTRIBUTE_REPARSE_POINT) + return PHANTOM_SYMLINK_DONE; + + /* make it relative, if necessary */ + rel = make_relative_to(wtarget, wlink, relative, ARRAY_SIZE(relative)); + if (!rel) + return PHANTOM_SYMLINK_DONE; + + /* let Windows resolve the link by opening it */ + hnd = CreateFileW(rel, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return PHANTOM_SYMLINK_RETRY; + } + + if (!GetFileInformationByHandle(hnd, &fdata)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(hnd); + return PHANTOM_SYMLINK_RETRY; + } + CloseHandle(hnd); + + /* if target exists and is a file, we're done */ + if (!(fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + return PHANTOM_SYMLINK_DONE; + + /* otherwise recreate the symlink with directory flag */ + if (DeleteFileW(wlink) && CreateSymbolicLinkW(wlink, wtarget, 1)) + return PHANTOM_SYMLINK_DIRECTORY; + + errno = err_win_to_posix(GetLastError()); + return PHANTOM_SYMLINK_RETRY; +} + +/* keep track of newly created symlinks to non-existing targets */ +struct phantom_symlink_info { + struct phantom_symlink_info *next; + wchar_t *wlink; + wchar_t *wtarget; +}; + +static struct phantom_symlink_info *phantom_symlinks = NULL; +static CRITICAL_SECTION phantom_symlinks_cs; + +static void process_phantom_symlinks(void) +{ + struct phantom_symlink_info *current, **psi; + EnterCriticalSection(&phantom_symlinks_cs); + /* process phantom symlinks list */ + psi = &phantom_symlinks; + while ((current = *psi)) { + enum phantom_symlink_result result = process_phantom_symlink( + current->wtarget, current->wlink); + if (result == PHANTOM_SYMLINK_RETRY) { + psi = ¤t->next; + } else { + /* symlink was processed, remove from list */ + *psi = current->next; + free(current); + /* if symlink was a directory, start over */ + if (result == PHANTOM_SYMLINK_DIRECTORY) + psi = &phantom_symlinks; + } + } + LeaveCriticalSection(&phantom_symlinks_cs); +} + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -434,6 +559,8 @@ int mingw_mkdir(const char *path, int mode) return -1; ret = _wmkdir(wpath); + if (!ret) + process_phantom_symlinks(); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); return ret; @@ -2235,6 +2362,42 @@ int symlink(const char *target, const char *link) errno = err_win_to_posix(GetLastError()); return -1; } + + /* convert to directory symlink if target exists */ + switch (process_phantom_symlink(wtarget, wlink)) { + case PHANTOM_SYMLINK_RETRY: { + /* if target doesn't exist, add to phantom symlinks list */ + wchar_t wfullpath[MAX_LONG_PATH]; + struct phantom_symlink_info *psi; + + /* convert to absolute path to be independent of cwd */ + len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); + if (!len || len >= MAX_LONG_PATH) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* over-allocate and fill phantom_symlink_info structure */ + psi = xmalloc(sizeof(struct phantom_symlink_info) + + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); + psi->wlink = (wchar_t *)(psi + 1); + wcscpy(psi->wlink, wfullpath); + psi->wtarget = psi->wlink + len + 1; + wcscpy(psi->wtarget, wtarget); + + EnterCriticalSection(&phantom_symlinks_cs); + psi->next = phantom_symlinks; + phantom_symlinks = psi; + LeaveCriticalSection(&phantom_symlinks_cs); + break; + } + case PHANTOM_SYMLINK_DIRECTORY: + /* if we created a dir symlink, process other phantom symlinks */ + process_phantom_symlinks(); + break; + default: + break; + } return 0; } @@ -2745,6 +2908,7 @@ int wmain(int argc, const wchar_t **wargv) /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); + InitializeCriticalSection(&phantom_symlinks_cs); /* set up default file mode and file modes for stdin/out/err */ _fmode = _O_BINARY; From f41c197f037a0a60a38d21772ad860800b431853 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 30 May 2017 21:50:57 +0200 Subject: [PATCH 40/44] mingw: try to create symlinks without elevated permissions With Windows 10 Build 14972 in Developer Mode, a new flag is supported by CreateSymbolicLink() to create symbolic links even when running outside of an elevated session (which was previously required). This new flag is called SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE and has the numeric value 0x02. Previous Windows 10 versions will not understand that flag and return an ERROR_INVALID_PARAMETER, therefore we have to be careful to try passing that flag only when the build number indicates that it is supported. For more information about the new flag, see this blog post: https://blogs.windows.com/buildingapps/2016/12/02/symlinks-windows-10/ This patch is loosely based on the patch submitted by Samuel D. Leslie as https://github.com/git-for-windows/git/pull/1184. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index d7ee911b1e..ffa65cddb8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -285,6 +285,8 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } +static DWORD symlink_file_flags = 0, symlink_directory_flags = 1; + enum phantom_symlink_result { PHANTOM_SYMLINK_RETRY, PHANTOM_SYMLINK_DONE, @@ -370,7 +372,8 @@ process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink) return PHANTOM_SYMLINK_DONE; /* otherwise recreate the symlink with directory flag */ - if (DeleteFileW(wlink) && CreateSymbolicLinkW(wlink, wtarget, 1)) + if (DeleteFileW(wlink) && + CreateSymbolicLinkW(wlink, wtarget, symlink_directory_flags)) return PHANTOM_SYMLINK_DIRECTORY; errno = err_win_to_posix(GetLastError()); @@ -2358,7 +2361,7 @@ int symlink(const char *target, const char *link) wtarget[len] = '\\'; /* create file symlink */ - if (!CreateSymbolicLinkW(wlink, wtarget, 0)) { + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { errno = err_win_to_posix(GetLastError()); return -1; } @@ -2846,6 +2849,24 @@ static void maybe_redirect_std_handles(void) GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); } +static void adjust_symlink_flags(void) +{ + /* + * Starting with Windows 10 Build 14972, symbolic links can be created + * using CreateSymbolicLink() without elevation by passing the flag + * SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE (0x02) as last + * parameter, provided the Developer Mode has been enabled. Some + * earlier Windows versions complain about this flag with an + * ERROR_INVALID_PARAMETER, hence we have to test the build number + * specifically. + */ + if (GetVersion() >= 14972 << 16) { + symlink_file_flags |= 2; + symlink_directory_flags |= 2; + } + +} + #ifdef _MSC_VER #ifdef _DEBUG #include @@ -2878,6 +2899,7 @@ int wmain(int argc, const wchar_t **wargv) #endif maybe_redirect_std_handles(); + adjust_symlink_flags(); fsync_object_files = 1; /* determine size of argv and environ conversion buffer */ From 6d9162a22b9b6edebe3011406ec3361a992e9d05 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 11:13:45 +0200 Subject: [PATCH 41/44] Win32: symlink: move phantom symlink creation to a separate function Signed-off-by: Bert Belder --- compat/mingw.c | 91 +++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index ffa65cddb8..6253e04c42 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -413,6 +413,54 @@ static void process_phantom_symlinks(void) LeaveCriticalSection(&phantom_symlinks_cs); } +static int create_phantom_symlink(wchar_t *wtarget, wchar_t *wlink) +{ + int len; + + /* create file symlink */ + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* convert to directory symlink if target exists */ + switch (process_phantom_symlink(wtarget, wlink)) { + case PHANTOM_SYMLINK_RETRY: { + /* if target doesn't exist, add to phantom symlinks list */ + wchar_t wfullpath[MAX_LONG_PATH]; + struct phantom_symlink_info *psi; + + /* convert to absolute path to be independent of cwd */ + len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); + if (!len || len >= MAX_LONG_PATH) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* over-allocate and fill phantom_symlink_info structure */ + psi = xmalloc(sizeof(struct phantom_symlink_info) + + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); + psi->wlink = (wchar_t *)(psi + 1); + wcscpy(psi->wlink, wfullpath); + psi->wtarget = psi->wlink + len + 1; + wcscpy(psi->wtarget, wtarget); + + EnterCriticalSection(&phantom_symlinks_cs); + psi->next = phantom_symlinks; + phantom_symlinks = psi; + LeaveCriticalSection(&phantom_symlinks_cs); + break; + } + case PHANTOM_SYMLINK_DIRECTORY: + /* if we created a dir symlink, process other phantom symlinks */ + process_phantom_symlinks(); + break; + default: + break; + } + return 0; +} + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -2360,48 +2408,7 @@ int symlink(const char *target, const char *link) if (wtarget[len] == '/') wtarget[len] = '\\'; - /* create file symlink */ - if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - /* convert to directory symlink if target exists */ - switch (process_phantom_symlink(wtarget, wlink)) { - case PHANTOM_SYMLINK_RETRY: { - /* if target doesn't exist, add to phantom symlinks list */ - wchar_t wfullpath[MAX_LONG_PATH]; - struct phantom_symlink_info *psi; - - /* convert to absolute path to be independent of cwd */ - len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); - if (!len || len >= MAX_LONG_PATH) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - /* over-allocate and fill phantom_symlink_info structure */ - psi = xmalloc(sizeof(struct phantom_symlink_info) - + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); - psi->wlink = (wchar_t *)(psi + 1); - wcscpy(psi->wlink, wfullpath); - psi->wtarget = psi->wlink + len + 1; - wcscpy(psi->wtarget, wtarget); - - EnterCriticalSection(&phantom_symlinks_cs); - psi->next = phantom_symlinks; - phantom_symlinks = psi; - LeaveCriticalSection(&phantom_symlinks_cs); - break; - } - case PHANTOM_SYMLINK_DIRECTORY: - /* if we created a dir symlink, process other phantom symlinks */ - process_phantom_symlinks(); - break; - default: - break; - } - return 0; + return create_phantom_symlink(wtarget, wlink); } #ifndef _WINNT_H From c9bf9b17af57f275fc5c35ae654a78563d43ec92 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 23:42:09 +0200 Subject: [PATCH 42/44] Win32: symlink: add test for `symlink` attribute Signed-off-by: Bert Belder --- t/t2040-checkout-symlink-attr.sh | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 t/t2040-checkout-symlink-attr.sh diff --git a/t/t2040-checkout-symlink-attr.sh b/t/t2040-checkout-symlink-attr.sh new file mode 100755 index 0000000000..6b8a15116e --- /dev/null +++ b/t/t2040-checkout-symlink-attr.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +test_description='checkout symlinks with `symlink` attribute on Windows + +Ensures that Git for Windows creates symlinks of the right type, +as specified by the `symlink` attribute in `.gitattributes`.' + +# Tell MSYS to create native symlinks. Without this flag test-lib's +# prerequisite detection for SYMLINKS doesn't detect the right thing. +MSYS=winsymlinks:nativestrict && export MSYS + +. ./test-lib.sh + +if ! test_have_prereq MINGW,SYMLINKS +then + skip_all='skipping $0: MinGW-only test, which requires symlink support.' + test_done +fi + +# Adds a symlink to the index without clobbering the work tree. +cache_symlink () { + sha=$(printf '%s' "$1" | git hash-object --stdin -w) && + git update-index --add --cacheinfo 120000,$sha,"$2" +} + +# MSYS2 is very forgiving, it will resolve symlinks even if the +# symlink type isn't correct. To make this test meaningful, try +# them with a native, non-MSYS executable. +cat_native () { + filename=$(cygpath -w "$1") && + cmd.exe /c "type \"$filename\"" +} + +test_expect_success 'checkout symlinks with attr' ' + cache_symlink file1 file-link && + cache_symlink dir dir-link && + + printf "file-link symlink=file\ndir-link symlink=dir\n" >.gitattributes && + git add .gitattributes && + + git checkout . && + + mkdir dir && + echo "contents1" >file1 && + echo "contents2" >dir/file2 && + + test "$(cat_native file-link)" = "contents1" && + test "$(cat_native dir-link/file2)" = "contents2" +' + +test_done From f52192e0d7c94056eba90c5cf8d07ff072187118 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 11 Feb 2019 14:19:18 +0100 Subject: [PATCH 43/44] Introduce helper to create symlinks that knows about index_state On Windows, symbolic links actually have a type depending on the target: it can be a file or a directory. In certain circumstances, this poses problems, e.g. when a symbolic link is supposed to point into a submodule that is not checked out, so there is no way for Git to auto-detect the type. To help with that, we will add support over the course of the next commits to specify that symlink type via the Git attributes. This requires an index_state, though, something that Git for Windows' `symlink()` replacement cannot know about because the function signature is defined by the POSIX standard and not ours to change. So let's introduce a helper function to create symbolic links that *does* know about the index_state. Signed-off-by: Johannes Schindelin --- apply.c | 2 +- builtin/difftool.c | 2 +- builtin/init-db.c | 4 ++-- compat/mingw.c | 2 +- compat/mingw.h | 4 +++- entry.c | 2 +- git-compat-util.h | 9 +++++++++ merge-recursive.c | 2 +- refs/files-backend.c | 2 +- 9 files changed, 20 insertions(+), 9 deletions(-) diff --git a/apply.c b/apply.c index 27b6fd90f7..6a4426b3c5 100644 --- a/apply.c +++ b/apply.c @@ -4356,7 +4356,7 @@ static int try_create_file(struct apply_state *state, const char *path, /* Although buf:size is counted string, it also is NUL * terminated. */ - return !!symlink(buf, path); + return !!create_symlink(state && state->repo ? state->repo->index : NULL, buf, path); fd = open(path, O_CREAT | O_EXCL | O_WRONLY, (mode & 0100) ? 0777 : 0666); if (fd < 0) diff --git a/builtin/difftool.c b/builtin/difftool.c index 5704a76088..7870781123 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -505,7 +505,7 @@ static int run_dir_diff(const char *extcmd, int symlinks, const char *prefix, } add_path(&wtdir, wtdir_len, dst_path); if (symlinks) { - if (symlink(wtdir.buf, rdir.buf)) { + if (create_symlink(lstate.istate, wtdir.buf, rdir.buf)) { ret = error_errno("could not symlink '%s' to '%s'", wtdir.buf, rdir.buf); goto finish; } diff --git a/builtin/init-db.c b/builtin/init-db.c index 94df241ad5..1a132596e2 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -76,7 +76,7 @@ static void copy_templates_1(struct strbuf *path, struct strbuf *template_path, if (strbuf_readlink(&lnk, template_path->buf, st_template.st_size) < 0) die_errno(_("cannot readlink '%s'"), template_path->buf); - if (symlink(lnk.buf, path->buf)) + if (create_symlink(NULL, lnk.buf, path->buf)) die_errno(_("cannot symlink '%s' '%s'"), lnk.buf, path->buf); strbuf_release(&lnk); @@ -278,7 +278,7 @@ static int create_default_files(const char *template_path, path = git_path_buf(&buf, "tXXXXXX"); if (!close(xmkstemp(path)) && !unlink(path) && - !symlink("testing", path) && + !create_symlink(NULL, "testing", path) && !lstat(path, &st1) && S_ISLNK(st1.st_mode)) unlink(path); /* good */ diff --git a/compat/mingw.c b/compat/mingw.c index 6253e04c42..9f4c83e5ab 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2388,7 +2388,7 @@ int link(const char *oldpath, const char *newpath) return 0; } -int symlink(const char *target, const char *link) +int mingw_create_symlink(struct index_state *index, const char *target, const char *link) { wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; int len; diff --git a/compat/mingw.h b/compat/mingw.h index 17d4746f55..14bd14ad8e 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -214,8 +214,10 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); -int symlink(const char *target, const char *link); int readlink(const char *path, char *buf, size_t bufsiz); +struct index_state; +int mingw_create_symlink(struct index_state *index, const char *target, const char *link); +#define create_symlink mingw_create_symlink /* * replacements of existing functions diff --git a/entry.c b/entry.c index 6fd72b30c8..045b33048a 100644 --- a/entry.c +++ b/entry.c @@ -289,7 +289,7 @@ static int write_entry(struct cache_entry *ce, if (!has_symlinks || to_tempfile) goto write_file_entry; - ret = symlink(new_blob, path); + ret = create_symlink(state->istate, new_blob, path); free(new_blob); if (ret) return error_errno("unable to create symlink %s", path); diff --git a/git-compat-util.h b/git-compat-util.h index 49c76f3eca..3be10d55be 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -406,6 +406,15 @@ static inline char *git_find_last_dir_sep(const char *path) #define find_last_dir_sep git_find_last_dir_sep #endif +#ifndef create_symlink +struct index_state; +static inline int git_create_symlink(struct index_state *index, const char *target, const char *link) +{ + return symlink(target, link); +} +#define create_symlink git_create_symlink +#endif + #ifndef query_user_email #define query_user_email() NULL #endif diff --git a/merge-recursive.c b/merge-recursive.c index 4851825aeb..255347d23e 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -1011,7 +1011,7 @@ static int update_file_flags(struct merge_options *o, char *lnk = xmemdupz(buf, size); safe_create_leading_directories_const(path); unlink(path); - if (symlink(lnk, path)) + if (create_symlink(&o->orig_index, lnk, path)) ret = err(o, _("failed to symlink '%s': %s"), path, strerror(errno)); free(lnk); diff --git a/refs/files-backend.c b/refs/files-backend.c index dd8abe9185..bf5c7b9e40 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -1786,7 +1786,7 @@ static int create_ref_symlink(struct ref_lock *lock, const char *target) #ifndef NO_SYMLINK_HEAD char *ref_path = get_locked_file_path(&lock->lk); unlink(ref_path); - ret = symlink(target, ref_path); + ret = create_symlink(NULL, target, ref_path); free(ref_path); if (ret) From 2b6197db861ca7228046c34b348d7f152845b61a Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 11:51:51 +0200 Subject: [PATCH 44/44] mingw: allow to specify the symlink type in .gitattributes On Windows, symbolic links have a type: a "file symlink" must point at a file, and a "directory symlink" must point at a directory. If the type of symlink does not match its target, it doesn't work. Git does not record the type of symlink in the index or in a tree. On checkout it'll guess the type, which only works if the target exists at the time the symlink is created. This may often not be the case, for example when the link points at a directory inside a submodule. By specifying `symlink=file` or `symlink=dir` the user can specify what type of symlink Git should create, so Git doesn't have to rely on unreliable heuristics. Signed-off-by: Bert Belder Signed-off-by: Johannes Schindelin --- Documentation/gitattributes.txt | 30 +++++++++++++++++ compat/mingw.c | 58 ++++++++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt index bdd11a2ddd..269356d9b2 100644 --- a/Documentation/gitattributes.txt +++ b/Documentation/gitattributes.txt @@ -382,6 +382,36 @@ sign `$` upon checkout. Any byte sequence that begins with with `$Id$` upon check-in. +`symlink` +^^^^^^^^^ + +On Windows, symbolic links have a type: a "file symlink" must point at +a file, and a "directory symlink" must point at a directory. If the +type of symlink does not match its target, it doesn't work. + +Git does not record the type of symlink in the index or in a tree. On +checkout it'll guess the type, which only works if the target exists +at the time the symlink is created. This may often not be the case, +for example when the link points at a directory inside a submodule. + +The `symlink` attribute allows you to explicitly set the type of symlink +to `file` or `dir`, so Git doesn't have to guess. If you have a set of +symlinks that point at other files, you can do: + +------------------------ +*.gif symlink=file +------------------------ + +To tell Git that a symlink points at a directory, use: + +------------------------ +tools_folder symlink=dir +------------------------ + +The `symlink` attribute is ignored on platforms other than Windows, +since they don't distinguish between different types of symlinks. + + `filter` ^^^^^^^^ diff --git a/compat/mingw.c b/compat/mingw.c index 9f4c83e5ab..3a34c3ad66 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -9,6 +9,7 @@ #include "win32/lazyload.h" #include "../config.h" #include "dir.h" +#include "../attr.h" #define HCAST(type, handle) ((type)(intptr_t)handle) @@ -2388,6 +2389,37 @@ int link(const char *oldpath, const char *newpath) return 0; } +enum symlink_type { + SYMLINK_TYPE_UNSPECIFIED = 0, + SYMLINK_TYPE_FILE, + SYMLINK_TYPE_DIRECTORY, +}; + +static enum symlink_type check_symlink_attr(struct index_state *index, const char *link) +{ + static struct attr_check *check; + const char *value; + + if (!index) + return SYMLINK_TYPE_UNSPECIFIED; + + if (!check) + check = attr_check_initl("symlink", NULL); + + git_check_attr(index, link, check); + + value = check->items[0].value; + if (ATTR_UNSET(value)) + return SYMLINK_TYPE_UNSPECIFIED; + if (!strcmp(value, "file")) + return SYMLINK_TYPE_FILE; + if (!strcmp(value, "dir") || !strcmp(value, "directory")) + return SYMLINK_TYPE_DIRECTORY; + + warning(_("ignoring invalid symlink type '%s' for '%s'"), value, link); + return SYMLINK_TYPE_UNSPECIFIED; +} + int mingw_create_symlink(struct index_state *index, const char *target, const char *link) { wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; @@ -2408,7 +2440,31 @@ int mingw_create_symlink(struct index_state *index, const char *target, const ch if (wtarget[len] == '/') wtarget[len] = '\\'; - return create_phantom_symlink(wtarget, wlink); + switch (check_symlink_attr(index, link)) { + case SYMLINK_TYPE_UNSPECIFIED: + /* Create a phantom symlink: it is initially created as a file + * symlink, but may change to a directory symlink later if/when + * the target exists. */ + return create_phantom_symlink(wtarget, wlink); + case SYMLINK_TYPE_FILE: + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) + break; + return 0; + case SYMLINK_TYPE_DIRECTORY: + if (!CreateSymbolicLinkW(wlink, wtarget, + symlink_directory_flags)) + break; + /* There may be dangling phantom symlinks that point at this + * one, which should now morph into directory symlinks. */ + process_phantom_symlinks(); + return 0; + default: + BUG("unhandled symlink type"); + } + + /* CreateSymbolicLinkW failed. */ + errno = err_win_to_posix(GetLastError()); + return -1; } #ifndef _WINNT_H