From 57c4bd7855a2c459074a7ce59a7522c7d4f0143c Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 1/4] pack-objects (mingw): demonstrate a segmentation fault with large deltas There is a problem in the way 9ac3f0e5b3e4 (pack-objects: fix performance issues on packing large deltas, 2018-07-22) initializes that mutex in the `packing_data` struct. The problem manifests in a segmentation fault on Windows, when a mutex (AKA critical section) is accessed without being initialized. (With pthreads, you apparently do not really have to initialize them?) This was reported in https://github.com/git-for-windows/git/issues/1839. Signed-off-by: Johannes Schindelin --- t/t7422-submodule-long-path.sh | 105 +++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100755 t/t7422-submodule-long-path.sh diff --git a/t/t7422-submodule-long-path.sh b/t/t7422-submodule-long-path.sh new file mode 100755 index 0000000000..1c03c76080 --- /dev/null +++ b/t/t7422-submodule-long-path.sh @@ -0,0 +1,105 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +longpath=${longpath:0:90}; export longpath + +test_expect_failure 'submodule with a long path' ' + GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ + git -c init.defaultBranch=long init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git checkout -B long && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin long + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout long && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ + git -c init.defaultBranch=long init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git checkout -B long && + git submodule add ../parent foo && + git submodule update --init --recursive && + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin long + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From 6ef2eaaace1e8735c3b2b68b2f3cf44085557c63 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 28 Jul 2015 21:07:41 +0200 Subject: [PATCH 2/4] mingw: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts, added support for chdir(), etc] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Original-test-by: Andrey Rogozhnikov Signed-off-by: Karsten Blees Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- Documentation/config/core.txt | 7 ++ compat/mingw.c | 148 ++++++++++++++++++++++++++------- compat/mingw.h | 75 +++++++++++++++-- compat/win32/dirent.c | 16 ++-- compat/win32/fscache.c | 16 ++-- t/t2031-checkout-long-paths.sh | 102 +++++++++++++++++++++++ t/t7422-submodule-long-path.sh | 24 +++--- 7 files changed, 327 insertions(+), 61 deletions(-) create mode 100755 t/t2031-checkout-long-paths.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 66dc907000..2db1633425 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -598,6 +598,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/compat/mingw.c b/compat/mingw.c index 67a1e902d0..ef4256dad0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -231,6 +231,7 @@ static int core_restrict_inherited_handles = -1; static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; int core_fscache; +int core_long_paths; int mingw_core_config(const char *var, const char *value, void *cb) { @@ -247,6 +248,11 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); @@ -293,8 +299,8 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; if (DeleteFileW(wpathname)) @@ -326,7 +332,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -347,8 +353,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -427,15 +433,18 @@ static int set_hidden_flag(const wchar_t *path, int set) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; + wchar_t wpath[MAX_LONG_PATH]; if (!is_valid_win32_path(path, 0)) { errno = EINVAL; return -1; } - if (xutftowcs_path(wpath, path) < 0) + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); @@ -521,7 +530,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd, create = (oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL); - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; open_fn_t open_fn; va_start(args, oflags); @@ -540,7 +549,7 @@ int mingw_open (const char *filename, int oflags, ...) if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); - else if (xutftowcs_path(wfilename, filename) < 0) + else if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = open_fn(wfilename, oflags, mode); @@ -598,14 +607,14 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); else if (!is_valid_win32_path(filename, 1)) { int create = otype && strchr(otype, 'w'); errno = create ? EINVAL : ENOENT; return NULL; - } else if (xutftowcs_path(wfilename, filename) < 0) + } else if (xutftowcs_long_path(wfilename, filename) < 0) return NULL; if (xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) @@ -627,14 +636,14 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); else if (!is_valid_win32_path(filename, 1)) { int create = otype && strchr(otype, 'w'); errno = create ? EINVAL : ENOENT; return NULL; - } else if (xutftowcs_path(wfilename, filename) < 0) + } else if (xutftowcs_long_path(wfilename, filename) < 0) return NULL; if (xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) @@ -691,27 +700,33 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; if (!strcmp("nul", filename) || !strcmp("/dev/null", filename)) return 0; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { - wchar_t wdirname[MAX_PATH]; - if (xutftowcs_path(wdirname, dirname) < 0) + int result; + wchar_t wdirname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -759,8 +774,8 @@ static int has_valid_directory_prefix(wchar_t *wfilename) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -931,8 +946,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -1002,6 +1017,7 @@ char *mingw_mktemp(char *template) wchar_t wtemplate[MAX_PATH]; int offset = 0; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; @@ -1633,6 +1649,10 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen if (*argv && !strcmp(cmd, *argv)) wcmd[0] = L'\0'; + /* + * Paths to executables and to the current directory do not support + * long paths, therefore we cannot use xutftowcs_long_path() here. + */ else if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -2284,8 +2304,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -2599,9 +2620,9 @@ int mingw_raise(int sig) int link(const char *oldpath, const char *newpath) { - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!CreateHardLinkW(wnewpath, woldpath, NULL)) { @@ -2669,8 +2690,8 @@ int mingw_is_mount_point(struct strbuf *path) { WIN32_FIND_DATAW findbuf = { 0 }; HANDLE handle; - wchar_t wfilename[MAX_PATH]; - int wlen = xutftowcs_path(wfilename, path->buf); + wchar_t wfilename[MAX_LONG_PATH]; + int wlen = xutftowcs_long_path(wfilename, path->buf); if (wlen < 0) die(_("could not get long path for '%s'"), path->buf); @@ -3034,6 +3055,68 @@ not_a_reserved_name: } } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + #if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from @@ -3195,6 +3278,9 @@ int wmain(int argc, const wchar_t **wargv) /* initialize Unicode console */ winansi_init(); + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + /* invoke the real main() using our utf8 version of argv. */ exit_status = main(argc, argv); diff --git a/compat/mingw.h b/compat/mingw.h index 8f21e2c14c..92fd55e56e 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -12,6 +12,7 @@ typedef _sigset_t sigset_t; #endif extern int core_fscache; +extern int core_long_paths; int mingw_core_config(const char *var, const char *value, void *cb); #define platform_core_config mingw_core_config @@ -501,6 +502,42 @@ char *mingw_strbuf_realpath(struct strbuf *resolved, const char *path); int is_valid_win32_path(const char *path, int allow_literal_nul); #define is_valid_path(path) is_valid_win32_path(path, 0) +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -558,17 +595,45 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) return xutftowcsn(wcs, utf, wcslen, -1); } +/** + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). + */ +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) +{ + int result = xutftowcsn(wcs, utf, wcslen, utflen); + if (result < 0 && errno == ERANGE) + errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); + return result; +} + /** * Simplified file system specific variant of xutftowcsn, assumes output * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... */ static inline int xutftowcs_path(wchar_t *wcs, const char *utf) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); - if (result < 0 && errno == ERANGE) - errno = ENAMETOOLONG; - return result; + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); } /** diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 53b6715bf0..c693886b42 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -62,19 +62,23 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, core_long_paths)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; @@ -87,7 +91,7 @@ DIR *dirent_opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(dirent_DIR) + MAX_PATH); + dir = xmalloc(sizeof(dirent_DIR) + MAX_LONG_PATH); dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; dir->dd_handle = h; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index ac527c56ba..b3319ff936 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -73,7 +73,7 @@ struct fsentry { struct heap_fsentry { struct fsentry ent; - char dummy[MAX_PATH]; + char dummy[MAX_LONG_PATH]; }; /* @@ -115,7 +115,7 @@ static void fsentry_init(struct fsentry *fse, struct fsentry *list, const char *name, size_t len) { fse->list = list; - if (len > MAX_PATH) + if (len > MAX_LONG_PATH) BUG("Trying to allocate fsentry for long path '%.*s'", (int)len, name); memcpy(fse->dirent.d_name, name, len); @@ -216,7 +216,7 @@ static struct fsentry *fseentry_create_entry(struct fscache *cache, static struct fsentry *fsentry_create_list(struct fscache *cache, const struct fsentry *dir, int *dir_not_found) { - wchar_t pattern[MAX_PATH]; + wchar_t pattern[MAX_LONG_PATH]; NTSTATUS status; IO_STATUS_BLOCK iosb; PFILE_FULL_DIR_INFORMATION di; @@ -227,13 +227,11 @@ static struct fsentry *fsentry_create_list(struct fscache *cache, const struct f *dir_not_found = 0; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->dirent.d_name, MAX_PATH, - dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->dirent.d_name, + MAX_LONG_PATH, dir->len, MAX_PATH - 2, + core_long_paths)) < 0) return NULL; - } /* handle CWD */ if (!wlen) { diff --git a/t/t2031-checkout-long-paths.sh b/t/t2031-checkout-long-paths.sh new file mode 100755 index 0000000000..f30f8920ca --- /dev/null +++ b/t/t2031-checkout-long-paths.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test ! -d longpa* +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa*/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >>$(ls longpa*/longtestfile) && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + if test -z "$debug" + then + rm -rf longpa~1 + fi +' + +# check that the template used in the test won't be too long: +abspath="$(pwd)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7422-submodule-long-path.sh b/t/t7422-submodule-long-path.sh index 1c03c76080..0da221b45a 100755 --- a/t/t7422-submodule-long-path.sh +++ b/t/t7422-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -longpath=${longpath:0:90}; export longpath +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ git -c init.defaultBranch=long init --bare remote && test_create_repo bundle1 && @@ -58,7 +63,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ git -c init.defaultBranch=long init --bare super && test_create_repo child && @@ -100,6 +105,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From 1ac27852ff2a885ee5ea02a3f2607e49fa74033e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 5 Jul 2014 00:00:36 +0200 Subject: [PATCH 3/4] Win32: fix 'lstat("dir/")' with long paths Use a suffciently large buffer to strip the trailing slash. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index ef4256dad0..5915691ab1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -846,7 +846,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) static int do_stat_internal(int follow, const char *file_name, struct stat *buf) { int namelen; - char alt_name[PATH_MAX]; + char alt_name[MAX_LONG_PATH]; if (!do_lstat(follow, file_name, buf)) return 0; @@ -862,7 +862,7 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return -1; while (namelen && file_name[namelen-1] == '/') --namelen; - if (!namelen || namelen >= PATH_MAX) + if (!namelen || namelen >= MAX_LONG_PATH) return -1; memcpy(alt_name, file_name, namelen); From 6097142dc229dd176c1954413618cad0554eddba Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Aug 2017 01:28:22 +0200 Subject: [PATCH 4/4] mingw: ensure that core.longPaths is handled *always* A ton of Git commands simply do not read (or at least parse) the core.* settings. This is not good, as Git for Windows relies on the core.longPaths setting to be read quite early on. So let's just make sure that all commands read the config and give platform_core_config() a chance. This patch teaches tons of Git commands to respect the config setting `core.longPaths = true`, including `pack-refs`, thereby fixing https://github.com/git-for-windows/git/issues/1218 Signed-off-by: Johannes Schindelin --- builtin/archive.c | 2 ++ builtin/bisect--helper.c | 2 ++ builtin/bundle.c | 2 ++ builtin/check-ref-format.c | 2 ++ builtin/clone.c | 2 ++ builtin/column.c | 2 ++ builtin/credential-store.c | 3 +++ builtin/fetch-pack.c | 2 ++ builtin/get-tar-commit-id.c | 2 ++ builtin/log.c | 1 + builtin/ls-remote.c | 2 ++ builtin/mailinfo.c | 2 ++ builtin/mailsplit.c | 2 ++ builtin/merge-index.c | 3 +++ builtin/merge-tree.c | 2 ++ builtin/mktag.c | 1 + builtin/mktree.c | 2 ++ builtin/pack-refs.c | 1 + builtin/prune-packed.c | 2 ++ builtin/prune.c | 3 +++ builtin/reflog.c | 1 + builtin/remote-ext.c | 2 ++ builtin/remote.c | 1 + builtin/rev-parse.c | 1 + builtin/show-index.c | 2 ++ builtin/show-ref.c | 1 + builtin/stripspace.c | 5 ++--- builtin/submodule--helper.c | 1 + builtin/upload-archive.c | 3 +++ http-backend.c | 1 + refs.c | 2 +- 31 files changed, 56 insertions(+), 4 deletions(-) diff --git a/builtin/archive.c b/builtin/archive.c index 45d11669aa..708243cd7d 100644 --- a/builtin/archive.c +++ b/builtin/archive.c @@ -9,6 +9,7 @@ #include "parse-options.h" #include "pkt-line.h" #include "sideband.h" +#include "config.h" static void create_output_file(const char *output_file) { @@ -95,6 +96,7 @@ int cmd_archive(int argc, const char **argv, const char *prefix) OPT_END() }; + git_config(git_default_config, NULL); argc = parse_options(argc, argv, prefix, local_opts, NULL, PARSE_OPT_KEEP_ALL); diff --git a/builtin/bisect--helper.c b/builtin/bisect--helper.c index 9d9540a0ab..c20df85cfd 100644 --- a/builtin/bisect--helper.c +++ b/builtin/bisect--helper.c @@ -9,6 +9,7 @@ #include "prompt.h" #include "quote.h" #include "revision.h" +#include "config.h" static GIT_PATH_FUNC(git_path_bisect_terms, "BISECT_TERMS") static GIT_PATH_FUNC(git_path_bisect_expected_rev, "BISECT_EXPECTED_REV") @@ -1074,6 +1075,7 @@ int cmd_bisect__helper(int argc, const char **argv, const char *prefix) }; struct bisect_terms terms = { .term_good = NULL, .term_bad = NULL }; + git_config(git_default_config, NULL); argc = parse_options(argc, argv, prefix, options, git_bisect_helper_usage, PARSE_OPT_KEEP_DASHDASH | PARSE_OPT_KEEP_UNKNOWN); diff --git a/builtin/bundle.c b/builtin/bundle.c index ea6948110b..a9c2b849c1 100644 --- a/builtin/bundle.c +++ b/builtin/bundle.c @@ -3,6 +3,7 @@ #include "parse-options.h" #include "cache.h" #include "bundle.h" +#include "config.h" /* * Basic handler for bundle files to connect repositories via sneakernet. @@ -109,6 +110,7 @@ static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) { }; const char* bundle_file; + git_config(git_default_config, NULL); argc = parse_options_cmd_bundle(argc, argv, prefix, builtin_bundle_verify_usage, options, &bundle_file); /* bundle internals use argv[1] as further parameters */ diff --git a/builtin/check-ref-format.c b/builtin/check-ref-format.c index bc67d3f0a8..abee1be472 100644 --- a/builtin/check-ref-format.c +++ b/builtin/check-ref-format.c @@ -6,6 +6,7 @@ #include "refs.h" #include "builtin.h" #include "strbuf.h" +#include "config.h" static const char builtin_check_ref_format_usage[] = "git check-ref-format [--normalize] [] \n" @@ -58,6 +59,7 @@ int cmd_check_ref_format(int argc, const char **argv, const char *prefix) int flags = 0; const char *refname; + git_config(git_default_config, NULL); if (argc == 2 && !strcmp(argv[1], "-h")) usage(builtin_check_ref_format_usage); diff --git a/builtin/clone.c b/builtin/clone.c index eeb74c0217..73d815fd6d 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -990,6 +990,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) struct transport_ls_refs_options transport_ls_refs_options = TRANSPORT_LS_REFS_OPTIONS_INIT; + git_config(platform_core_config, NULL); + packet_trace_identity("clone"); git_config(git_clone_config, NULL); diff --git a/builtin/column.c b/builtin/column.c index 40d4b3bee2..9898d9adba 100644 --- a/builtin/column.c +++ b/builtin/column.c @@ -34,6 +34,8 @@ int cmd_column(int argc, const char **argv, const char *prefix) OPT_END() }; + git_config(platform_core_config, NULL); + /* This one is special and must be the first one */ if (argc > 1 && starts_with(argv[1], "--command=")) { command = argv[1] + 10; diff --git a/builtin/credential-store.c b/builtin/credential-store.c index ae3c1ba75f..8f22ae12a4 100644 --- a/builtin/credential-store.c +++ b/builtin/credential-store.c @@ -4,6 +4,7 @@ #include "credential.h" #include "string-list.h" #include "parse-options.h" +#include "config.h" static struct lock_file credential_lock; @@ -165,6 +166,8 @@ int cmd_credential_store(int argc, const char **argv, const char *prefix) umask(077); + git_config(git_default_config, NULL); + argc = parse_options(argc, (const char **)argv, prefix, options, usage, 0); if (argc != 1) usage_with_options(usage, options); diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index c2d96f4c89..dbd0a6c1be 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -5,6 +5,7 @@ #include "connect.h" #include "oid-array.h" #include "protocol.h" +#include "config.h" static const char fetch_pack_usage[] = "git fetch-pack [--all] [--stdin] [--quiet | -q] [--keep | -k] [--thin] " @@ -57,6 +58,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct packet_reader reader; enum protocol_version version; + git_config(git_default_config, NULL); fetch_if_missing = 0; packet_trace_identity("fetch-pack"); diff --git a/builtin/get-tar-commit-id.c b/builtin/get-tar-commit-id.c index 491af9202d..a623518ae7 100644 --- a/builtin/get-tar-commit-id.c +++ b/builtin/get-tar-commit-id.c @@ -6,6 +6,7 @@ #include "tar.h" #include "builtin.h" #include "quote.h" +#include "config.h" static const char builtin_get_tar_commit_id_usage[] = "git get-tar-commit-id"; @@ -27,6 +28,7 @@ int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix) if (argc != 1) usage(builtin_get_tar_commit_id_usage); + git_config(git_default_config, NULL); n = read_in_full(0, buffer, HEADERSIZE); if (n < 0) die_errno("git get-tar-commit-id: read error"); diff --git a/builtin/log.c b/builtin/log.c index 6102893fcc..c840041ec5 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -2296,6 +2296,7 @@ int cmd_cherry(int argc, const char **argv, const char *prefix) OPT_END() }; + git_config(git_default_config, NULL); argc = parse_options(argc, argv, prefix, options, cherry_usage, 0); switch (argc) { diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index 1794548c71..3cc501f998 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -4,6 +4,7 @@ #include "ref-filter.h" #include "remote.h" #include "refs.h" +#include "config.h" static const char * const ls_remote_usage[] = { N_("git ls-remote [--heads] [--tags] [--refs] [--upload-pack=]\n" @@ -86,6 +87,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) UNLEAK(sorting); + git_config(git_default_config, NULL); if (argc > 1) { int i; CALLOC_ARRAY(pattern, argc); diff --git a/builtin/mailinfo.c b/builtin/mailinfo.c index 01d16ef9e5..df6cc41721 100644 --- a/builtin/mailinfo.c +++ b/builtin/mailinfo.c @@ -8,6 +8,7 @@ #include "strbuf.h" #include "mailinfo.h" #include "parse-options.h" +#include "config.h" static const char * const mailinfo_usage[] = { /* TRANSLATORS: keep <> in "<" mail ">" info. */ @@ -78,6 +79,7 @@ int cmd_mailinfo(int argc, const char **argv, const char *prefix) OPT_END() }; + git_config(git_default_config, NULL); setup_mailinfo(&mi); meta_charset.policy = CHARSET_DEFAULT; diff --git a/builtin/mailsplit.c b/builtin/mailsplit.c index 664400b816..472d2eb8a4 100644 --- a/builtin/mailsplit.c +++ b/builtin/mailsplit.c @@ -8,6 +8,7 @@ #include "builtin.h" #include "string-list.h" #include "strbuf.h" +#include "config.h" static const char git_mailsplit_usage[] = "git mailsplit [-d] [-f] [-b] [--keep-cr] -o [(|)...]"; @@ -276,6 +277,7 @@ int cmd_mailsplit(int argc, const char **argv, const char *prefix) const char **argp; static const char *stdin_only[] = { "-", NULL }; + git_config(git_default_config, NULL); for (argp = argv+1; *argp; argp++) { const char *arg = *argp; diff --git a/builtin/merge-index.c b/builtin/merge-index.c index c0383fe9df..24e5c1f6ea 100644 --- a/builtin/merge-index.c +++ b/builtin/merge-index.c @@ -1,6 +1,7 @@ #define USE_THE_INDEX_COMPATIBILITY_MACROS #include "builtin.h" #include "run-command.h" +#include "config.h" static const char *pgm; static int one_shot, quiet; @@ -77,6 +78,8 @@ int cmd_merge_index(int argc, const char **argv, const char *prefix) */ signal(SIGCHLD, SIG_DFL); + git_config(git_default_config, NULL); + if (argc < 3) usage("git merge-index [-o] [-q] (-a | [--] [...])"); diff --git a/builtin/merge-tree.c b/builtin/merge-tree.c index de8520778d..a3d4f02472 100644 --- a/builtin/merge-tree.c +++ b/builtin/merge-tree.c @@ -7,6 +7,7 @@ #include "blob.h" #include "exec-cmd.h" #include "merge-blobs.h" +#include "config.h" static const char merge_tree_usage[] = "git merge-tree "; @@ -378,6 +379,7 @@ int cmd_merge_tree(int argc, const char **argv, const char *prefix) if (argc != 4) usage(merge_tree_usage); + git_config(git_default_config, NULL); buf1 = get_tree_descriptor(r, t+0, argv[1]); buf2 = get_tree_descriptor(r, t+1, argv[2]); buf3 = get_tree_descriptor(r, t+2, argv[3]); diff --git a/builtin/mktag.c b/builtin/mktag.c index dddcccdd36..9dc048a9a2 100644 --- a/builtin/mktag.c +++ b/builtin/mktag.c @@ -84,6 +84,7 @@ int cmd_mktag(int argc, const char **argv, const char *prefix) builtin_mktag_options, builtin_mktag_usage, 0); + git_config(git_default_config, NULL); if (strbuf_read(&buf, 0, 0) < 0) die_errno(_("could not read from stdin")); diff --git a/builtin/mktree.c b/builtin/mktree.c index 891991b00d..ed6fae0460 100644 --- a/builtin/mktree.c +++ b/builtin/mktree.c @@ -8,6 +8,7 @@ #include "tree.h" #include "parse-options.h" #include "object-store.h" +#include "config.h" static struct treeent { unsigned mode; @@ -157,6 +158,7 @@ int cmd_mktree(int ac, const char **av, const char *prefix) OPT_END() }; + git_config(git_default_config, NULL); ac = parse_options(ac, av, prefix, option, mktree_usage, 0); getline_fn = nul_term_line ? strbuf_getline_nul : strbuf_getline_lf; diff --git a/builtin/pack-refs.c b/builtin/pack-refs.c index cfbd5c36c7..28f79b9938 100644 --- a/builtin/pack-refs.c +++ b/builtin/pack-refs.c @@ -3,6 +3,7 @@ #include "parse-options.h" #include "refs.h" #include "repository.h" +#include "config.h" static char const * const pack_refs_usage[] = { N_("git pack-refs []"), diff --git a/builtin/prune-packed.c b/builtin/prune-packed.c index b7b9281a8c..83c4fd17a6 100644 --- a/builtin/prune-packed.c +++ b/builtin/prune-packed.c @@ -1,6 +1,7 @@ #include "builtin.h" #include "parse-options.h" #include "prune-packed.h" +#include "config.h" static const char * const prune_packed_usage[] = { N_("git prune-packed [-n | --dry-run] [-q | --quiet]"), @@ -18,6 +19,7 @@ int cmd_prune_packed(int argc, const char **argv, const char *prefix) OPT_END() }; + git_config(git_default_config, NULL); argc = parse_options(argc, argv, prefix, prune_packed_options, prune_packed_usage, 0); diff --git a/builtin/prune.c b/builtin/prune.c index 02c6ab7cba..847864dbe5 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -9,6 +9,7 @@ #include "prune-packed.h" #include "object-store.h" #include "shallow.h" +#include "config.h" static const char * const prune_usage[] = { N_("git prune [-n] [-v] [--progress] [--expire