Merge branch 'long-paths'

This commit is contained in:
Johannes Schindelin 2018-11-15 12:23:43 +01:00
commit 9dfe22e227
37 changed files with 1750 additions and 149 deletions

View File

@ -64,6 +64,9 @@ all advice messages.
set their identity configuration.
mergeConflict::
Shown when various commands stop because of conflicts.
nameTooLong::
Advice shown if a filepath operation is attempted where the
path was too long.
nestedTag::
Shown when a user attempts to recursively tag a tag object.
pushAlreadyExists::

View File

@ -710,6 +710,19 @@ relatively high IO latencies. When enabled, Git will do the
index comparison to the filesystem data in parallel, allowing
overlapping IO's. Defaults to true.
core.fscache::
Enable additional caching of file system data for some operations.
+
Git for Windows uses this to bulk-read and cache lstat data of entire
directories (instead of doing lstat file by file).
core.longpaths::
Enable long path (> 260) support for builtin commands in Git for
Windows. This is disabled by default, as long paths are not supported
by Windows Explorer, cmd.exe and the Git for Windows tool chain
(msys, bash, tcl, perl...). Only enable this if you know what you're
doing and are prepared to live with a few quirks.
core.unsetenvvars::
Windows-only: comma-separated list of environment variables'
names that need to be unset before spawning any other process.

View File

@ -61,6 +61,7 @@ static struct {
[ADVICE_IGNORED_HOOK] = { "ignoredHook" },
[ADVICE_IMPLICIT_IDENTITY] = { "implicitIdentity" },
[ADVICE_MERGE_CONFLICT] = { "mergeConflict" },
[ADVICE_NAME_TOO_LONG] = { "nameTooLong" },
[ADVICE_NESTED_TAG] = { "nestedTag" },
[ADVICE_OBJECT_NAME_WARNING] = { "objectNameWarning" },
[ADVICE_PUSH_ALREADY_EXISTS] = { "pushAlreadyExists" },

View File

@ -28,6 +28,7 @@ enum advice_type {
ADVICE_IGNORED_HOOK,
ADVICE_IMPLICIT_IDENTITY,
ADVICE_MERGE_CONFLICT,
ADVICE_NAME_TOO_LONG,
ADVICE_NESTED_TAG,
ADVICE_OBJECT_NAME_WARNING,
ADVICE_PUSH_ALREADY_EXISTS,

View File

@ -493,6 +493,10 @@ int cmd_add(int argc,
die_in_unpopulated_submodule(repo->index, prefix);
die_path_inside_submodule(repo->index, &pathspec);
enable_fscache(0);
/* We do not really re-read the index but update the up-to-date flags */
preload_index(repo->index, &pathspec, 0);
if (add_new_files) {
int baselen;
@ -605,5 +609,6 @@ finish:
free(ps_matched);
dir_clear(&dir);
clear_pathspec(&pathspec);
enable_fscache(0);
return exit_status;
}

View File

@ -415,6 +415,7 @@ static int checkout_worktree(const struct checkout_opts *opts,
if (pc_workers > 1)
init_parallel_checkout();
enable_fscache(the_repository->index->cache_nr);
for (pos = 0; pos < the_repository->index->cache_nr; pos++) {
struct cache_entry *ce = the_repository->index->cache[pos];
if (ce->ce_flags & CE_MATCHED) {
@ -440,6 +441,7 @@ static int checkout_worktree(const struct checkout_opts *opts,
errs |= run_parallel_checkout(&state, pc_workers, pc_threshold,
NULL, NULL);
mem_pool_discard(&ce_mem_pool, should_validate_cache_entries());
disable_fscache();
remove_marked_cache_entries(the_repository->index, 1);
remove_scheduled_dirs();
errs |= finish_delayed_checkout(&state, opts->show_progress);

View File

@ -26,6 +26,7 @@
#include "pathspec.h"
#include "help.h"
#include "prompt.h"
#include "advice.h"
static int require_force = -1; /* unset */
static int interactive;
@ -221,6 +222,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag,
quote_path(path->buf, prefix, &quoted, 0);
errno = saved_errno;
warning_errno(_(msg_warn_remove_failed), quoted.buf);
if (saved_errno == ENAMETOOLONG) {
advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed."));
}
*dir_gone = 0;
}
ret = res;
@ -256,6 +260,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag,
quote_path(path->buf, prefix, &quoted, 0);
errno = saved_errno;
warning_errno(_(msg_warn_remove_failed), quoted.buf);
if (saved_errno == ENAMETOOLONG) {
advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed."));
}
*dir_gone = 0;
ret = 1;
}
@ -299,6 +306,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag,
quote_path(path->buf, prefix, &quoted, 0);
errno = saved_errno;
warning_errno(_(msg_warn_remove_failed), quoted.buf);
if (saved_errno == ENAMETOOLONG) {
advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed."));
}
*dir_gone = 0;
ret = 1;
}
@ -1042,6 +1052,7 @@ int cmd_clean(int argc,
if (repo_read_index(the_repository) < 0)
die(_("index file corrupt"));
enable_fscache(the_repository->index->cache_nr);
pl = add_pattern_list(&dir, EXC_CMDL, "--exclude option");
for (i = 0; i < exclude_list.nr; i++)
@ -1108,6 +1119,9 @@ int cmd_clean(int argc,
qname = quote_path(item->string, NULL, &buf, 0);
errno = saved_errno;
warning_errno(_(msg_warn_remove_failed), qname);
if (saved_errno == ENAMETOOLONG) {
advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed."));
}
errors++;
} else if (!quiet) {
qname = quote_path(item->string, NULL, &buf, 0);
@ -1116,6 +1130,7 @@ int cmd_clean(int argc,
}
}
disable_fscache();
strbuf_release(&abs_path);
strbuf_release(&buf);
string_list_clear(&del_list, 0);

View File

@ -1623,6 +1623,7 @@ struct repository *repo UNUSED)
PATHSPEC_PREFER_FULL,
prefix, argv);
enable_fscache(0);
if (status_format != STATUS_FORMAT_PORCELAIN &&
status_format != STATUS_FORMAT_PORCELAIN_V2)
progress_flag = REFRESH_PROGRESS;
@ -1663,6 +1664,7 @@ struct repository *repo UNUSED)
wt_status_print(&s);
wt_status_collect_free_buffers(&s);
disable_fscache();
return 0;
}

View File

@ -34,7 +34,7 @@ struct fsm_health_data
struct wt_moved
{
wchar_t wpath[MAX_PATH + 1];
wchar_t wpath[MAX_LONG_PATH + 1];
BY_HANDLE_FILE_INFORMATION bhfi;
} wt_moved;
};
@ -143,8 +143,8 @@ static int has_worktree_moved(struct fsmonitor_daemon_state *state,
return 0;
case CTX_INIT:
if (xutftowcs_path(data->wt_moved.wpath,
state->path_worktree_watch.buf) < 0) {
if (xutftowcs_long_path(data->wt_moved.wpath,
state->path_worktree_watch.buf) < 0) {
error(_("could not convert to wide characters: '%s'"),
state->path_worktree_watch.buf);
return -1;

View File

@ -28,7 +28,7 @@ struct one_watch
DWORD count;
struct strbuf path;
wchar_t wpath_longname[MAX_PATH + 1];
wchar_t wpath_longname[MAX_LONG_PATH + 1];
DWORD wpath_longname_len;
HANDLE hDir;
@ -131,8 +131,8 @@ normalize:
*/
static void check_for_shortnames(struct one_watch *watch)
{
wchar_t buf_in[MAX_PATH + 1];
wchar_t buf_out[MAX_PATH + 1];
wchar_t buf_in[MAX_LONG_PATH + 1];
wchar_t buf_out[MAX_LONG_PATH + 1];
wchar_t *last;
wchar_t *p;
@ -197,8 +197,8 @@ static enum get_relative_result get_relative_longname(
const wchar_t *wpath, DWORD wpath_len,
wchar_t *wpath_longname, size_t bufsize_wpath_longname)
{
wchar_t buf_in[2 * MAX_PATH + 1];
wchar_t buf_out[MAX_PATH + 1];
wchar_t buf_in[2 * MAX_LONG_PATH + 1];
wchar_t buf_out[MAX_LONG_PATH + 1];
DWORD root_len;
DWORD out_len;
@ -298,10 +298,10 @@ static struct one_watch *create_watch(const char *path)
FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE;
HANDLE hDir;
DWORD len_longname;
wchar_t wpath[MAX_PATH + 1];
wchar_t wpath_longname[MAX_PATH + 1];
wchar_t wpath[MAX_LONG_PATH + 1];
wchar_t wpath_longname[MAX_LONG_PATH + 1];
if (xutftowcs_path(wpath, path) < 0) {
if (xutftowcs_long_path(wpath, path) < 0) {
error(_("could not convert to wide characters: '%s'"), path);
return NULL;
}
@ -545,7 +545,7 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state)
struct string_list cookie_list = STRING_LIST_INIT_DUP;
struct fsmonitor_batch *batch = NULL;
const char *p = watch->buffer;
wchar_t wpath_longname[MAX_PATH + 1];
wchar_t wpath_longname[MAX_LONG_PATH + 1];
/*
* If the kernel gets more events than will fit in the kernel

View File

@ -69,8 +69,8 @@ static int check_remote_protocol(wchar_t *wpath)
*/
int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info)
{
wchar_t wpath[MAX_PATH];
wchar_t wfullpath[MAX_PATH];
wchar_t wpath[MAX_LONG_PATH];
wchar_t wfullpath[MAX_LONG_PATH];
size_t wlen;
UINT driveType;
@ -78,7 +78,7 @@ int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info)
* Do everything in wide chars because the drive letter might be
* a multi-byte sequence. See win32_has_dos_drive_prefix().
*/
if (xutftowcs_path(wpath, path) < 0) {
if (xutftowcs_long_path(wpath, path) < 0) {
return -1;
}
@ -97,7 +97,7 @@ int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info)
* slashes to backslashes. This is essential to get GetDriveTypeW()
* correctly handle some UNC "\\server\share\..." paths.
*/
if (!GetFullPathNameW(wpath, MAX_PATH, wfullpath, NULL)) {
if (!GetFullPathNameW(wpath, MAX_LONG_PATH, wfullpath, NULL)) {
return -1;
}

View File

@ -338,6 +338,17 @@ static inline int getrlimit(int resource, struct rlimit *rlp)
return 0;
}
/*
* The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC.
* Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch.
*/
static inline long long filetime_to_hnsec(const FILETIME *ft)
{
long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime;
/* Windows to Unix Epoch conversion */
return winTime - 116444736000000000LL;
}
/*
* Use mingw specific stat()/lstat()/fstat() implementations on Windows,
* including our own struct stat with 64 bit st_size and nanosecond-precision
@ -354,6 +365,13 @@ struct timespec {
#endif
#endif
static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts)
{
long long hnsec = filetime_to_hnsec(ft);
ts->tv_sec = (time_t)(hnsec / 10000000);
ts->tv_nsec = (hnsec % 10000000) * 100;
}
struct mingw_stat {
_dev_t st_dev;
_ino_t st_ino;
@ -386,7 +404,7 @@ int mingw_fstat(int fd, struct stat *buf);
#ifdef lstat
#undef lstat
#endif
#define lstat mingw_lstat
extern int (*lstat)(const char *file_name, struct stat *buf);
int mingw_utime(const char *file_name, const struct utimbuf *times);

View File

@ -14,6 +14,7 @@
#include "symlinks.h"
#include "trace2.h"
#include "win32.h"
#include "win32/fscache.h"
#include "win32/lazyload.h"
#include "wrapper.h"
#include "write-or-die.h"
@ -274,6 +275,28 @@ enum hide_dotfiles_type {
static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY;
static char *unset_environment_variables;
int core_fscache;
int are_long_paths_enabled(void)
{
/* default to `false` during initialization */
static const int fallback = 0;
static int enabled = -1;
if (enabled < 0) {
/* avoid infinite recursion */
if (!the_repository)
return fallback;
if (the_repository->config &&
the_repository->config->hash_initialized &&
repo_config_get_bool(the_repository, "core.longpaths", &enabled) < 0)
enabled = 0;
}
return enabled < 0 ? fallback : enabled;
}
int mingw_core_config(const char *var, const char *value,
const struct config_context *ctx UNUSED,
@ -287,6 +310,11 @@ int mingw_core_config(const char *var, const char *value,
return 0;
}
if (!strcmp(var, "core.fscache")) {
core_fscache = git_config_bool(var, value);
return 0;
}
if (!strcmp(var, "core.unsetenvvars")) {
if (!value)
return config_error_nonbool(var);
@ -350,7 +378,7 @@ process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink)
{
HANDLE hnd;
BY_HANDLE_FILE_INFORMATION fdata;
wchar_t relative[MAX_PATH];
wchar_t relative[MAX_LONG_PATH];
const wchar_t *rel;
/* check that wlink is still a file symlink */
@ -454,8 +482,8 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf)
int mingw_unlink(const char *pathname, int handle_in_use_error)
{
int tries = 0;
wchar_t wpathname[MAX_PATH];
if (xutftowcs_path(wpathname, pathname) < 0)
wchar_t wpathname[MAX_LONG_PATH];
if (xutftowcs_long_path(wpathname, pathname) < 0)
return -1;
if (DeleteFileW(wpathname))
@ -487,7 +515,7 @@ static int is_dir_empty(const wchar_t *wpath)
{
WIN32_FIND_DATAW findbuf;
HANDLE handle;
wchar_t wbuf[MAX_PATH + 2];
wchar_t wbuf[MAX_LONG_PATH + 2];
wcscpy(wbuf, wpath);
wcscat(wbuf, L"\\*");
handle = FindFirstFileW(wbuf, &findbuf);
@ -508,7 +536,7 @@ static int is_dir_empty(const wchar_t *wpath)
int mingw_rmdir(const char *pathname)
{
int tries = 0;
wchar_t wpathname[MAX_PATH];
wchar_t wpathname[MAX_LONG_PATH];
struct stat st;
/*
@ -530,7 +558,7 @@ int mingw_rmdir(const char *pathname)
return -1;
}
if (xutftowcs_path(wpathname, pathname) < 0)
if (xutftowcs_long_path(wpathname, pathname) < 0)
return -1;
do {
@ -599,15 +627,18 @@ static int set_hidden_flag(const wchar_t *path, int set)
int mingw_mkdir(const char *path, int mode UNUSED)
{
int ret;
wchar_t wpath[MAX_PATH];
wchar_t wpath[MAX_LONG_PATH];
if (!is_valid_win32_path(path, 0)) {
errno = EINVAL;
return -1;
}
if (xutftowcs_path(wpath, path) < 0)
/* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */
if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248,
are_long_paths_enabled()) < 0)
return -1;
ret = _wmkdir(wpath);
if (!ret)
process_phantom_symlinks();
@ -773,7 +804,7 @@ int mingw_open (const char *filename, int oflags, ...)
va_list args;
unsigned mode;
int fd, create = (oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL);
wchar_t wfilename[MAX_PATH];
wchar_t wfilename[MAX_LONG_PATH];
open_fn_t open_fn;
WIN32_FILE_ATTRIBUTE_DATA fdata;
@ -806,7 +837,7 @@ int mingw_open (const char *filename, int oflags, ...)
if (filename && !strcmp(filename, "/dev/null"))
wcscpy(wfilename, L"nul");
else if (xutftowcs_path(wfilename, filename) < 0)
else if (xutftowcs_long_path(wfilename, filename) < 0)
return -1;
/*
@ -892,14 +923,14 @@ FILE *mingw_fopen (const char *filename, const char *otype)
{
int hide = needs_hiding(filename);
FILE *file;
wchar_t wfilename[MAX_PATH], wotype[4];
wchar_t wfilename[MAX_LONG_PATH], wotype[4];
if (filename && !strcmp(filename, "/dev/null"))
wcscpy(wfilename, L"nul");
else if (!is_valid_win32_path(filename, 1)) {
int create = otype && strchr(otype, 'w');
errno = create ? EINVAL : ENOENT;
return NULL;
} else if (xutftowcs_path(wfilename, filename) < 0)
} else if (xutftowcs_long_path(wfilename, filename) < 0)
return NULL;
if (xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0)
@ -921,14 +952,14 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream)
{
int hide = needs_hiding(filename);
FILE *file;
wchar_t wfilename[MAX_PATH], wotype[4];
wchar_t wfilename[MAX_LONG_PATH], wotype[4];
if (filename && !strcmp(filename, "/dev/null"))
wcscpy(wfilename, L"nul");
else if (!is_valid_win32_path(filename, 1)) {
int create = otype && strchr(otype, 'w');
errno = create ? EINVAL : ENOENT;
return NULL;
} else if (xutftowcs_path(wfilename, filename) < 0)
} else if (xutftowcs_long_path(wfilename, filename) < 0)
return NULL;
if (xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0)
@ -978,7 +1009,7 @@ ssize_t mingw_write(int fd, const void *buf, size_t len)
HANDLE h = (HANDLE) _get_osfhandle(fd);
if (GetFileType(h) != FILE_TYPE_PIPE) {
if (orig == EINVAL) {
wchar_t path[MAX_PATH];
wchar_t path[MAX_LONG_PATH];
DWORD ret = GetFinalPathNameByHandleW(h, path,
ARRAY_SIZE(path), 0);
UINT drive_type = ret > 0 && ret < ARRAY_SIZE(path) ?
@ -1015,20 +1046,23 @@ ssize_t mingw_write(int fd, const void *buf, size_t len)
int mingw_access(const char *filename, int mode)
{
wchar_t wfilename[MAX_PATH];
wchar_t wfilename[MAX_LONG_PATH];
if (!strcmp("nul", filename) || !strcmp("/dev/null", filename))
return 0;
if (xutftowcs_path(wfilename, filename) < 0)
if (xutftowcs_long_path(wfilename, filename) < 0)
return -1;
/* X_OK is not supported by the MSVCRT version */
return _waccess(wfilename, mode & ~X_OK);
}
/* cached length of current directory for handle_long_path */
static int current_directory_len = 0;
int mingw_chdir(const char *dirname)
{
wchar_t wdirname[MAX_PATH];
if (xutftowcs_path(wdirname, dirname) < 0)
int result;
wchar_t wdirname[MAX_LONG_PATH];
if (xutftowcs_long_path(wdirname, dirname) < 0)
return -1;
if (has_symlinks) {
@ -1047,35 +1081,19 @@ int mingw_chdir(const char *dirname)
CloseHandle(hnd);
}
return _wchdir(normalize_ntpath(wdirname));
result = _wchdir(normalize_ntpath(wdirname));
current_directory_len = GetCurrentDirectoryW(0, NULL);
return result;
}
int mingw_chmod(const char *filename, int mode)
{
wchar_t wfilename[MAX_PATH];
if (xutftowcs_path(wfilename, filename) < 0)
wchar_t wfilename[MAX_LONG_PATH];
if (xutftowcs_long_path(wfilename, filename) < 0)
return -1;
return _wchmod(wfilename, mode);
}
/*
* The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC.
* Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch.
*/
static inline long long filetime_to_hnsec(const FILETIME *ft)
{
long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime;
/* Windows to Unix Epoch conversion */
return winTime - 116444736000000000LL;
}
static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts)
{
long long hnsec = filetime_to_hnsec(ft);
ts->tv_sec = (time_t)(hnsec / 10000000);
ts->tv_nsec = (hnsec % 10000000) * 100;
}
/**
* Verifies that safe_create_leading_directories() would succeed.
*/
@ -1205,8 +1223,8 @@ int mingw_lstat(const char *file_name, struct stat *buf)
WIN32_FILE_ATTRIBUTE_DATA fdata;
DWORD reparse_tag = 0;
int link_len = 0;
wchar_t wfilename[MAX_PATH];
int wlen = xutftowcs_path(wfilename, file_name);
wchar_t wfilename[MAX_LONG_PATH];
int wlen = xutftowcs_long_path(wfilename, file_name);
if (wlen < 0)
return -1;
@ -1221,7 +1239,7 @@ int mingw_lstat(const char *file_name, struct stat *buf)
if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) {
/* for reparse points, get the link tag and length */
if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) {
char tmpbuf[MAX_PATH];
char tmpbuf[MAX_LONG_PATH];
if (read_reparse_point(wfilename, FALSE, tmpbuf,
&link_len, &reparse_tag) < 0)
@ -1268,6 +1286,8 @@ int mingw_lstat(const char *file_name, struct stat *buf)
return -1;
}
int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat;
static int get_file_info_by_handle(HANDLE hnd, struct stat *buf)
{
BY_HANDLE_FILE_INFORMATION fdata;
@ -1293,12 +1313,12 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf)
int mingw_stat(const char *file_name, struct stat *buf)
{
wchar_t wfile_name[MAX_PATH];
wchar_t wfile_name[MAX_LONG_PATH];
HANDLE hnd;
int result;
/* open the file and let Windows resolve the links */
if (xutftowcs_path(wfile_name, file_name) < 0)
if (xutftowcs_long_path(wfile_name, file_name) < 0)
return -1;
hnd = CreateFileW(wfile_name, 0,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL,
@ -1366,10 +1386,10 @@ int mingw_utime (const char *file_name, const struct utimbuf *times)
FILETIME mft, aft;
int rc;
DWORD attrs;
wchar_t wfilename[MAX_PATH];
wchar_t wfilename[MAX_LONG_PATH];
HANDLE osfilehandle;
if (xutftowcs_path(wfilename, file_name) < 0)
if (xutftowcs_long_path(wfilename, file_name) < 0)
return -1;
/* must have write permission */
@ -2073,6 +2093,10 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen
if (*argv && !strcmp(cmd, *argv))
wcmd[0] = L'\0';
/*
* Paths to executables and to the current directory do not support
* long paths, therefore we cannot use xutftowcs_long_path() here.
*/
else if (xutftowcs_path(wcmd, cmd) < 0)
return -1;
if (dir && xutftowcs_path(wdir, dir) < 0)
@ -2762,12 +2786,12 @@ int mingw_rename(const char *pold, const char *pnew)
static int supports_file_rename_info_ex = 1;
DWORD attrs = INVALID_FILE_ATTRIBUTES, gle;
int tries = 0;
wchar_t wpold[MAX_PATH], wpnew[MAX_PATH];
wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH];
int wpnew_len;
if (xutftowcs_path(wpold, pold) < 0)
if (xutftowcs_long_path(wpold, pold) < 0)
return -1;
wpnew_len = xutftowcs_path(wpnew, pnew);
wpnew_len = xutftowcs_long_path(wpnew, pnew);
if (wpnew_len < 0)
return -1;
@ -2797,9 +2821,9 @@ repeat:
* flex array so that the structure has to be allocated on
* the heap. As we declare this structure ourselves though
* we can avoid the allocation and define FileName to have
* MAX_PATH bytes.
* MAX_LONG_PATH bytes.
*/
WCHAR FileName[MAX_PATH];
WCHAR FileName[MAX_LONG_PATH];
} rename_info = { 0 };
HANDLE old_handle = INVALID_HANDLE_VALUE;
BOOL success;
@ -3153,9 +3177,9 @@ int mingw_raise(int sig)
int link(const char *oldpath, const char *newpath)
{
wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH];
if (xutftowcs_path(woldpath, oldpath) < 0 ||
xutftowcs_path(wnewpath, newpath) < 0)
wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH];
if (xutftowcs_long_path(woldpath, oldpath) < 0 ||
xutftowcs_long_path(wnewpath, newpath) < 0)
return -1;
if (!CreateHardLinkW(wnewpath, woldpath, NULL)) {
@ -3167,7 +3191,7 @@ int link(const char *oldpath, const char *newpath)
int symlink(const char *target, const char *link)
{
wchar_t wtarget[MAX_PATH], wlink[MAX_PATH];
wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH];
int len;
/* fail if symlinks are disabled or API is not supported (WinXP) */
@ -3176,8 +3200,8 @@ int symlink(const char *target, const char *link)
return -1;
}
if ((len = xutftowcs_path(wtarget, target)) < 0
|| xutftowcs_path(wlink, link) < 0)
if ((len = xutftowcs_long_path(wtarget, target)) < 0
|| xutftowcs_long_path(wlink, link) < 0)
return -1;
/* convert target dir separators to backslashes */
@ -3231,12 +3255,12 @@ int symlink(const char *target, const char *link)
int readlink(const char *path, char *buf, size_t bufsiz)
{
WCHAR wpath[MAX_PATH];
char tmpbuf[MAX_PATH];
WCHAR wpath[MAX_LONG_PATH];
char tmpbuf[MAX_LONG_PATH];
int len;
DWORD tag;
if (xutftowcs_path(wpath, path) < 0)
if (xutftowcs_long_path(wpath, path) < 0)
return -1;
if (read_reparse_point(wpath, TRUE, tmpbuf, &len, &tag) < 0)
@ -3305,12 +3329,14 @@ pid_t waitpid(pid_t pid, int *status, int options)
return -1;
}
int (*win32_is_mount_point)(struct strbuf *path) = mingw_is_mount_point;
int mingw_is_mount_point(struct strbuf *path)
{
WIN32_FIND_DATAW findbuf = { 0 };
HANDLE handle;
wchar_t wfilename[MAX_PATH];
int wlen = xutftowcs_path(wfilename, path->buf);
wchar_t wfilename[MAX_LONG_PATH];
int wlen = xutftowcs_long_path(wfilename, path->buf);
if (wlen < 0)
die(_("could not get long path for '%s'"), path->buf);
@ -3453,9 +3479,9 @@ static size_t append_system_bin_dirs(char *path, size_t size)
static int is_system32_path(const char *path)
{
WCHAR system32[MAX_PATH], wpath[MAX_PATH];
WCHAR system32[MAX_LONG_PATH], wpath[MAX_LONG_PATH];
if (xutftowcs_path(wpath, path) < 0 ||
if (xutftowcs_long_path(wpath, path) < 0 ||
!GetSystemDirectoryW(system32, ARRAY_SIZE(system32)) ||
_wcsicmp(system32, wpath))
return 0;
@ -3891,6 +3917,73 @@ not_a_reserved_name:
}
}
int handle_long_path(wchar_t *path, int len, int max_path, int expand)
{
int result;
wchar_t buf[MAX_LONG_PATH];
/*
* we don't need special handling if path is relative to the current
* directory, and current directory + path don't exceed the desired
* max_path limit. This should cover > 99 % of cases with minimal
* performance impact (git almost always uses relative paths).
*/
if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) &&
(current_directory_len + len < max_path))
return len;
/*
* handle everything else:
* - absolute paths: "C:\dir\file"
* - absolute UNC paths: "\\server\share\dir\file"
* - absolute paths on current drive: "\dir\file"
* - relative paths on other drive: "X:file"
* - prefixed paths: "\\?\...", "\\.\..."
*/
/* convert to absolute path using GetFullPathNameW */
result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL);
if (!result) {
errno = err_win_to_posix(GetLastError());
return -1;
}
/*
* return absolute path if it fits within max_path (even if
* "cwd + path" doesn't due to '..' components)
*/
if (result < max_path) {
/* Be careful not to add a drive prefix if there was none */
if (is_wdir_sep(path[0]) &&
!is_wdir_sep(buf[0]) && buf[1] == L':' && is_wdir_sep(buf[2]))
wcscpy(path, buf + 2);
else
wcscpy(path, buf);
return result;
}
/* error out if we shouldn't expand the path or buf is too small */
if (!expand || result >= MAX_LONG_PATH - 6) {
errno = ENAMETOOLONG;
return -1;
}
/* prefix full path with "\\?\" or "\\?\UNC\" */
if (buf[0] == '\\') {
/* ...unless already prefixed */
if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.'))
return len;
wcscpy(path, L"\\\\?\\UNC\\");
wcscpy(path + 8, buf + 2);
return result + 6;
} else {
wcscpy(path, L"\\\\?\\");
wcscpy(path + 4, buf);
return result + 4;
}
}
#if !defined(_MSC_VER)
/*
* Disable MSVCRT command line wildcard expansion (__getmainargs called from
@ -4061,6 +4154,9 @@ int wmain(int argc, const wchar_t **wargv)
InitializeCriticalSection(&pinfo_cs);
InitializeCriticalSection(&phantom_symlinks_cs);
/* initialize critical section for fscache */
InitializeCriticalSection(&fscache_cs);
/* set up default file mode and file modes for stdin/out/err */
_fmode = _O_BINARY;
_setmode(_fileno(stdin), _O_BINARY);
@ -4070,6 +4166,9 @@ int wmain(int argc, const wchar_t **wargv)
/* initialize Unicode console */
winansi_init();
/* init length of current directory for handle_long_path */
current_directory_len = GetCurrentDirectoryW(0, NULL);
/* invoke the real main() using our utf8 version of argv. */
exit_status = main(argc, argv);

View File

@ -1,5 +1,8 @@
#include "mingw-posix.h"
extern int core_fscache;
int are_long_paths_enabled(void);
struct config_context;
int mingw_core_config(const char *var, const char *value,
const struct config_context *ctx, void *cb);
@ -38,7 +41,8 @@ static inline void convert_slashes(char *path)
}
struct strbuf;
int mingw_is_mount_point(struct strbuf *path);
#define is_mount_point mingw_is_mount_point
extern int (*win32_is_mount_point)(struct strbuf *path);
#define is_mount_point win32_is_mount_point
#define CAN_UNLINK_MOUNT_POINTS 1
#define PATH_SEP ';'
char *mingw_query_user_email(void);
@ -75,6 +79,42 @@ int is_path_owned_by_current_sid(const char *path, struct strbuf *report);
int is_valid_win32_path(const char *path, int allow_literal_nul);
#define is_valid_path(path) is_valid_win32_path(path, 0)
/**
* Max length of long paths (exceeding MAX_PATH). The actual maximum supported
* by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller
* value to limit required stack memory.
*/
#define MAX_LONG_PATH 4096
/**
* Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs.
*
* With expand == false, the function checks for over-long paths and fails
* with ENAMETOOLONG. The path parameter is not modified, except if cwd + path
* exceeds max_path, but the resulting absolute path doesn't (e.g. due to
* eliminating '..' components). The path parameter must point to a buffer
* of max_path wide characters.
*
* With expand == true, an over-long path is automatically converted in place
* to an absolute path prefixed with '\\?\', and the new length is returned.
* The path parameter must point to a buffer of MAX_LONG_PATH wide characters.
*
* Parameters:
* path: path to check and / or convert
* len: size of path on input (number of wide chars without \0)
* max_path: max short path length to check (usually MAX_PATH = 260, but just
* 248 for CreateDirectoryW)
* expand: false to only check the length, true to expand the path to a
* '\\?\'-prefixed absolute path
*
* Return:
* length of the resulting path, or -1 on failure
*
* Errors:
* ENAMETOOLONG if path is too long
*/
int handle_long_path(wchar_t *path, int len, int max_path, int expand);
/**
* Converts UTF-8 encoded string to UTF-16LE.
*
@ -132,17 +172,45 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen)
return xutftowcsn(wcs, utf, wcslen, -1);
}
/**
* Simplified file system specific wrapper of xutftowcsn and handle_long_path.
* Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least
* MAX_LONG_PATH wide chars (see handle_long_path).
*/
static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf,
size_t wcslen, int utflen, int max_path, int expand)
{
int result = xutftowcsn(wcs, utf, wcslen, utflen);
if (result < 0 && errno == ERANGE)
errno = ENAMETOOLONG;
if (result >= 0)
result = handle_long_path(wcs, result, max_path, expand);
return result;
}
/**
* Simplified file system specific variant of xutftowcsn, assumes output
* buffer size is MAX_PATH wide chars and input string is \0-terminated,
* fails with ENAMETOOLONG if input string is too long.
* fails with ENAMETOOLONG if input string is too long. Typically used for
* Windows APIs that don't support long paths, e.g. SetCurrentDirectory,
* LoadLibrary, CreateProcess...
*/
static inline int xutftowcs_path(wchar_t *wcs, const char *utf)
{
int result = xutftowcsn(wcs, utf, MAX_PATH, -1);
if (result < 0 && errno == ERANGE)
errno = ENAMETOOLONG;
return result;
return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0);
}
/**
* Simplified file system specific variant of xutftowcsn for Windows APIs
* that support long paths via '\\?\'-prefix, assumes output buffer size is
* MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too
* long. The 'core.longpaths' git-config option controls whether the path
* is only checked or expanded to a long path.
*/
static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf)
{
return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH,
are_long_paths_enabled());
}
/**

View File

@ -1,15 +1,21 @@
#include "../../git-compat-util.h"
struct DIR {
struct dirent dd_dir; /* includes d_type */
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
typedef struct dirent_DIR {
struct DIR base_dir; /* extend base struct DIR */
HANDLE dd_handle; /* FindFirstFile handle */
int dd_stat; /* 0-based index */
};
struct dirent dd_dir; /* includes d_type */
} dirent_DIR;
#pragma GCC diagnostic pop
DIR *(*opendir)(const char *dirname) = dirent_opendir;
static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata)
{
/* convert UTF-16 name to UTF-8 */
xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name));
/* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */
xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3);
/* Set file type, based on WIN32_FIND_DATA */
if ((fdata->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
@ -21,41 +27,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata)
ent->d_type = DT_REG;
}
DIR *opendir(const char *name)
{
wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */
WIN32_FIND_DATAW fdata;
HANDLE h;
int len;
DIR *dir;
/* convert name to UTF-16 and check length < MAX_PATH */
if ((len = xutftowcs_path(pattern, name)) < 0)
return NULL;
/* append optional '/' and wildcard '*' */
if (len && !is_dir_sep(pattern[len - 1]))
pattern[len++] = '/';
pattern[len++] = '*';
pattern[len] = 0;
/* open find handle */
h = FindFirstFileW(pattern, &fdata);
if (h == INVALID_HANDLE_VALUE) {
DWORD err = GetLastError();
errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
return NULL;
}
/* initialize DIR structure and copy first dir entry */
dir = xmalloc(sizeof(DIR));
dir->dd_handle = h;
dir->dd_stat = 0;
finddata2dirent(&dir->dd_dir, &fdata);
return dir;
}
struct dirent *readdir(DIR *dir)
static struct dirent *dirent_readdir(dirent_DIR *dir)
{
if (!dir) {
errno = EBADF; /* No set_errno for mingw */
@ -82,7 +54,7 @@ struct dirent *readdir(DIR *dir)
return &dir->dd_dir;
}
int closedir(DIR *dir)
static int dirent_closedir(dirent_DIR *dir)
{
if (!dir) {
errno = EBADF;
@ -93,3 +65,44 @@ int closedir(DIR *dir)
free(dir);
return 0;
}
DIR *dirent_opendir(const char *name)
{
wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */
WIN32_FIND_DATAW fdata;
HANDLE h;
int len;
dirent_DIR *dir;
/* convert name to UTF-16 and check length */
if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1,
MAX_PATH - 2,
are_long_paths_enabled())) < 0)
return NULL;
/*
* append optional '\' and wildcard '*'. Note: we need to use '\' as
* Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths.
*/
if (len && !is_dir_sep(pattern[len - 1]))
pattern[len++] = '\\';
pattern[len++] = '*';
pattern[len] = 0;
/* open find handle */
h = FindFirstFileW(pattern, &fdata);
if (h == INVALID_HANDLE_VALUE) {
DWORD err = GetLastError();
errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
return NULL;
}
/* initialize DIR structure and copy first dir entry */
dir = xmalloc(sizeof(dirent_DIR) + MAX_LONG_PATH);
dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir;
dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir;
dir->dd_handle = h;
dir->dd_stat = 0;
finddata2dirent(&dir->dd_dir, &fdata);
return (DIR*) dir;
}

View File

@ -1,20 +1,34 @@
#ifndef DIRENT_H
#define DIRENT_H
typedef struct DIR DIR;
#define DT_UNKNOWN 0
#define DT_DIR 1
#define DT_REG 2
#define DT_LNK 3
struct dirent {
unsigned char d_type; /* file type to prevent lstat after readdir */
char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */
unsigned char d_type; /* file type to prevent lstat after readdir */
char d_name[/* FLEX_ARRAY */]; /* file name */
};
DIR *opendir(const char *dirname);
struct dirent *readdir(DIR *dir);
int closedir(DIR *dir);
/*
* Base DIR structure, contains pointers to readdir/closedir implementations so
* that opendir may choose a concrete implementation on a call-by-call basis.
*/
typedef struct DIR {
struct dirent *(*preaddir)(struct DIR *dir);
int (*pclosedir)(struct DIR *dir);
} DIR;
/* default dirent implementation */
extern DIR *dirent_opendir(const char *dirname);
#define opendir git_opendir
/* current dirent implementation */
extern DIR *(*opendir)(const char *dirname);
#define readdir(dir) (dir->preaddir(dir))
#define closedir(dir) (dir->pclosedir(dir))
#endif /* DIRENT_H */

782
compat/win32/fscache.c Normal file
View File

@ -0,0 +1,782 @@
#include "../../git-compat-util.h"
#include "../../hashmap.h"
#include "../win32.h"
#include "fscache.h"
#include "../../dir.h"
#include "../../abspath.h"
#include "../../trace.h"
#include "config.h"
#include "../../mem-pool.h"
#include "ntifs.h"
static volatile long initialized;
static DWORD dwTlsIndex;
CRITICAL_SECTION fscache_cs;
/*
* Store one fscache per thread to avoid thread contention and locking.
* This is ok because multi-threaded access is 1) uncommon and 2) always
* splitting up the cache entries across multiple threads so there isn't
* any overlap between threads anyway.
*/
struct fscache {
volatile long enabled;
struct hashmap map;
struct mem_pool mem_pool;
unsigned int lstat_requests;
unsigned int opendir_requests;
unsigned int fscache_requests;
unsigned int fscache_misses;
/*
* 32k wide characters translates to 64kB, which is the maximum that
* Windows 8.1 and earlier can handle. On network drives, not only
* the client's Windows version matters, but also the server's,
* therefore we need to keep this to 64kB.
*/
WCHAR buffer[32 * 1024];
};
static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE);
/*
* An entry in the file system cache. Used for both entire directory listings
* and file entries.
*/
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
struct fsentry {
struct hashmap_entry ent;
mode_t st_mode;
ULONG reparse_tag;
/* Pointer to the directory listing, or NULL for the listing itself. */
struct fsentry *list;
/* Pointer to the next file entry of the list. */
struct fsentry *next;
union {
/* Reference count of the directory listing. */
volatile long refcnt;
struct {
/* More stat members (only used for file entries). */
off64_t st_size;
struct timespec st_atim;
struct timespec st_mtim;
struct timespec st_ctim;
} s;
} u;
/* Length of name. */
unsigned short len;
/*
* Name of the entry. For directory listings: relative path of the
* directory, without trailing '/' (empty for cwd()). For file entries:
* name of the file. Typically points to the end of the structure if
* the fsentry is allocated on the heap (see fsentry_alloc), or to a
* local variable if on the stack (see fsentry_init).
*/
struct dirent dirent;
};
#pragma GCC diagnostic pop
#pragma GCC diagnostic push
#ifdef __clang__
#pragma GCC diagnostic ignored "-Wflexible-array-extensions"
#endif
struct heap_fsentry {
union {
struct fsentry ent;
char dummy[sizeof(struct fsentry) + MAX_LONG_PATH];
} u;
};
#pragma GCC diagnostic pop
/*
* Compares the paths of two fsentry structures for equality.
*/
static int fsentry_cmp(void *cmp_data UNUSED,
const struct fsentry *fse1, const struct fsentry *fse2,
void *keydata UNUSED)
{
int res;
if (fse1 == fse2)
return 0;
/* compare the list parts first */
if (fse1->list != fse2->list &&
(res = fsentry_cmp(NULL, fse1->list ? fse1->list : fse1,
fse2->list ? fse2->list : fse2, NULL)))
return res;
/* if list parts are equal, compare len and name */
if (fse1->len != fse2->len)
return fse1->len - fse2->len;
return fspathncmp(fse1->dirent.d_name, fse2->dirent.d_name, fse1->len);
}
/*
* Calculates the hash code of an fsentry structure's path.
*/
static unsigned int fsentry_hash(const struct fsentry *fse)
{
unsigned int hash = fse->list ? fse->list->ent.hash : 0;
return hash ^ memihash(fse->dirent.d_name, fse->len);
}
/*
* Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp.
*/
static void fsentry_init(struct fsentry *fse, struct fsentry *list,
const char *name, size_t len)
{
fse->list = list;
if (len > MAX_LONG_PATH)
BUG("Trying to allocate fsentry for long path '%.*s'",
(int)len, name);
memcpy(fse->dirent.d_name, name, len);
fse->dirent.d_name[len] = 0;
fse->len = len;
hashmap_entry_init(&fse->ent, fsentry_hash(fse));
}
/*
* Allocate an fsentry structure on the heap.
*/
static struct fsentry *fsentry_alloc(struct fscache *cache, struct fsentry *list, const char *name,
size_t len)
{
/* overallocate fsentry and copy the name to the end */
struct fsentry *fse =
mem_pool_alloc(&cache->mem_pool, sizeof(*fse) + len + 1);
/* init the rest of the structure */
fsentry_init(fse, list, name, len);
fse->next = NULL;
fse->u.refcnt = 1;
return fse;
}
/*
* Add a reference to an fsentry.
*/
inline static void fsentry_addref(struct fsentry *fse)
{
if (fse->list)
fse = fse->list;
InterlockedIncrement(&(fse->u.refcnt));
}
/*
* Release the reference to an fsentry.
*/
static void fsentry_release(struct fsentry *fse)
{
if (fse->list)
fse = fse->list;
InterlockedDecrement(&(fse->u.refcnt));
}
static int xwcstoutfn(char *utf, int utflen, const wchar_t *wcs, int wcslen)
{
if (!wcs || !utf || utflen < 1) {
errno = EINVAL;
return -1;
}
utflen = WideCharToMultiByte(CP_UTF8, 0, wcs, wcslen, utf, utflen, NULL, NULL);
if (utflen)
return utflen;
errno = ERANGE;
return -1;
}
/*
* Allocate and initialize an fsentry from a FILE_FULL_DIR_INFORMATION structure.
*/
static struct fsentry *fseentry_create_entry(struct fscache *cache,
struct fsentry *list,
PFILE_FULL_DIR_INFORMATION fdata)
{
char buf[MAX_PATH * 3];
int len;
struct fsentry *fse;
len = xwcstoutfn(buf, ARRAY_SIZE(buf), fdata->FileName, fdata->FileNameLength / sizeof(wchar_t));
fse = fsentry_alloc(cache, list, buf, len);
fse->reparse_tag =
fdata->FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT ?
fdata->EaSize : 0;
fse->st_mode = file_attr_to_st_mode(fdata->FileAttributes,
fdata->EaSize, buf);
fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG :
S_ISDIR(fse->st_mode) ? DT_DIR : DT_LNK;
fse->u.s.st_size = S_ISLNK(fse->st_mode) ? MAX_LONG_PATH :
fdata->EndOfFile.LowPart |
(((off_t)fdata->EndOfFile.HighPart) << 32);
filetime_to_timespec((FILETIME *)&(fdata->LastAccessTime),
&(fse->u.s.st_atim));
filetime_to_timespec((FILETIME *)&(fdata->LastWriteTime),
&(fse->u.s.st_mtim));
filetime_to_timespec((FILETIME *)&(fdata->CreationTime),
&(fse->u.s.st_ctim));
return fse;
}
/*
* Create an fsentry-based directory listing (similar to opendir / readdir).
* Dir should not contain trailing '/'. Use an empty string for the current
* directory (not "."!).
*/
static struct fsentry *fsentry_create_list(struct fscache *cache, const struct fsentry *dir,
int *dir_not_found)
{
wchar_t pattern[MAX_LONG_PATH];
NTSTATUS status;
IO_STATUS_BLOCK iosb;
PFILE_FULL_DIR_INFORMATION di;
HANDLE h;
int wlen;
struct fsentry *list, **phead;
DWORD err;
*dir_not_found = 0;
/* convert name to UTF-16 and check length */
if ((wlen = xutftowcs_path_ex(pattern, dir->dirent.d_name,
MAX_LONG_PATH, dir->len, MAX_PATH - 2,
are_long_paths_enabled())) < 0)
return NULL;
/* handle CWD */
if (!wlen) {
wlen = GetCurrentDirectoryW(ARRAY_SIZE(pattern), pattern);
if (!wlen || wlen >= (ssize_t)ARRAY_SIZE(pattern)) {
errno = wlen ? ENAMETOOLONG : err_win_to_posix(GetLastError());
return NULL;
}
}
h = CreateFileW(pattern, FILE_LIST_DIRECTORY,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
if (h == INVALID_HANDLE_VALUE) {
err = GetLastError();
*dir_not_found = 1; /* or empty directory */
errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
trace_printf_key(&trace_fscache, "fscache: error(%d) '%s'\n",
errno, dir->dirent.d_name);
return NULL;
}
/* allocate object to hold directory listing */
list = fsentry_alloc(cache, NULL, dir->dirent.d_name, dir->len);
list->st_mode = S_IFDIR;
list->dirent.d_type = DT_DIR;
/* walk directory and build linked list of fsentry structures */
phead = &list->next;
status = NtQueryDirectoryFile(h, NULL, 0, 0, &iosb, cache->buffer,
sizeof(cache->buffer), FileFullDirectoryInformation, FALSE, NULL, FALSE);
if (!NT_SUCCESS(status)) {
/*
* NtQueryDirectoryFile returns STATUS_INVALID_PARAMETER when
* asked to enumerate an invalid directory (ie it is a file
* instead of a directory). Verify that is the actual cause
* of the error.
*/
if (status == (NTSTATUS)STATUS_INVALID_PARAMETER) {
DWORD attributes = GetFileAttributesW(pattern);
if (!(attributes & FILE_ATTRIBUTE_DIRECTORY))
status = ERROR_DIRECTORY;
}
goto Error;
}
di = (PFILE_FULL_DIR_INFORMATION)(cache->buffer);
for (;;) {
*phead = fseentry_create_entry(cache, list, di);
phead = &(*phead)->next;
/* If there is no offset in the entry, the buffer has been exhausted. */
if (di->NextEntryOffset == 0) {
status = NtQueryDirectoryFile(h, NULL, 0, 0, &iosb, cache->buffer,
sizeof(cache->buffer), FileFullDirectoryInformation, FALSE, NULL, FALSE);
if (!NT_SUCCESS(status)) {
if (status == STATUS_NO_MORE_FILES)
break;
goto Error;
}
di = (PFILE_FULL_DIR_INFORMATION)(cache->buffer);
continue;
}
/* Advance to the next entry. */
di = (PFILE_FULL_DIR_INFORMATION)(((PUCHAR)di) + di->NextEntryOffset);
}
CloseHandle(h);
return list;
Error:
trace_printf_key(&trace_fscache,
"fscache: status(%ld) unable to query directory "
"contents '%s'\n", status, dir->dirent.d_name);
CloseHandle(h);
fsentry_release(list);
return NULL;
}
/*
* Adds a directory listing to the cache.
*/
static void fscache_add(struct fscache *cache, struct fsentry *fse)
{
if (fse->list)
fse = fse->list;
for (; fse; fse = fse->next)
hashmap_add(&cache->map, &fse->ent);
}
/*
* Clears the cache.
*/
static void fscache_clear(struct fscache *cache)
{
mem_pool_discard(&cache->mem_pool, 0);
mem_pool_init(&cache->mem_pool, 0);
hashmap_clear(&cache->map);
hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0);
cache->lstat_requests = cache->opendir_requests = 0;
cache->fscache_misses = cache->fscache_requests = 0;
}
/*
* Checks if the cache is enabled for the given path.
*/
static int do_fscache_enabled(struct fscache *cache, const char *path)
{
return cache->enabled > 0 && !is_absolute_path(path);
}
int fscache_enabled(const char *path)
{
struct fscache *cache = fscache_getcache();
return cache ? do_fscache_enabled(cache, path) : 0;
}
/*
* Looks up or creates a cache entry for the specified key.
*/
static struct fsentry *fscache_get(struct fscache *cache, struct fsentry *key)
{
struct fsentry *fse;
int dir_not_found;
cache->fscache_requests++;
/* check if entry is in cache */
fse = hashmap_get_entry(&cache->map, key, ent, NULL);
if (fse) {
if (fse->st_mode)
fsentry_addref(fse);
else
fse = NULL; /* non-existing directory */
return fse;
}
/* if looking for a file, check if directory listing is in cache */
if (!fse && key->list) {
fse = hashmap_get_entry(&cache->map, key->list, ent, NULL);
if (fse) {
/*
* dir entry without file entry, or dir does not
* exist -> file doesn't exist
*/
errno = ENOENT;
return NULL;
}
}
/* create the directory listing */
fse = fsentry_create_list(cache, key->list ? key->list : key, &dir_not_found);
/* leave on error (errno set by fsentry_create_list) */
if (!fse) {
if (dir_not_found && key->list) {
/*
* Record that the directory does not exist (or is
* empty, which for all practical matters is the same
* thing as far as fscache is concerned).
*/
fse = fsentry_alloc(cache, key->list->list,
key->list->dirent.d_name,
key->list->len);
fse->st_mode = 0;
hashmap_add(&cache->map, &fse->ent);
}
return NULL;
}
/* add directory listing to the cache */
cache->fscache_misses++;
fscache_add(cache, fse);
/* lookup file entry if requested (fse already points to directory) */
if (key->list)
fse = hashmap_get_entry(&cache->map, key, ent, NULL);
if (fse && !fse->st_mode)
fse = NULL; /* non-existing directory */
/* return entry or ENOENT */
if (fse)
fsentry_addref(fse);
else
errno = ENOENT;
return fse;
}
/*
* Enables the cache. Note that the cache is read-only, changes to
* the working directory are NOT reflected in the cache while enabled.
*/
int fscache_enable(size_t initial_size)
{
int fscache;
struct fscache *cache;
int result = 0;
/* allow the cache to be disabled entirely */
fscache = git_env_bool("GIT_TEST_FSCACHE", -1);
if (fscache != -1)
core_fscache = fscache;
if (!core_fscache)
return 0;
/*
* refcount the global fscache initialization so that the
* opendir and lstat function pointers are redirected if
* any threads are using the fscache.
*/
EnterCriticalSection(&fscache_cs);
if (!initialized) {
if (!dwTlsIndex) {
dwTlsIndex = TlsAlloc();
if (dwTlsIndex == TLS_OUT_OF_INDEXES) {
LeaveCriticalSection(&fscache_cs);
return 0;
}
}
/* redirect opendir and lstat to the fscache implementations */
opendir = fscache_opendir;
lstat = fscache_lstat;
win32_is_mount_point = fscache_is_mount_point;
}
initialized++;
LeaveCriticalSection(&fscache_cs);
/* refcount the thread specific initialization */
cache = fscache_getcache();
if (cache) {
cache->enabled++;
} else {
cache = (struct fscache *)xcalloc(1, sizeof(*cache));
cache->enabled = 1;
/*
* avoid having to rehash by leaving room for the parent dirs.
* '4' was determined empirically by testing several repos
*/
hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, initial_size * 4);
mem_pool_init(&cache->mem_pool, 0);
if (!TlsSetValue(dwTlsIndex, cache))
BUG("TlsSetValue error");
}
trace_printf_key(&trace_fscache, "fscache: enable\n");
return result;
}
/*
* Disables the cache.
*/
void fscache_disable(void)
{
struct fscache *cache;
if (!core_fscache)
return;
/* update the thread specific fscache initialization */
cache = fscache_getcache();
if (!cache)
BUG("fscache_disable() called on a thread where fscache has not been initialized");
if (!cache->enabled)
BUG("fscache_disable() called on an fscache that is already disabled");
cache->enabled--;
if (!cache->enabled) {
TlsSetValue(dwTlsIndex, NULL);
trace_printf_key(&trace_fscache, "fscache_disable: lstat %u, opendir %u, "
"total requests/misses %u/%u\n",
cache->lstat_requests, cache->opendir_requests,
cache->fscache_requests, cache->fscache_misses);
mem_pool_discard(&cache->mem_pool, 0);
hashmap_clear(&cache->map);
free(cache);
}
/* update the global fscache initialization */
EnterCriticalSection(&fscache_cs);
initialized--;
if (!initialized) {
/* reset opendir and lstat to the original implementations */
opendir = dirent_opendir;
lstat = mingw_lstat;
win32_is_mount_point = mingw_is_mount_point;
}
LeaveCriticalSection(&fscache_cs);
trace_printf_key(&trace_fscache, "fscache: disable\n");
return;
}
/*
* Flush cached stats result when fscache is enabled.
*/
void fscache_flush(void)
{
struct fscache *cache = fscache_getcache();
if (cache && cache->enabled) {
fscache_clear(cache);
}
}
/*
* Lstat replacement, uses the cache if enabled, otherwise redirects to
* mingw_lstat.
*/
int fscache_lstat(const char *filename, struct stat *st)
{
int dirlen, base, len;
#pragma GCC diagnostic push
#ifdef __clang__
#pragma GCC diagnostic ignored "-Wflexible-array-extensions"
#endif
struct heap_fsentry key[2];
#pragma GCC diagnostic pop
struct fsentry *fse;
struct fscache *cache = fscache_getcache();
if (!cache || !do_fscache_enabled(cache, filename))
return mingw_lstat(filename, st);
cache->lstat_requests++;
/* split filename into path + name */
len = strlen(filename);
if (len && is_dir_sep(filename[len - 1]))
len--;
base = len;
while (base && !is_dir_sep(filename[base - 1]))
base--;
dirlen = base ? base - 1 : 0;
/* lookup entry for path + name in cache */
fsentry_init(&key[0].u.ent, NULL, filename, dirlen);
fsentry_init(&key[1].u.ent, &key[0].u.ent, filename + base, len - base);
fse = fscache_get(cache, &key[1].u.ent);
if (!fse) {
errno = ENOENT;
return -1;
}
/*
* Special case symbolic links: FindFirstFile()/FindNextFile() did not
* provide us with the length of the target path.
*/
if (fse->u.s.st_size == MAX_LONG_PATH && S_ISLNK(fse->st_mode)) {
char buf[MAX_LONG_PATH];
int len = readlink(filename, buf, sizeof(buf) - 1);
if (len > 0)
fse->u.s.st_size = len;
}
/* copy stat data */
st->st_ino = 0;
st->st_gid = 0;
st->st_uid = 0;
st->st_dev = 0;
st->st_rdev = 0;
st->st_nlink = 1;
st->st_mode = fse->st_mode;
st->st_size = fse->u.s.st_size;
st->st_atim = fse->u.s.st_atim;
st->st_mtim = fse->u.s.st_mtim;
st->st_ctim = fse->u.s.st_ctim;
/* don't forget to release fsentry */
fsentry_release(fse);
return 0;
}
/*
* is_mount_point() replacement, uses cache if enabled, otherwise falls
* back to mingw_is_mount_point().
*/
int fscache_is_mount_point(struct strbuf *path)
{
int dirlen, base, len;
#pragma GCC diagnostic push
#ifdef __clang__
#pragma GCC diagnostic ignored "-Wflexible-array-extensions"
#endif
struct heap_fsentry key[2];
#pragma GCC diagnostic pop
struct fsentry *fse;
struct fscache *cache = fscache_getcache();
if (!cache || !do_fscache_enabled(cache, path->buf))
return mingw_is_mount_point(path);
cache->lstat_requests++;
/* split path into path + name */
len = path->len;
if (len && is_dir_sep(path->buf[len - 1]))
len--;
base = len;
while (base && !is_dir_sep(path->buf[base - 1]))
base--;
dirlen = base ? base - 1 : 0;
/* lookup entry for path + name in cache */
fsentry_init(&key[0].u.ent, NULL, path->buf, dirlen);
fsentry_init(&key[1].u.ent, &key[0].u.ent, path->buf + base, len - base);
fse = fscache_get(cache, &key[1].u.ent);
if (!fse)
return mingw_is_mount_point(path);
return fse->reparse_tag == IO_REPARSE_TAG_MOUNT_POINT;
}
typedef struct fscache_DIR {
struct DIR base_dir; /* extend base struct DIR */
struct fsentry *pfsentry;
struct dirent *dirent;
} fscache_DIR;
/*
* Readdir replacement.
*/
static struct dirent *fscache_readdir(DIR *base_dir)
{
fscache_DIR *dir = (fscache_DIR*) base_dir;
struct fsentry *next = dir->pfsentry->next;
if (!next)
return NULL;
dir->pfsentry = next;
dir->dirent = &next->dirent;
return dir->dirent;
}
/*
* Closedir replacement.
*/
static int fscache_closedir(DIR *base_dir)
{
fscache_DIR *dir = (fscache_DIR*) base_dir;
fsentry_release(dir->pfsentry);
free(dir);
return 0;
}
/*
* Opendir replacement, uses a directory listing from the cache if enabled,
* otherwise calls original dirent implementation.
*/
DIR *fscache_opendir(const char *dirname)
{
struct heap_fsentry key;
struct fsentry *list;
fscache_DIR *dir;
int len;
struct fscache *cache = fscache_getcache();
if (!cache || !do_fscache_enabled(cache, dirname))
return dirent_opendir(dirname);
cache->opendir_requests++;
/* prepare name (strip trailing '/', replace '.') */
len = strlen(dirname);
if ((len == 1 && dirname[0] == '.') ||
(len && is_dir_sep(dirname[len - 1])))
len--;
/* get directory listing from cache */
fsentry_init(&key.u.ent, NULL, dirname, len);
list = fscache_get(cache, &key.u.ent);
if (!list)
return NULL;
/* alloc and return DIR structure */
dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR));
dir->base_dir.preaddir = fscache_readdir;
dir->base_dir.pclosedir = fscache_closedir;
dir->pfsentry = list;
return (DIR*) dir;
}
struct fscache *fscache_getcache(void)
{
return (struct fscache *)TlsGetValue(dwTlsIndex);
}
void fscache_merge(struct fscache *dest)
{
struct hashmap_iter iter;
struct hashmap_entry *e;
struct fscache *cache = fscache_getcache();
/*
* Only do the merge if fscache was enabled and we have a dest
* cache to merge into.
*/
if (!dest) {
fscache_enable(0);
return;
}
if (!cache)
BUG("fscache_merge() called on a thread where fscache has not been initialized");
TlsSetValue(dwTlsIndex, NULL);
trace_printf_key(&trace_fscache, "fscache_merge: lstat %u, opendir %u, "
"total requests/misses %u/%u\n",
cache->lstat_requests, cache->opendir_requests,
cache->fscache_requests, cache->fscache_misses);
/*
* This is only safe because the primary thread we're merging into
* isn't being used so the critical section only needs to prevent
* the the child threads from stomping on each other.
*/
EnterCriticalSection(&fscache_cs);
hashmap_iter_init(&cache->map, &iter);
while ((e = hashmap_iter_next(&iter)))
hashmap_add(&dest->map, e);
mem_pool_combine(&dest->mem_pool, &cache->mem_pool);
dest->lstat_requests += cache->lstat_requests;
dest->opendir_requests += cache->opendir_requests;
dest->fscache_requests += cache->fscache_requests;
dest->fscache_misses += cache->fscache_misses;
initialized--;
LeaveCriticalSection(&fscache_cs);
free(cache);
}

36
compat/win32/fscache.h Normal file
View File

@ -0,0 +1,36 @@
#ifndef FSCACHE_H
#define FSCACHE_H
/*
* The fscache is thread specific. enable_fscache() must be called
* for each thread where caching is desired.
*/
extern CRITICAL_SECTION fscache_cs;
int fscache_enable(size_t initial_size);
#define enable_fscache(initial_size) fscache_enable(initial_size)
void fscache_disable(void);
#define disable_fscache() fscache_disable()
int fscache_enabled(const char *path);
#define is_fscache_enabled(path) fscache_enabled(path)
void fscache_flush(void);
#define flush_fscache() fscache_flush()
DIR *fscache_opendir(const char *dir);
int fscache_lstat(const char *file_name, struct stat *buf);
int fscache_is_mount_point(struct strbuf *path);
/* opaque fscache structure */
struct fscache;
struct fscache *fscache_getcache(void);
#define getcache_fscache() fscache_getcache()
void fscache_merge(struct fscache *dest);
#define merge_fscache(dest) fscache_merge(dest)
#endif

131
compat/win32/ntifs.h Normal file
View File

@ -0,0 +1,131 @@
#ifndef _NTIFS_
#define _NTIFS_
/*
* Copy necessary structures and definitions out of the Windows DDK
* to enable calling NtQueryDirectoryFile()
*/
typedef _Return_type_success_(return >= 0) LONG NTSTATUS;
#define NT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0)
#if !defined(_NTSECAPI_) && !defined(_WINTERNL_) && \
!defined(__UNICODE_STRING_DEFINED)
#define __UNICODE_STRING_DEFINED
typedef struct _UNICODE_STRING {
USHORT Length;
USHORT MaximumLength;
PWSTR Buffer;
} UNICODE_STRING;
typedef UNICODE_STRING *PUNICODE_STRING;
typedef const UNICODE_STRING *PCUNICODE_STRING;
#endif /* !_NTSECAPI_ && !_WINTERNL_ && !__UNICODE_STRING_DEFINED */
typedef enum _FILE_INFORMATION_CLASS {
FileDirectoryInformation = 1,
FileFullDirectoryInformation,
FileBothDirectoryInformation,
FileBasicInformation,
FileStandardInformation,
FileInternalInformation,
FileEaInformation,
FileAccessInformation,
FileNameInformation,
FileRenameInformation,
FileLinkInformation,
FileNamesInformation,
FileDispositionInformation,
FilePositionInformation,
FileFullEaInformation,
FileModeInformation,
FileAlignmentInformation,
FileAllInformation,
FileAllocationInformation,
FileEndOfFileInformation,
FileAlternateNameInformation,
FileStreamInformation,
FilePipeInformation,
FilePipeLocalInformation,
FilePipeRemoteInformation,
FileMailslotQueryInformation,
FileMailslotSetInformation,
FileCompressionInformation,
FileObjectIdInformation,
FileCompletionInformation,
FileMoveClusterInformation,
FileQuotaInformation,
FileReparsePointInformation,
FileNetworkOpenInformation,
FileAttributeTagInformation,
FileTrackingInformation,
FileIdBothDirectoryInformation,
FileIdFullDirectoryInformation,
FileValidDataLengthInformation,
FileShortNameInformation,
FileIoCompletionNotificationInformation,
FileIoStatusBlockRangeInformation,
FileIoPriorityHintInformation,
FileSfioReserveInformation,
FileSfioVolumeInformation,
FileHardLinkInformation,
FileProcessIdsUsingFileInformation,
FileNormalizedNameInformation,
FileNetworkPhysicalNameInformation,
FileIdGlobalTxDirectoryInformation,
FileIsRemoteDeviceInformation,
FileAttributeCacheInformation,
FileNumaNodeInformation,
FileStandardLinkInformation,
FileRemoteProtocolInformation,
FileMaximumInformation
} FILE_INFORMATION_CLASS, *PFILE_INFORMATION_CLASS;
typedef struct _FILE_FULL_DIR_INFORMATION {
ULONG NextEntryOffset;
ULONG FileIndex;
LARGE_INTEGER CreationTime;
LARGE_INTEGER LastAccessTime;
LARGE_INTEGER LastWriteTime;
LARGE_INTEGER ChangeTime;
LARGE_INTEGER EndOfFile;
LARGE_INTEGER AllocationSize;
ULONG FileAttributes;
ULONG FileNameLength;
ULONG EaSize;
WCHAR FileName[1];
} FILE_FULL_DIR_INFORMATION, *PFILE_FULL_DIR_INFORMATION;
typedef struct _IO_STATUS_BLOCK {
union {
NTSTATUS Status;
PVOID Pointer;
} u;
ULONG_PTR Information;
} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK;
typedef VOID
(NTAPI *PIO_APC_ROUTINE)(
IN PVOID ApcContext,
IN PIO_STATUS_BLOCK IoStatusBlock,
IN ULONG Reserved);
NTSYSCALLAPI
NTSTATUS
NTAPI
NtQueryDirectoryFile(
_In_ HANDLE FileHandle,
_In_opt_ HANDLE Event,
_In_opt_ PIO_APC_ROUTINE ApcRoutine,
_In_opt_ PVOID ApcContext,
_Out_ PIO_STATUS_BLOCK IoStatusBlock,
_Out_writes_bytes_(Length) PVOID FileInformation,
_In_ ULONG Length,
_In_ FILE_INFORMATION_CLASS FileInformationClass,
_In_ BOOLEAN ReturnSingleEntry,
_In_opt_ PUNICODE_STRING FileName,
_In_ BOOLEAN RestartScan
);
#define STATUS_NO_MORE_FILES ((NTSTATUS)0x80000006L)
#endif

View File

@ -509,7 +509,7 @@ endif
compat/win32/path-utils.o \
compat/win32/pthread.o compat/win32/syslog.o \
compat/win32/trace2_win32_process_info.o \
compat/win32/dirent.o
compat/win32/dirent.o compat/win32/fscache.o
COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY \
-DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" \
-DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\"
@ -714,7 +714,7 @@ ifeq ($(uname_S),MINGW)
compat/win32/flush.o \
compat/win32/path-utils.o \
compat/win32/pthread.o compat/win32/syslog.o \
compat/win32/dirent.o
compat/win32/dirent.o compat/win32/fscache.o
BASIC_CFLAGS += -DWIN32
EXTLIBS += -lws2_32
GITLIBS += git.res

View File

@ -301,7 +301,8 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
compat/win32/trace2_win32_process_info.c
compat/win32/dirent.c
compat/nedmalloc/nedmalloc.c
compat/strdup.c)
compat/strdup.c
compat/win32/fscache.c)
set(NO_UNIX_SOCKETS 1)
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")

66
dir.c
View File

@ -1156,16 +1156,64 @@ static int add_patterns(const char *fname, const char *base, int baselen,
size_t size = 0;
char *buf;
if (flags & PATTERN_NOFOLLOW)
fd = open_nofollow(fname, O_RDONLY);
else
fd = open(fname, O_RDONLY);
if (fd < 0 || fstat(fd, &st) < 0) {
if (fd < 0)
warn_on_fopen_errors(fname);
/*
* A performance optimization for status.
*
* During a status scan, git looks in each directory for a .gitignore
* file before scanning the directory. Since .gitignore files are not
* that common, we can waste a lot of time looking for files that are
* not there. Fortunately, the fscache already knows if the directory
* contains a .gitignore file, since it has already read the directory
* and it already has the stat-data.
*
* If the fscache is enabled, use the fscache-lstat() interlude to see
* if the file exists (in the fscache hash maps) before trying to open()
* it.
*
* This causes problem when the .gitignore file is a symlink, because
* we call lstat() rather than stat() on the symlnk and the resulting
* stat-data is for the symlink itself rather than the target file.
* We CANNOT use stat() here because the fscache DOES NOT install an
* interlude for stat() and mingw_stat() always calls "open-fstat-close"
* on the file and defeats the purpose of the optimization here. Since
* symlinks are even more rare than .gitignore files, we force a fstat()
* after our open() to get stat-data for the target file.
*
* Since `clang`'s `-Wunreachable-code` mode is clever, it would figure
* out that on non-Windows platforms, this `lstat()` is unreachable.
* We do want to keep the conditional block for the sake of Windows,
* though, so let's use the `NOT_CONSTANT()` trick to suppress that error.
*/
if (NOT_CONSTANT(is_fscache_enabled(fname))) {
if (lstat(fname, &st) < 0) {
fd = -1;
} else {
fd = open(fname, O_RDONLY);
if (fd < 0)
warn_on_fopen_errors(fname);
else if (S_ISLNK(st.st_mode) && fstat(fd, &st) < 0) {
warn_on_fopen_errors(fname);
close(fd);
fd = -1;
}
}
} else {
if (flags & PATTERN_NOFOLLOW)
fd = open_nofollow(fname, O_RDONLY);
else
close(fd);
fd = open(fname, O_RDONLY);
if (fd < 0 || fstat(fd, &st) < 0) {
if (fd < 0)
warn_on_fopen_errors(fname);
else {
close(fd);
fd = -1;
}
}
}
if (fd < 0) {
if (!istate)
return -1;
r = read_skip_worktree_file_from_index(istate, fname,

View File

@ -411,6 +411,9 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca
}
finish:
/* Flush cached lstat in fscache after writing to disk. */
flush_fscache();
if (state->refresh_cache) {
if (!fstat_done && lstat(ce->name, &st) < 0)
return error_errno("unable to stat just-written file %s",

View File

@ -760,6 +760,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
save_commit_buffer = 0;
trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
enable_fscache(0);
for (ref = *refs; ref; ref = ref->next) {
struct commit *commit;
@ -784,6 +785,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
if (!cutoff || cutoff < commit->date)
cutoff = commit->date;
}
disable_fscache();
trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
/*

View File

@ -162,9 +162,11 @@ static inline int is_xplatform_dir_sep(int c)
/* pull in Windows compatibility stuff */
#include "compat/win32/path-utils.h"
#include "compat/mingw.h"
#include "compat/win32/fscache.h"
#elif defined(_MSC_VER)
#include "compat/win32/path-utils.h"
#include "compat/msvc.h"
#include "compat/win32/fscache.h"
#endif
/* used on Mac OS X */
@ -1050,6 +1052,45 @@ static inline int is_missing_file_error(int errno_)
return (errno_ == ENOENT || errno_ == ENOTDIR);
}
/*
* Enable/disable a read-only cache for file system data on platforms that
* support it.
*
* Implementing a live-cache is complicated and requires special platform
* support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used
* to mark sections of git code that extensively read from the file system
* without modifying anything. Implementations can use this to cache e.g. stat
* data or even file content without the need to synchronize with the file
* system.
*/
/* opaque fscache structure */
struct fscache;
#ifndef enable_fscache
#define enable_fscache(x) /* noop */
#endif
#ifndef disable_fscache
#define disable_fscache() /* noop */
#endif
#ifndef is_fscache_enabled
#define is_fscache_enabled(path) (0)
#endif
#ifndef flush_fscache
#define flush_fscache() /* noop */
#endif
#ifndef getcache_fscache
#define getcache_fscache() (NULL) /* noop */
#endif
#ifndef merge_fscache
#define merge_fscache(dest) /* noop */
#endif
int cmd_main(int, const char **);
/*

View File

@ -7,7 +7,9 @@
#include "git-compat-util.h"
#include "mem-pool.h"
#include "gettext.h"
#include "trace.h"
static struct trace_key trace_mem_pool = TRACE_KEY_INIT(MEMPOOL);
#define BLOCK_GROWTH_SIZE (1024 * 1024 - sizeof(struct mp_block))
/*
@ -65,12 +67,20 @@ void mem_pool_init(struct mem_pool *pool, size_t initial_size)
if (initial_size > 0)
mem_pool_alloc_block(pool, initial_size, NULL);
trace_printf_key(&trace_mem_pool,
"mem_pool (%p): init (%"PRIuMAX") initial size\n",
(void *)pool, (uintmax_t)initial_size);
}
void mem_pool_discard(struct mem_pool *pool, int invalidate_memory)
{
struct mp_block *block, *block_to_free;
trace_printf_key(&trace_mem_pool,
"mem_pool (%p): discard (%"PRIuMAX") unused\n",
(void *)pool,
(uintmax_t)(pool->mp_block->end - pool->mp_block->next_free));
block = pool->mp_block;
while (block)
{

View File

@ -1260,6 +1260,7 @@ elif host_machine.system() == 'windows'
'compat/winansi.c',
'compat/win32/dirent.c',
'compat/win32/flush.c',
'compat/win32/fscache.c',
'compat/win32/path-utils.c',
'compat/win32/pthread.c',
'compat/win32/syslog.c',

View File

@ -640,6 +640,7 @@ static void write_items_sequentially(struct checkout *state)
{
size_t i;
flush_fscache();
for (i = 0; i < parallel_checkout.nr; i++) {
struct parallel_checkout_item *pc_item = &parallel_checkout.items[i];
write_pc_item(pc_item, state);

View File

@ -20,6 +20,8 @@
#include "trace2.h"
#include "config.h"
static struct fscache *fscache;
/*
* Mostly randomly chosen maximum thread counts: we
* cap the parallelism to 20 threads, and we want
@ -57,6 +59,7 @@ static void *preload_thread(void *_data)
nr = index->cache_nr - p->offset;
last_nr = nr;
enable_fscache(nr);
do {
struct cache_entry *ce = *cep++;
struct stat st;
@ -100,6 +103,7 @@ static void *preload_thread(void *_data)
pthread_mutex_unlock(&pd->mutex);
}
cache_def_clear(&cache);
merge_fscache(fscache);
return NULL;
}
@ -118,6 +122,7 @@ void preload_index(struct index_state *index,
if (!HAVE_THREADS || !core_preload_index)
return;
fscache = getcache_fscache();
threads = index->cache_nr / THREAD_COST;
if ((index->cache_nr > 1) && (threads < 2) && git_env_bool("GIT_TEST_PRELOAD_INDEX", 0))
threads = 2;

View File

@ -1512,6 +1512,7 @@ int refresh_index(struct index_state *istate, unsigned int flags,
typechange_fmt = in_porcelain ? "T\t%s\n" : "%s: needs update\n";
added_fmt = in_porcelain ? "A\t%s\n" : "%s: needs update\n";
unmerged_fmt = in_porcelain ? "U\t%s\n" : "%s: needs merge\n";
enable_fscache(0);
/*
* Use the multi-threaded preload_index() to refresh most of the
* cache entries quickly then in the single threaded loop below,
@ -1606,6 +1607,7 @@ int refresh_index(struct index_state *istate, unsigned int flags,
display_progress(progress, istate->cache_nr);
stop_progress(&progress);
trace_performance_leave("refresh index");
disable_fscache();
return has_errors;
}

View File

@ -479,6 +479,9 @@ GIT_TEST_NAME_HASH_VERSION=<int>, when set, causes 'git pack-objects' to
assume '--name-hash-version=<n>'.
GIT_TEST_FSCACHE=<boolean> exercises the uncommon fscache code path
which adds a cache below mingw's lstat and dirent implementations.
Naming Tests
------------

View File

@ -270,6 +270,7 @@ integration_tests = [
't2026-checkout-pathspec-file.sh',
't2027-checkout-track.sh',
't2030-unresolve-info.sh',
't2031-checkout-long-paths.sh',
't2050-git-dir-relative.sh',
't2060-switch.sh',
't2070-restore.sh',
@ -888,6 +889,7 @@ integration_tests = [
't7422-submodule-output.sh',
't7423-submodule-symlinks.sh',
't7424-submodule-mixed-ref-formats.sh',
't7429-submodule-long-path.sh',
't7450-bad-git-dotfiles.sh',
't7500-commit-template-squash-signoff.sh',
't7501-commit-basic-functionality.sh',

View File

@ -106,4 +106,24 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs
test_cmp expect actual
'
test_expect_success MINGW 'no unnecessary opendir() with fscache' '
git clone . fscache-test &&
(
cd fscache-test &&
git config core.fscache 1 &&
echo "/excluded/*" >.git/info/sparse-checkout &&
for f in $(test_seq 10)
do
sha1=$(echo $f | git hash-object -w --stdin) &&
git update-index --add \
--cacheinfo 100644,$sha1,excluded/$f || exit 1
done &&
test_tick &&
git commit -m excluded &&
GIT_TRACE_FSCACHE=1 git status >out 2>err &&
grep excluded err >grep.out &&
test_line_count = 1 grep.out
)
'
test_done

111
t/t2031-checkout-long-paths.sh Executable file
View File

@ -0,0 +1,111 @@
#!/bin/sh
test_description='checkout long paths on Windows
Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths'
. ./test-lib.sh
if test_have_prereq !MINGW
then
skip_all='skipping MINGW specific long paths test'
test_done
fi
test_expect_success setup '
p=longpathxx && # -> 10
p=$p$p$p$p$p && # -> 50
p=$p$p$p$p$p && # -> 250
path=${p}/longtestfile && # -> 263 (MAX_PATH = 260)
blob=$(echo foobar | git hash-object -w --stdin) &&
printf "100644 %s 0\t%s\n" "$blob" "$path" |
git update-index --add --index-info &&
git commit -m initial -q
'
test_expect_success 'checkout of long paths without core.longpaths fails' '
git config core.longpaths false &&
test_must_fail git checkout -f 2>error &&
grep -q "Filename too long" error &&
test ! -d longpa*
'
test_expect_success 'checkout of long paths with core.longpaths works' '
git config core.longpaths true &&
git checkout -f &&
test_path_is_file longpa*/longtestfile
'
test_expect_success 'update of long paths' '
echo frotz >>$(ls longpa*/longtestfile) &&
echo $path > expect &&
git ls-files -m > actual &&
test_cmp expect actual &&
git add $path &&
git commit -m second &&
git grep "frotz" HEAD -- $path
'
test_expect_success cleanup '
# bash cannot delete the trash dir if it contains a long path
# lets help cleaning up (unless in debug mode)
if test -z "$debug"
then
rm -rf longpa~1
fi
'
# check that the template used in the test won't be too long:
abspath="$(pwd)"/testdir
test ${#abspath} -gt 230 ||
test_set_prereq SHORTABSPATH
test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' '
p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef &&
p=y$p$p$p$p &&
subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" &&
# Now, $abspath/$subdir has exactly 254 characters, and is inside CWD
p2="$abspath/$subdir" &&
test 254 = ${#p2} &&
# Be careful to overcome path limitations of the MSys tools and split
# the $subdir into two parts. ($subdir2 has to contain 16 chars and a
# slash somewhere following; that is why we asked for abspath <= 230 and
# why we placed a slash near the end of the $subdir template.)
subdir2=${subdir#????????????????*/} &&
subdir1=testdir/${subdir%/$subdir2} &&
mkdir -p "$subdir1" &&
i=0 &&
# The most important case is when absolute path is 258 characters long,
# and that will be when i == 4.
while test $i -le 7
do
mkdir -p $subdir2 &&
touch $subdir2/one-file &&
mv ${subdir2%%/*} "$subdir1/" &&
subdir2=z${subdir2} &&
i=$(($i+1)) ||
exit 1
done &&
# now check that git is able to clear the tree:
(cd testdir &&
git init &&
git config core.longpaths yes &&
git clean -fdx) &&
test ! -d "$subdir1"
'
test_expect_success SYMLINKS_WINDOWS 'leave drive-less, short paths intact' '
printf "/Program Files" >symlink-target &&
symlink_target_oid="$(git hash-object -w --stdin <symlink-target)" &&
git update-index --add --cacheinfo 120000,$symlink_target_oid,PF &&
git -c core.symlinks=true checkout -- PF &&
cmd //c dir >actual &&
grep "<SYMLINKD\\?> *PF *\\[\\\\Program Files\\]" actual
'
test_done

View File

@ -35,6 +35,42 @@ fill () {
}
test_expect_success MINGW 'fscache flush cache' '
git init fscache-test &&
cd fscache-test &&
git config core.fscache 1 &&
echo A > test.txt &&
git add test.txt &&
git commit -m A &&
echo B >> test.txt &&
git checkout . &&
test -z "$(git status -s)" &&
echo A > expect.txt &&
test_cmp expect.txt test.txt &&
cd .. &&
rm -rf fscache-test
'
test_expect_success MINGW 'fscache flush cache dir' '
git init fscache-test &&
cd fscache-test &&
git config core.fscache 1 &&
echo A > test.txt &&
git add test.txt &&
git commit -m A &&
rm test.txt &&
mkdir test.txt &&
touch test.txt/test.txt &&
git checkout . &&
test -z "$(git status -s)" &&
echo A > expect.txt &&
test_cmp expect.txt test.txt &&
cd .. &&
rm -rf fscache-test
'
test_expect_success setup '
fill x y z >same &&
fill 1 2 3 4 5 6 7 8 >one &&

110
t/t7429-submodule-long-path.sh Executable file
View File

@ -0,0 +1,110 @@
#!/bin/sh
#
# Copyright (c) 2013 Doug Kelly
#
test_description='Test submodules with a path near PATH_MAX
This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows)
'
TEST_NO_CREATE_REPO=1
. ./test-lib.sh
# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"),
# which effectively limits the maximum length to PATH_MAX / 2 minus some
# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110)
longpath36=0123456789abcdefghijklmnopqrstuvwxyz
longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36
# the git database must fit within PATH_MAX, which limits the submodule name
# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some
# overhead from the test case)
pwd=$(pwd)
pwdlen=$(echo "$pwd" | wc -c)
longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen)))
test_expect_success 'submodule with a long path' '
git config --global protocol.file.allow always &&
GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \
git -c init.defaultBranch=long init --bare remote &&
test_create_repo bundle1 &&
(
cd bundle1 &&
test_commit "shoot" &&
git rev-parse --verify HEAD >../expect
) &&
mkdir home &&
(
cd home &&
git clone ../remote test &&
cd test &&
git checkout -B long &&
git submodule add ../bundle1 $longpath &&
test_commit "sogood" &&
(
cd $longpath &&
git rev-parse --verify HEAD >actual &&
test_cmp ../../../expect actual
) &&
git push origin long
) &&
mkdir home2 &&
(
cd home2 &&
git clone ../remote test &&
cd test &&
git checkout long &&
git submodule update --init &&
(
cd $longpath &&
git rev-parse --verify HEAD >actual &&
test_cmp ../../../expect actual
)
)
'
test_expect_success 'recursive submodule with a long path' '
GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \
git -c init.defaultBranch=long init --bare super &&
test_create_repo child &&
(
cd child &&
test_commit "shoot" &&
git rev-parse --verify HEAD >../expect
) &&
test_create_repo parent &&
(
cd parent &&
git submodule add ../child $longpath &&
test_commit "aim"
) &&
mkdir home3 &&
(
cd home3 &&
git clone ../super test &&
cd test &&
git checkout -B long &&
git submodule add ../parent foo &&
git submodule update --init --recursive &&
test_commit "sogood" &&
(
cd foo/$longpath &&
git rev-parse --verify HEAD >actual &&
test_cmp ../../../../expect actual
) &&
git push origin long
) &&
mkdir home4 &&
(
cd home4 &&
git clone ../super test --recursive &&
(
cd test/foo/$longpath &&
git rev-parse --verify HEAD >actual &&
test_cmp ../../../../expect actual
)
)
'
test_done

View File

@ -1823,7 +1823,9 @@ static void mark_new_skip_worktree(struct pattern_list *pl,
* 2. Widen worktree according to sparse-checkout file.
* Matched entries will have skip_wt_flag cleared (i.e. "in")
*/
enable_fscache(istate->cache_nr);
clear_ce_flags(istate, select_flag, skip_wt_flag, pl, show_progress);
disable_fscache();
}
static void populate_from_existing_patterns(struct unpack_trees_options *o,