git-for-windows/git
1
0
Fork 0
mirror of https://github.com/git-for-windows/git.git synced 2026-02-03 18:59:59 -06:00
Code Issues Packages Projects Releases 335 Wiki Activity

Merge branch 'fscache'

Browse Source
This commit is contained in:
Johannes Schindelin 2018-11-15 12:23:43 +01:00
commit cc3a447c67
25 changed files with 889 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Documentation/config/core.adoc
View File

@ -710,6 +710,12 @@ relatively high IO latencies. When enabled, Git will do the
index comparison to the filesystem data in parallel, allowing
overlapping IO's. Defaults to true.
core.fscache::
Enable additional caching of file system data for some operations.
+
Git for Windows uses this to bulk-read and cache lstat data of entire
directories (instead of doing lstat file by file).
core.unsetenvvars::
Windows-only: comma-separated list of environment variables'
names that need to be unset before spawning any other process.

5
builtin/add.c
View File

@ -493,6 +493,10 @@ int cmd_add(int argc,
die_in_unpopulated_submodule(repo->index, prefix);
die_path_inside_submodule(repo->index, &pathspec);
enable_fscache(1);
/* We do not really re-read the index but update the up-to-date flags */
preload_index(repo->index, &pathspec, 0);
if (add_new_files) {
int baselen;
@ -605,5 +609,6 @@ finish:
free(ps_matched);
dir_clear(&dir);
clear_pathspec(&pathspec);
enable_fscache(0);
return exit_status;
}

2
builtin/checkout.c
View File

@ -415,6 +415,7 @@ static int checkout_worktree(const struct checkout_opts *opts,
if (pc_workers > 1)
init_parallel_checkout();
enable_fscache(1);
for (pos = 0; pos < the_repository->index->cache_nr; pos++) {
struct cache_entry *ce = the_repository->index->cache[pos];
if (ce->ce_flags & CE_MATCHED) {
@ -440,6 +441,7 @@ static int checkout_worktree(const struct checkout_opts *opts,
errs |= run_parallel_checkout(&state, pc_workers, pc_threshold,
NULL, NULL);
mem_pool_discard(&ce_mem_pool, should_validate_cache_entries());
enable_fscache(0);
remove_marked_cache_entries(the_repository->index, 1);
remove_scheduled_dirs();
errs |= finish_delayed_checkout(&state, opts->show_progress);

1
builtin/commit.c
View File

@ -1623,6 +1623,7 @@ struct repository *repo UNUSED)
PATHSPEC_PREFER_FULL,
prefix, argv);
enable_fscache(1);
if (status_format != STATUS_FORMAT_PORCELAIN &&
status_format != STATUS_FORMAT_PORCELAIN_V2)
progress_flag = REFRESH_PROGRESS;

20
compat/mingw-posix.h
View File

@ -338,6 +338,17 @@ static inline int getrlimit(int resource, struct rlimit *rlp)
return 0;
}
/*
* The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC.
* Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch.
*/
static inline long long filetime_to_hnsec(const FILETIME *ft)
{
long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime;
/* Windows to Unix Epoch conversion */
return winTime - 116444736000000000LL;
}
/*
* Use mingw specific stat()/lstat()/fstat() implementations on Windows,
* including our own struct stat with 64 bit st_size and nanosecond-precision
@ -354,6 +365,13 @@ struct timespec {
#endif
#endif
static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts)
{
long long hnsec = filetime_to_hnsec(ft);
ts->tv_sec = (time_t)(hnsec / 10000000);
ts->tv_nsec = (hnsec % 10000000) * 100;
}
struct mingw_stat {
_dev_t st_dev;
_ino_t st_ino;
@ -386,7 +404,7 @@ int mingw_fstat(int fd, struct stat *buf);
#ifdef lstat
#undef lstat
#endif
#define lstat mingw_lstat
extern int (*lstat)(const char *file_name, struct stat *buf);
int mingw_utime(const char *file_name, const struct utimbuf *times);

26
compat/mingw.c
View File

@ -274,6 +274,7 @@ enum hide_dotfiles_type {
static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY;
static char *unset_environment_variables;
int core_fscache;
int mingw_core_config(const char *var, const char *value,
const struct config_context *ctx UNUSED,
@ -287,6 +288,11 @@ int mingw_core_config(const char *var, const char *value,
return 0;
}
if (!strcmp(var, "core.fscache")) {
core_fscache = git_config_bool(var, value);
return 0;
}
if (!strcmp(var, "core.unsetenvvars")) {
if (!value)
return config_error_nonbool(var);
@ -1058,24 +1064,6 @@ int mingw_chmod(const char *filename, int mode)
return _wchmod(wfilename, mode);
}
/*
* The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC.
* Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch.
*/
static inline long long filetime_to_hnsec(const FILETIME *ft)
{
long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime;
/* Windows to Unix Epoch conversion */
return winTime - 116444736000000000LL;
}
static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts)
{
long long hnsec = filetime_to_hnsec(ft);
ts->tv_sec = (time_t)(hnsec / 10000000);
ts->tv_nsec = (hnsec % 10000000) * 100;
}
/**
* Verifies that safe_create_leading_directories() would succeed.
*/
@ -1268,6 +1256,8 @@ int mingw_lstat(const char *file_name, struct stat *buf)
return -1;
}
int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat;
static int get_file_info_by_handle(HANDLE hnd, struct stat *buf)
{
BY_HANDLE_FILE_INFORMATION fdata;

2
compat/mingw.h
View File

@ -1,5 +1,7 @@
#include "mingw-posix.h"
extern int core_fscache;
struct config_context;
int mingw_core_config(const char *var, const char *value,
const struct config_context *ctx, void *cb);

90
compat/win32/dirent.c
View File

@ -1,15 +1,21 @@
#include "../../git-compat-util.h"
struct DIR {
struct dirent dd_dir; /* includes d_type */
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
typedef struct dirent_DIR {
struct DIR base_dir; /* extend base struct DIR */
HANDLE dd_handle; /* FindFirstFile handle */
int dd_stat; /* 0-based index */
};
struct dirent dd_dir; /* includes d_type */
} dirent_DIR;
#pragma GCC diagnostic pop
DIR *(*opendir)(const char *dirname) = dirent_opendir;
static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata)
{
/* convert UTF-16 name to UTF-8 */
xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name));
/* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */
xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3);
/* Set file type, based on WIN32_FIND_DATA */
if ((fdata->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
@ -21,41 +27,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata)
ent->d_type = DT_REG;
}
DIR *opendir(const char *name)
{
wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */
WIN32_FIND_DATAW fdata;
HANDLE h;
int len;
DIR *dir;
/* convert name to UTF-16 and check length < MAX_PATH */
if ((len = xutftowcs_path(pattern, name)) < 0)
return NULL;
/* append optional '/' and wildcard '*' */
if (len && !is_dir_sep(pattern[len - 1]))
pattern[len++] = '/';
pattern[len++] = '*';
pattern[len] = 0;
/* open find handle */
h = FindFirstFileW(pattern, &fdata);
if (h == INVALID_HANDLE_VALUE) {
DWORD err = GetLastError();
errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
return NULL;
}
/* initialize DIR structure and copy first dir entry */
dir = xmalloc(sizeof(DIR));
dir->dd_handle = h;
dir->dd_stat = 0;
finddata2dirent(&dir->dd_dir, &fdata);
return dir;
}
struct dirent *readdir(DIR *dir)
static struct dirent *dirent_readdir(dirent_DIR *dir)
{
if (!dir) {
errno = EBADF; /* No set_errno for mingw */
@ -82,7 +54,7 @@ struct dirent *readdir(DIR *dir)
return &dir->dd_dir;
}
int closedir(DIR *dir)
static int dirent_closedir(dirent_DIR *dir)
{
if (!dir) {
errno = EBADF;
@ -93,3 +65,39 @@ int closedir(DIR *dir)
free(dir);
return 0;
}
DIR *dirent_opendir(const char *name)
{
wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */
WIN32_FIND_DATAW fdata;
HANDLE h;
int len;
dirent_DIR *dir;
/* convert name to UTF-16 and check length < MAX_PATH */
if ((len = xutftowcs_path(pattern, name)) < 0)
return NULL;
/* append optional '/' and wildcard '*' */
if (len && !is_dir_sep(pattern[len - 1]))
pattern[len++] = '/';
pattern[len++] = '*';
pattern[len] = 0;
/* open find handle */
h = FindFirstFileW(pattern, &fdata);
if (h == INVALID_HANDLE_VALUE) {
DWORD err = GetLastError();
errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
return NULL;
}
/* initialize DIR structure and copy first dir entry */
dir = xmalloc(sizeof(dirent_DIR) + MAX_PATH);
dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir;
dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir;
dir->dd_handle = h;
dir->dd_stat = 0;
finddata2dirent(&dir->dd_dir, &fdata);
return (DIR*) dir;
}

28
compat/win32/dirent.h
View File

@ -1,20 +1,34 @@
#ifndef DIRENT_H
#define DIRENT_H
typedef struct DIR DIR;
#define DT_UNKNOWN 0
#define DT_DIR 1
#define DT_REG 2
#define DT_LNK 3
struct dirent {
unsigned char d_type; /* file type to prevent lstat after readdir */
char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */
unsigned char d_type; /* file type to prevent lstat after readdir */
char d_name[/* FLEX_ARRAY */]; /* file name */
};
DIR *opendir(const char *dirname);
struct dirent *readdir(DIR *dir);
int closedir(DIR *dir);
/*
* Base DIR structure, contains pointers to readdir/closedir implementations so
* that opendir may choose a concrete implementation on a call-by-call basis.
*/
typedef struct DIR {
struct dirent *(*preaddir)(struct DIR *dir);
int (*pclosedir)(struct DIR *dir);
} DIR;
/* default dirent implementation */
extern DIR *dirent_opendir(const char *dirname);
#define opendir git_opendir
/* current dirent implementation */
extern DIR *(*opendir)(const char *dirname);
#define readdir(dir) (dir->preaddir(dir))
#define closedir(dir) (dir->pclosedir(dir))
#endif /* DIRENT_H */

601
compat/win32/fscache.c Normal file
View File

@ -0,0 +1,601 @@
#include "../../git-compat-util.h"
#include "../../hashmap.h"
#include "../win32.h"
#include "fscache.h"
#include "../../dir.h"
#include "../../abspath.h"
#include "../../trace.h"
#include "config.h"
static int initialized;
static volatile long enabled;
static struct hashmap map;
static CRITICAL_SECTION mutex;
static unsigned int lstat_requests;
static unsigned int opendir_requests;
static unsigned int fscache_requests;
static unsigned int fscache_misses;
static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE);
/*
* An entry in the file system cache. Used for both entire directory listings
* and file entries.
*/
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
struct fsentry {
struct hashmap_entry ent;
mode_t st_mode;
/* Pointer to the directory listing, or NULL for the listing itself. */
struct fsentry *list;
/* Pointer to the next file entry of the list. */
struct fsentry *next;
union {
/* Reference count of the directory listing. */
volatile long refcnt;
/* Handle to wait on the loading thread. */
HANDLE hwait;
struct {
/* More stat members (only used for file entries). */
off64_t st_size;
struct timespec st_atim;
struct timespec st_mtim;
struct timespec st_ctim;
} s;
} u;
/* Length of name. */
unsigned short len;
/*
* Name of the entry. For directory listings: relative path of the
* directory, without trailing '/' (empty for cwd()). For file entries:
* name of the file. Typically points to the end of the structure if
* the fsentry is allocated on the heap (see fsentry_alloc), or to a
* local variable if on the stack (see fsentry_init).
*/
struct dirent dirent;
};
#pragma GCC diagnostic pop
#pragma GCC diagnostic push
#ifdef __clang__
#pragma GCC diagnostic ignored "-Wflexible-array-extensions"
#endif
struct heap_fsentry {
union {
struct fsentry ent;
char dummy[sizeof(struct fsentry) + MAX_PATH];
} u;
};
#pragma GCC diagnostic pop
/*
* Compares the paths of two fsentry structures for equality.
*/
static int fsentry_cmp(void *cmp_data UNUSED,
const struct fsentry *fse1, const struct fsentry *fse2,
void *keydata UNUSED)
{
int res;
if (fse1 == fse2)
return 0;
/* compare the list parts first */
if (fse1->list != fse2->list &&
(res = fsentry_cmp(NULL, fse1->list ? fse1->list : fse1,
fse2->list ? fse2->list : fse2, NULL)))
return res;
/* if list parts are equal, compare len and name */
if (fse1->len != fse2->len)
return fse1->len - fse2->len;
return fspathncmp(fse1->dirent.d_name, fse2->dirent.d_name, fse1->len);
}
/*
* Calculates the hash code of an fsentry structure's path.
*/
static unsigned int fsentry_hash(const struct fsentry *fse)
{
unsigned int hash = fse->list ? fse->list->ent.hash : 0;
return hash ^ memihash(fse->dirent.d_name, fse->len);
}
/*
* Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp.
*/
static void fsentry_init(struct fsentry *fse, struct fsentry *list,
const char *name, size_t len)
{
fse->list = list;
if (len > MAX_PATH)
BUG("Trying to allocate fsentry for long path '%.*s'",
(int)len, name);
memcpy(fse->dirent.d_name, name, len);
fse->dirent.d_name[len] = 0;
fse->len = len;
hashmap_entry_init(&fse->ent, fsentry_hash(fse));
}
/*
* Allocate an fsentry structure on the heap.
*/
static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name,
size_t len)
{
/* overallocate fsentry and copy the name to the end */
struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1);
/* init the rest of the structure */
fsentry_init(fse, list, name, len);
fse->next = NULL;
fse->u.refcnt = 1;
return fse;
}
/*
* Add a reference to an fsentry.
*/
inline static void fsentry_addref(struct fsentry *fse)
{
if (fse->list)
fse = fse->list;
InterlockedIncrement(&(fse->u.refcnt));
}
/*
* Release the reference to an fsentry, frees the memory if its the last ref.
*/
static void fsentry_release(struct fsentry *fse)
{
if (fse->list)
fse = fse->list;
if (InterlockedDecrement(&(fse->u.refcnt)))
return;
while (fse) {
struct fsentry *next = fse->next;
free(fse);
fse = next;
}
}
/*
* Allocate and initialize an fsentry from a WIN32_FIND_DATA structure.
*/
static struct fsentry *fseentry_create_entry(struct fsentry *list,
const WIN32_FIND_DATAW *fdata)
{
char buf[MAX_PATH * 3];
int len;
struct fsentry *fse;
len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf));
fse = fsentry_alloc(list, buf, len);
fse->st_mode = file_attr_to_st_mode(fdata->FileAttributes,
fdata->EaSize);
fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG :
S_ISDIR(fse->st_mode) ? DT_DIR : DT_LNK;
fse->u.s.st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32)
| fdata->nFileSizeLow;
filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->u.s.st_atim));
filetime_to_timespec(&(fdata->ftLastWriteTime), &(fse->u.s.st_mtim));
filetime_to_timespec(&(fdata->ftCreationTime), &(fse->u.s.st_ctim));
return fse;
}
/*
* Create an fsentry-based directory listing (similar to opendir / readdir).
* Dir should not contain trailing '/'. Use an empty string for the current
* directory (not "."!).
*/
static struct fsentry *fsentry_create_list(const struct fsentry *dir,
int *dir_not_found)
{
wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */
WIN32_FIND_DATAW fdata;
HANDLE h;
int wlen;
struct fsentry *list, **phead;
DWORD err;
*dir_not_found = 0;
/* convert name to UTF-16 and check length < MAX_PATH */
if ((wlen = xutftowcsn(pattern, dir->dirent.d_name, MAX_PATH,
dir->len)) < 0) {
if (errno == ERANGE)
errno = ENAMETOOLONG;
return NULL;
}
/* append optional '/' and wildcard '*' */
if (wlen)
pattern[wlen++] = '/';
pattern[wlen++] = '*';
pattern[wlen] = 0;
/* open find handle */
h = FindFirstFileExW(pattern, FindExInfoBasic, &fdata, FindExSearchNameMatch,
NULL, FIND_FIRST_EX_LARGE_FETCH);
if (h == INVALID_HANDLE_VALUE) {
err = GetLastError();
*dir_not_found = 1; /* or empty directory */
errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err);
trace_printf_key(&trace_fscache, "fscache: error(%d) '%s'\n",
errno, dir->dirent.d_name);
return NULL;
}
/* allocate object to hold directory listing */
list = fsentry_alloc(NULL, dir->dirent.d_name, dir->len);
list->st_mode = S_IFDIR;
list->dirent.d_type = DT_DIR;
/* walk directory and build linked list of fsentry structures */
phead = &list->next;
do {
*phead = fseentry_create_entry(list, &fdata);
phead = &(*phead)->next;
} while (FindNextFileW(h, &fdata));
/* remember result of last FindNextFile, then close find handle */
err = GetLastError();
FindClose(h);
/* return the list if we've got all the files */
if (err == ERROR_NO_MORE_FILES)
return list;
/* otherwise free the list and return error */
fsentry_release(list);
errno = err_win_to_posix(err);
return NULL;
}
/*
* Adds a directory listing to the cache.
*/
static void fscache_add(struct fsentry *fse)
{
if (fse->list)
fse = fse->list;
for (; fse; fse = fse->next)
hashmap_add(&map, &fse->ent);
}
/*
* Clears the cache.
*/
static void fscache_clear(void)
{
hashmap_clear_and_free(&map, struct fsentry, ent);
hashmap_init(&map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0);
lstat_requests = opendir_requests = 0;
fscache_misses = fscache_requests = 0;
}
/*
* Checks if the cache is enabled for the given path.
*/
int fscache_enabled(const char *path)
{
return enabled > 0 && !is_absolute_path(path);
}
/*
* Looks up a cache entry, waits if its being loaded by another thread.
* The mutex must be owned by the calling thread.
*/
static struct fsentry *fscache_get_wait(struct fsentry *key)
{
struct fsentry *fse = hashmap_get_entry(&map, key, ent, NULL);
/* return if its a 'real' entry (future entries have refcnt == 0) */
if (!fse || fse->list || fse->u.refcnt)
return fse;
/* create an event and link our key to the future entry */
key->u.hwait = CreateEvent(NULL, TRUE, FALSE, NULL);
key->next = fse->next;
fse->next = key;
/* wait for the loading thread to signal us */
LeaveCriticalSection(&mutex);
WaitForSingleObject(key->u.hwait, INFINITE);
CloseHandle(key->u.hwait);
EnterCriticalSection(&mutex);
/* repeat cache lookup */
return hashmap_get_entry(&map, key, ent, NULL);
}
/*
* Looks up or creates a cache entry for the specified key.
*/
static struct fsentry *fscache_get(struct fsentry *key)
{
struct fsentry *fse, *future, *waiter;
int dir_not_found;
EnterCriticalSection(&mutex);
fscache_requests++;
/* check if entry is in cache */
fse = fscache_get_wait(key);
if (fse) {
if (fse->st_mode)
fsentry_addref(fse);
else
fse = NULL; /* non-existing directory */
LeaveCriticalSection(&mutex);
return fse;
}
/* if looking for a file, check if directory listing is in cache */
if (!fse && key->list) {
fse = fscache_get_wait(key->list);
if (fse) {
LeaveCriticalSection(&mutex);
/*
* dir entry without file entry, or dir does not
* exist -> file doesn't exist
*/
errno = ENOENT;
return NULL;
}
}
/* add future entry to indicate that we're loading it */
future = key->list ? key->list : key;
future->next = NULL;
future->u.refcnt = 0;
hashmap_add(&map, &future->ent);
/* create the directory listing (outside mutex!) */
LeaveCriticalSection(&mutex);
fse = fsentry_create_list(future, &dir_not_found);
EnterCriticalSection(&mutex);
/* remove future entry and signal waiting threads */
hashmap_remove(&map, &future->ent, NULL);
waiter = future->next;
while (waiter) {
HANDLE h = waiter->u.hwait;
waiter = waiter->next;
SetEvent(h);
}
/* leave on error (errno set by fsentry_create_list) */
if (!fse) {
if (dir_not_found && key->list) {
/*
* Record that the directory does not exist (or is
* empty, which for all practical matters is the same
* thing as far as fscache is concerned).
*/
fse = fsentry_alloc(key->list->list,
key->list->dirent.d_name,
key->list->len);
fse->st_mode = 0;
hashmap_add(&map, &fse->ent);
}
LeaveCriticalSection(&mutex);
return NULL;
}
/* add directory listing to the cache */
fscache_misses++;
fscache_add(fse);
/* lookup file entry if requested (fse already points to directory) */
if (key->list)
fse = hashmap_get_entry(&map, key, ent, NULL);
if (fse && !fse->st_mode)
fse = NULL; /* non-existing directory */
/* return entry or ENOENT */
if (fse)
fsentry_addref(fse);
else
errno = ENOENT;
LeaveCriticalSection(&mutex);
return fse;
}
/*
* Enables or disables the cache. Note that the cache is read-only, changes to
* the working directory are NOT reflected in the cache while enabled.
*/
int fscache_enable(int enable)
{
int result;
if (!initialized) {
int fscache = git_env_bool("GIT_TEST_FSCACHE", -1);
/* allow the cache to be disabled entirely */
if (fscache != -1)
core_fscache = fscache;
if (!core_fscache)
return 0;
InitializeCriticalSection(&mutex);
lstat_requests = opendir_requests = 0;
fscache_misses = fscache_requests = 0;
hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, 0);
initialized = 1;
}
result = enable ? InterlockedIncrement(&enabled)
: InterlockedDecrement(&enabled);
if (enable && result == 1) {
/* redirect opendir and lstat to the fscache implementations */
opendir = fscache_opendir;
lstat = fscache_lstat;
} else if (!enable && !result) {
/* reset opendir and lstat to the original implementations */
opendir = dirent_opendir;
lstat = mingw_lstat;
EnterCriticalSection(&mutex);
trace_printf_key(&trace_fscache, "fscache: lstat %u, opendir %u, "
"total requests/misses %u/%u\n",
lstat_requests, opendir_requests,
fscache_requests, fscache_misses);
fscache_clear();
LeaveCriticalSection(&mutex);
}
trace_printf_key(&trace_fscache, "fscache: enable(%d)\n", enable);
return result;
}
/*
* Flush cached stats result when fscache is enabled.
*/
void fscache_flush(void)
{
if (enabled) {
EnterCriticalSection(&mutex);
fscache_clear();
LeaveCriticalSection(&mutex);
}
}
/*
* Lstat replacement, uses the cache if enabled, otherwise redirects to
* mingw_lstat.
*/
int fscache_lstat(const char *filename, struct stat *st)
{
int dirlen, base, len;
#pragma GCC diagnostic push
#ifdef __clang__
#pragma GCC diagnostic ignored "-Wflexible-array-extensions"
#endif
struct heap_fsentry key[2];
#pragma GCC diagnostic pop
struct fsentry *fse;
if (!fscache_enabled(filename))
return mingw_lstat(filename, st);
lstat_requests++;
/* split filename into path + name */
len = strlen(filename);
if (len && is_dir_sep(filename[len - 1]))
len--;
base = len;
while (base && !is_dir_sep(filename[base - 1]))
base--;
dirlen = base ? base - 1 : 0;
/* lookup entry for path + name in cache */
fsentry_init(&key[0].u.ent, NULL, filename, dirlen);
fsentry_init(&key[1].u.ent, &key[0].u.ent, filename + base, len - base);
fse = fscache_get(&key[1].u.ent);
if (!fse) {
errno = ENOENT;
return -1;
}
/*
* Special case symbolic links: FindFirstFile()/FindNextFile() did not
* provide us with the length of the target path.
*/
if (fse->u.s.st_size == MAX_PATH && S_ISLNK(fse->st_mode)) {
char buf[MAX_LONG_PATH];
int len = readlink(filename, buf, sizeof(buf) - 1);
if (len > 0)
fse->u.s.st_size = len;
}
/* copy stat data */
st->st_ino = 0;
st->st_gid = 0;
st->st_uid = 0;
st->st_dev = 0;
st->st_rdev = 0;
st->st_nlink = 1;
st->st_mode = fse->st_mode;
st->st_size = fse->u.s.st_size;
st->st_atim = fse->u.s.st_atim;
st->st_mtim = fse->u.s.st_mtim;
st->st_ctim = fse->u.s.st_ctim;
/* don't forget to release fsentry */
fsentry_release(fse);
return 0;
}
typedef struct fscache_DIR {
struct DIR base_dir; /* extend base struct DIR */
struct fsentry *pfsentry;
struct dirent *dirent;
} fscache_DIR;
/*
* Readdir replacement.
*/
static struct dirent *fscache_readdir(DIR *base_dir)
{
fscache_DIR *dir = (fscache_DIR*) base_dir;
struct fsentry *next = dir->pfsentry->next;
if (!next)
return NULL;
dir->pfsentry = next;
dir->dirent = &next->dirent;
return dir->dirent;
}
/*
* Closedir replacement.
*/
static int fscache_closedir(DIR *base_dir)
{
fscache_DIR *dir = (fscache_DIR*) base_dir;
fsentry_release(dir->pfsentry);
free(dir);
return 0;
}
/*
* Opendir replacement, uses a directory listing from the cache if enabled,
* otherwise calls original dirent implementation.
*/
DIR *fscache_opendir(const char *dirname)
{
struct heap_fsentry key;
struct fsentry *list;
fscache_DIR *dir;
int len;
if (!fscache_enabled(dirname))
return dirent_opendir(dirname);
opendir_requests++;
/* prepare name (strip trailing '/', replace '.') */
len = strlen(dirname);
if ((len == 1 && dirname[0] == '.') ||
(len && is_dir_sep(dirname[len - 1])))
len--;
/* get directory listing from cache */
fsentry_init(&key.u.ent, NULL, dirname, len);
list = fscache_get(&key.u.ent);
if (!list)
return NULL;
/* alloc and return DIR structure */
dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR));
dir->base_dir.preaddir = fscache_readdir;
dir->base_dir.pclosedir = fscache_closedir;
dir->pfsentry = list;
return (DIR*) dir;
}

16
compat/win32/fscache.h Normal file
View File

@ -0,0 +1,16 @@
#ifndef FSCACHE_H
#define FSCACHE_H
int fscache_enable(int enable);
#define enable_fscache(x) fscache_enable(x)
int fscache_enabled(const char *path);
#define is_fscache_enabled(path) fscache_enabled(path)
void fscache_flush(void);
#define flush_fscache() fscache_flush()
DIR *fscache_opendir(const char *dir);
int fscache_lstat(const char *file_name, struct stat *buf);
#endif

4
config.mak.uname
View File

@ -509,7 +509,7 @@ endif
compat/win32/path-utils.o \
compat/win32/pthread.o compat/win32/syslog.o \
compat/win32/trace2_win32_process_info.o \
compat/win32/dirent.o
compat/win32/dirent.o compat/win32/fscache.o
COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY \
-DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" \
-DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\"
@ -714,7 +714,7 @@ ifeq ($(uname_S),MINGW)
compat/win32/flush.o \
compat/win32/path-utils.o \
compat/win32/pthread.o compat/win32/syslog.o \
compat/win32/dirent.o
compat/win32/dirent.o compat/win32/fscache.o
BASIC_CFLAGS += -DWIN32
EXTLIBS += -lws2_32
GITLIBS += git.res

3
contrib/buildsystems/CMakeLists.txt
View File

@ -301,7 +301,8 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
compat/win32/trace2_win32_process_info.c
compat/win32/dirent.c
compat/nedmalloc/nedmalloc.c
compat/strdup.c)
compat/strdup.c
compat/win32/fscache.c)
set(NO_UNIX_SOCKETS 1)
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")

66
dir.c
View File

@ -1156,16 +1156,64 @@ static int add_patterns(const char *fname, const char *base, int baselen,
size_t size = 0;
char *buf;
if (flags & PATTERN_NOFOLLOW)
fd = open_nofollow(fname, O_RDONLY);
else
fd = open(fname, O_RDONLY);
if (fd < 0 || fstat(fd, &st) < 0) {
if (fd < 0)
warn_on_fopen_errors(fname);
/*
* A performance optimization for status.
*
* During a status scan, git looks in each directory for a .gitignore
* file before scanning the directory. Since .gitignore files are not
* that common, we can waste a lot of time looking for files that are
* not there. Fortunately, the fscache already knows if the directory
* contains a .gitignore file, since it has already read the directory
* and it already has the stat-data.
*
* If the fscache is enabled, use the fscache-lstat() interlude to see
* if the file exists (in the fscache hash maps) before trying to open()
* it.
*
* This causes problem when the .gitignore file is a symlink, because
* we call lstat() rather than stat() on the symlnk and the resulting
* stat-data is for the symlink itself rather than the target file.
* We CANNOT use stat() here because the fscache DOES NOT install an
* interlude for stat() and mingw_stat() always calls "open-fstat-close"
* on the file and defeats the purpose of the optimization here. Since
* symlinks are even more rare than .gitignore files, we force a fstat()
* after our open() to get stat-data for the target file.
*
* Since `clang`'s `-Wunreachable-code` mode is clever, it would figure
* out that on non-Windows platforms, this `lstat()` is unreachable.
* We do want to keep the conditional block for the sake of Windows,
* though, so let's use the `NOT_CONSTANT()` trick to suppress that error.
*/
if (NOT_CONSTANT(is_fscache_enabled(fname))) {
if (lstat(fname, &st) < 0) {
fd = -1;
} else {
fd = open(fname, O_RDONLY);
if (fd < 0)
warn_on_fopen_errors(fname);
else if (S_ISLNK(st.st_mode) && fstat(fd, &st) < 0) {
warn_on_fopen_errors(fname);
close(fd);
fd = -1;
}
}
} else {
if (flags & PATTERN_NOFOLLOW)
fd = open_nofollow(fname, O_RDONLY);
else
close(fd);
fd = open(fname, O_RDONLY);
if (fd < 0 || fstat(fd, &st) < 0) {
if (fd < 0)
warn_on_fopen_errors(fname);
else {
close(fd);
fd = -1;
}
}
}
if (fd < 0) {
if (!istate)
return -1;
r = read_skip_worktree_file_from_index(istate, fname,

3
entry.c
View File

@ -411,6 +411,9 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca
}
finish:
/* Flush cached lstat in fscache after writing to disk. */
flush_fscache();
if (state->refresh_cache) {
if (!fstat_done && lstat(ce->name, &st) < 0)
return error_errno("unable to stat just-written file %s",

2
fetch-pack.c
View File

@ -760,6 +760,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
save_commit_buffer = 0;
trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
enable_fscache(1);
for (ref = *refs; ref; ref = ref->next) {
struct commit *commit;
@ -784,6 +785,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
if (!cutoff || cutoff < commit->date)
cutoff = commit->date;
}
enable_fscache(0);
trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
/*

25
git-compat-util.h
View File

@ -162,9 +162,11 @@ static inline int is_xplatform_dir_sep(int c)
/* pull in Windows compatibility stuff */
#include "compat/win32/path-utils.h"
#include "compat/mingw.h"
#include "compat/win32/fscache.h"
#elif defined(_MSC_VER)
#include "compat/win32/path-utils.h"
#include "compat/msvc.h"
#include "compat/win32/fscache.h"
#endif
/* used on Mac OS X */
@ -1050,6 +1052,29 @@ static inline int is_missing_file_error(int errno_)
return (errno_ == ENOENT || errno_ == ENOTDIR);
}
/*
* Enable/disable a read-only cache for file system data on platforms that
* support it.
*
* Implementing a live-cache is complicated and requires special platform
* support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used
* to mark sections of git code that extensively read from the file system
* without modifying anything. Implementations can use this to cache e.g. stat
* data or even file content without the need to synchronize with the file
* system.
*/
#ifndef enable_fscache
#define enable_fscache(x) /* noop */
#endif
#ifndef is_fscache_enabled
#define is_fscache_enabled(path) (0)
#endif
#ifndef flush_fscache
#define flush_fscache() /* noop */
#endif
int cmd_main(int, const char **);
/*

1
meson.build
View File

@ -1260,6 +1260,7 @@ elif host_machine.system() == 'windows'
'compat/winansi.c',
'compat/win32/dirent.c',
'compat/win32/flush.c',
'compat/win32/fscache.c',
'compat/win32/path-utils.c',
'compat/win32/pthread.c',
'compat/win32/syslog.c',

1
parallel-checkout.c
View File

@ -640,6 +640,7 @@ static void write_items_sequentially(struct checkout *state)
{
size_t i;
flush_fscache();
for (i = 0; i < parallel_checkout.nr; i++) {
struct parallel_checkout_item *pc_item = &parallel_checkout.items[i];
write_pc_item(pc_item, state);

3
preload-index.c
View File

@ -141,6 +141,7 @@ void preload_index(struct index_state *index,
pthread_mutex_init(&pd.mutex, NULL);
}
enable_fscache(1);
for (i = 0; i < threads; i++) {
struct thread_data *p = data+i;
int err;
@ -176,6 +177,8 @@ void preload_index(struct index_state *index,
trace2_data_intmax("index", NULL, "preload/sum_lstat", t2_sum_lstat);
trace2_region_leave("index", "preload", NULL);
enable_fscache(0);
}
int repo_read_index_preload(struct repository *repo,

2
read-cache.c
View File

@ -1512,6 +1512,7 @@ int refresh_index(struct index_state *istate, unsigned int flags,
typechange_fmt = in_porcelain ? "T\t%s\n" : "%s: needs update\n";
added_fmt = in_porcelain ? "A\t%s\n" : "%s: needs update\n";
unmerged_fmt = in_porcelain ? "U\t%s\n" : "%s: needs merge\n";
enable_fscache(1);
/*
* Use the multi-threaded preload_index() to refresh most of the
* cache entries quickly then in the single threaded loop below,
@ -1606,6 +1607,7 @@ int refresh_index(struct index_state *istate, unsigned int flags,
display_progress(progress, istate->cache_nr);
stop_progress(&progress);
trace_performance_leave("refresh index");
enable_fscache(0);
return has_errors;
}

3
t/README
View File

@ -479,6 +479,9 @@ GIT_TEST_NAME_HASH_VERSION=<int>, when set, causes 'git pack-objects' to
assume '--name-hash-version=<n>'.
GIT_TEST_FSCACHE=<boolean> exercises the uncommon fscache code path
which adds a cache below mingw's lstat and dirent implementations.
Naming Tests
------------

20
t/t1090-sparse-checkout-scope.sh
View File

@ -106,4 +106,24 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs
test_cmp expect actual
'
test_expect_success MINGW 'no unnecessary opendir() with fscache' '
git clone . fscache-test &&
(
cd fscache-test &&
git config core.fscache 1 &&
echo "/excluded/*" >.git/info/sparse-checkout &&
for f in $(test_seq 10)
do
sha1=$(echo $f | git hash-object -w --stdin) &&
git update-index --add \
--cacheinfo 100644,$sha1,excluded/$f || exit 1
done &&
test_tick &&
git commit -m excluded &&
GIT_TRACE_FSCACHE=1 git status >out 2>err &&
grep excluded err >grep.out &&
test_line_count = 1 grep.out
)
'
test_done

36
t/t7201-co.sh
View File

@ -35,6 +35,42 @@ fill () {
}
test_expect_success MINGW 'fscache flush cache' '
git init fscache-test &&
cd fscache-test &&
git config core.fscache 1 &&
echo A > test.txt &&
git add test.txt &&
git commit -m A &&
echo B >> test.txt &&
git checkout . &&
test -z "$(git status -s)" &&
echo A > expect.txt &&
test_cmp expect.txt test.txt &&
cd .. &&
rm -rf fscache-test
'
test_expect_success MINGW 'fscache flush cache dir' '
git init fscache-test &&
cd fscache-test &&
git config core.fscache 1 &&
echo A > test.txt &&
git add test.txt &&
git commit -m A &&
rm test.txt &&
mkdir test.txt &&
touch test.txt/test.txt &&
git checkout . &&
test -z "$(git status -s)" &&
echo A > expect.txt &&
test_cmp expect.txt test.txt &&
cd .. &&
rm -rf fscache-test
'
test_expect_success setup '
fill x y z >same &&
fill 1 2 3 4 5 6 7 8 >one &&

2
unpack-trees.c
View File

@ -1823,7 +1823,9 @@ static void mark_new_skip_worktree(struct pattern_list *pl,
* 2. Widen worktree according to sparse-checkout file.
* Matched entries will have skip_wt_flag cleared (i.e. "in")
*/
enable_fscache(istate->cache_nr);
clear_ce_flags(istate, select_flag, skip_wt_flag, pl, show_progress);
disable_fscache();
}
static void populate_from_existing_patterns(struct unpack_trees_options *o,