From e9169a308e34437a5ef44c08885a65b66a301e2c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Jun 2021 18:04:56 -0400 Subject: [PATCH 01/22] compat/fsmonitor/fsm-listen-win32: handle shortnames Teach FSMonitor daemon on Windows to recognize shortname paths as aliases of normal longname paths. FSMonitor clients, such as `git status`, should receive the longname spelling of changed files (when possible). Sometimes we receive FS events using the shortname, such as when a CMD shell runs "RENAME GIT~1 FOO" or "RMDIR GIT~1". The FS notification arrives using whatever combination of long and shortnames were used by the other process. (Shortnames do seem to be case normalized, however.) Use Windows GetLongPathNameW() to try to map the pathname spelling in the notification event into the normalized longname spelling. (This can fail if the file/directory is deleted, moved, or renamed, because we are asking the FS for the mapping in response to the event and after it has already happened, but we try.) Special case the shortname spelling of ".git" to avoid under-reporting these events. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-win32.c | 361 +++++++++++++++++++++++----- t/t7527-builtin-fsmonitor.sh | 65 +++++ 2 files changed, 371 insertions(+), 55 deletions(-) diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index c2d11acbc1..eb407b0748 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -25,6 +25,9 @@ struct one_watch DWORD count; struct strbuf path; + wchar_t wpath_longname[MAX_PATH + 1]; + DWORD wpath_longname_len; + HANDLE hDir; HANDLE hEvent; OVERLAPPED overlapped; @@ -34,6 +37,21 @@ struct one_watch * need to later call GetOverlappedResult() and possibly CancelIoEx(). */ BOOL is_active; + + /* + * Are shortnames enabled on the containing drive? This is + * always true for "C:/" drives and usually never true for + * other drives. + * + * We only set this for the worktree because we only need to + * convert shortname paths to longname paths for items we send + * to clients. (We don't care about shortname expansion for + * paths inside a GITDIR because we never send them to + * clients.) + */ + BOOL has_shortnames; + BOOL has_tilda; + wchar_t dotgit_shortname[16]; /* for 8.3 name */ }; struct fsmonitor_daemon_backend_data @@ -51,17 +69,18 @@ struct fsmonitor_daemon_backend_data }; /* - * Convert the WCHAR path from the notification into UTF8 and - * then normalize it. + * Convert the WCHAR path from the event into UTF8 and normalize it. + * + * `wpath_len` is in WCHARS not bytes. */ -static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, +static int normalize_path_in_utf8(wchar_t *wpath, DWORD wpath_len, struct strbuf *normalized_path) { int reserve; int len = 0; strbuf_reset(normalized_path); - if (!info->FileNameLength) + if (!wpath_len) goto normalize; /* @@ -70,12 +89,12 @@ static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, * sequence of 2 UTF8 characters. That should let us * avoid ERROR_INSUFFICIENT_BUFFER 99.9+% of the time. */ - reserve = info->FileNameLength + 1; + reserve = 2 * wpath_len + 1; strbuf_grow(normalized_path, reserve); for (;;) { - len = WideCharToMultiByte(CP_UTF8, 0, info->FileName, - info->FileNameLength / sizeof(WCHAR), + len = WideCharToMultiByte(CP_UTF8, 0, + wpath, wpath_len, normalized_path->buf, strbuf_avail(normalized_path) - 1, NULL, NULL); @@ -83,9 +102,7 @@ static int normalize_path_in_utf8(FILE_NOTIFY_INFORMATION *info, goto normalize; if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { error("[GLE %ld] could not convert path to UTF-8: '%.*ls'", - GetLastError(), - (int)(info->FileNameLength / sizeof(WCHAR)), - info->FileName); + GetLastError(), (int)wpath_len, wpath); return -1; } @@ -98,6 +115,148 @@ normalize: return strbuf_normalize_path(normalized_path); } +/* + * See if the worktree root directory has shortnames enabled. + * This will help us decide if we need to do an expensive shortname + * to longname conversion on every notification event. + * + * We do not want to create a file to test this, so we assume that the + * root directory contains a ".git" file or directory. (Out caller + * only calls us for the worktree root, so this should be fine.) + * + * Remember the spelling of the shortname for ".git" if it exists. + */ +static void check_for_shortnames(struct one_watch *watch) +{ + wchar_t buf_in[MAX_PATH + 1]; + wchar_t buf_out[MAX_PATH + 1]; + wchar_t *last_slash = NULL; + wchar_t *last_bslash = NULL; + wchar_t *last; + + /* build L"/.git" */ + wcscpy(buf_in, watch->wpath_longname); + wcscpy(buf_in + watch->wpath_longname_len, L".git"); + + if (!GetShortPathNameW(buf_in, buf_out, MAX_PATH)) + return; + + last_slash = wcsrchr(buf_out, L'/'); + last_bslash = wcsrchr(buf_out, L'\\'); + if (last_slash > last_bslash) + last = last_slash + 1; + else if (last_bslash) + last = last_bslash + 1; + else + last = buf_out; + + if (!wcscmp(last, L".git")) + return; + + watch->has_shortnames = 1; + wcsncpy(watch->dotgit_shortname, last, + ARRAY_SIZE(watch->dotgit_shortname)); + + /* + * The shortname for ".git" is usually of the form "GIT~1", so + * we should be able to avoid shortname to longname mapping on + * every notification event if the source string does not + * contain a "~". + * + * However, the documentation for GetLongPathNameW() says + * that there are filesystems that don't follow that pattern + * and warns against this optimization. + * + * Lets test this. + */ + if (wcschr(watch->dotgit_shortname, L'~')) + watch->has_tilda = 1; +} + +enum get_relative_result { + GRR_NO_CONVERSION_NEEDED, + GRR_HAVE_CONVERSION, + GRR_SHUTDOWN, +}; + +/* + * Info notification paths are relative to the root of the watch. + * If our CWD is still at the root, then we can use relative paths + * to convert from shortnames to longnames. If our process has a + * different CWD, then we need to construct an absolute path, do + * the conversion, and then return the root-relative portion. + * + * We use the longname form of the root as our basis and assume that + * it already has a trailing slash. + * + * `wpath_len` is in WCHARS not bytes. + */ +static enum get_relative_result get_relative_longname( + struct one_watch *watch, + const wchar_t *wpath, DWORD wpath_len, + wchar_t *wpath_longname) +{ + wchar_t buf_in[2 * MAX_PATH + 1]; + wchar_t buf_out[MAX_PATH + 1]; + DWORD root_len; + + /* Build L"/" */ + root_len = watch->wpath_longname_len; + wcsncpy(buf_in, watch->wpath_longname, root_len); + wcsncpy(buf_in + root_len, wpath, wpath_len); + buf_in[root_len + wpath_len] = 0; + + /* + * We don't actually know if the source pathname is a + * shortname or a longname. This routine allows either to be + * given as input. + */ + if (!GetLongPathNameW(buf_in, buf_out, MAX_PATH)) { + /* + * The shortname to longname conversion can fail for + * various reasons, for example if the file has been + * deleted. (That is, if we just received a + * delete-file notification event and the file is + * already gone, we can't ask the file system to + * lookup the longname for it. Likewise, for moves + * and renames where we are given the old name.) + * + * NEEDSWORK: Since deleting or moving a file or + * directory by shortname is rather obscure, I'm going + * ignore the failure and ask the caller to report the + * original relative path. This seemds kinder than + * failing here and forcing a resync. + */ + return GRR_NO_CONVERSION_NEEDED; + } + + if (!wcscmp(buf_in, buf_out)) { + /* + * The path does not have a shortname alias. + */ + return GRR_NO_CONVERSION_NEEDED; + } + + if (wcsncmp(buf_in, buf_out, root_len)) { + /* + * The spelling of the root directory portion of the computed + * longname has changed. This should not happen. Basically, + * it means that we don't know where (without recomputing the + * longname of just the root directory) to split out the + * relative path. Since this should not happen, I'm just + * going to let this fail and force a shutdown (because all + * subsequent events are probably going to see the same + * mismatch). + */ + return GRR_SHUTDOWN; + } + + /* Return the worktree root-relative portion of the longname. */ + + wcscpy(wpath_longname, buf_out + root_len); + return GRR_HAVE_CONVERSION; +} + void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { SetEvent(state->backend_data->hListener[LISTENER_SHUTDOWN]); @@ -111,7 +270,9 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, DWORD share_mode = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; HANDLE hDir; - wchar_t wpath[MAX_PATH]; + DWORD len_longname; + wchar_t wpath[MAX_PATH + 1]; + wchar_t wpath_longname[MAX_PATH + 1]; if (xutftowcs_path(wpath, path) < 0) { error(_("could not convert to wide characters: '%s'"), path); @@ -128,6 +289,20 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, return NULL; } + if (!GetLongPathNameW(wpath, wpath_longname, MAX_PATH)) { + error(_("[GLE %ld] could not get longname of '%s'"), + GetLastError(), path); + CloseHandle(hDir); + return NULL; + } + + len_longname = wcslen(wpath_longname); + if (wpath_longname[len_longname - 1] != L'/' && + wpath_longname[len_longname - 1] != L'\\') { + wpath_longname[len_longname++] = L'/'; + wpath_longname[len_longname] = 0; + } + CALLOC_ARRAY(watch, 1); watch->buf_len = sizeof(watch->buffer); /* assume full MAX_RDCW_BUF */ @@ -135,6 +310,9 @@ static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, strbuf_init(&watch->path, 0); strbuf_addstr(&watch->path, path); + wcscpy(watch->wpath_longname, wpath_longname); + watch->wpath_longname_len = len_longname; + watch->hDir = hDir; watch->hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); @@ -258,6 +436,62 @@ static void cancel_rdcw_watch(struct one_watch *watch) watch->is_active = FALSE; } +/* + * Process a single relative pathname event. + * Return 1 if we should shutdown. + */ +static int process_1_worktree_event( + struct string_list *cookie_list, + struct fsmonitor_batch **batch, + const struct strbuf *path, + enum fsmonitor_path_type t, + DWORD info_action) +{ + const char *slash; + + switch (t) { + case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: + /* special case cookie files within .git */ + + /* Use just the filename of the cookie file. */ + slash = find_last_dir_sep(path->buf); + string_list_append(cookie_list, + slash ? slash + 1 : path->buf); + break; + + case IS_INSIDE_DOT_GIT: + /* ignore everything inside of "/.git/" */ + break; + + case IS_DOT_GIT: + /* "/.git" was deleted (or renamed away) */ + if ((info_action == FILE_ACTION_REMOVED) || + (info_action == FILE_ACTION_RENAMED_OLD_NAME)) { + trace2_data_string("fsmonitor", NULL, + "fsm-listen/dotgit", + "removed"); + return 1; + } + break; + + case IS_WORKDIR_PATH: + /* queue normal pathname */ + if (!*batch) + *batch = fsmonitor_batch__new(); + fsmonitor_batch__add_path(*batch, path->buf); + break; + + case IS_GITDIR: + case IS_INSIDE_GITDIR: + case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: + default: + BUG("unexpected path classification '%d' for '%s'", + t, path->buf); + } + + return 0; +} + /* * Process filesystem events that happen anywhere (recursively) under the * root directory. For a normal working directory, this includes @@ -274,6 +508,7 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) struct string_list cookie_list = STRING_LIST_INIT_DUP; struct fsmonitor_batch *batch = NULL; const char *p = watch->buffer; + wchar_t wpath_longname[MAX_PATH + 1]; /* * If the kernel gets more events than will fit in the kernel @@ -306,54 +541,63 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) */ for (;;) { FILE_NOTIFY_INFORMATION *info = (void *)p; - const char *slash; + wchar_t *wpath = info->FileName; + DWORD wpath_len = info->FileNameLength / sizeof(WCHAR); enum fsmonitor_path_type t; + enum get_relative_result grr; - strbuf_reset(&path); - if (normalize_path_in_utf8(info, &path) == -1) + if (watch->has_shortnames) { + if (!wcscmp(wpath, watch->dotgit_shortname)) { + /* + * This event exactly matches the + * spelling of the shortname of + * ".git", so we can skip some steps. + * + * (This case is odd because the user + * can "rm -rf GIT~1" and we cannot + * use the filesystem to map it back + * to ".git".) + */ + strbuf_reset(&path); + strbuf_addstr(&path, ".git"); + t = IS_DOT_GIT; + goto process_it; + } + + if (watch->has_tilda && !wcschr(wpath, L'~')) { + /* + * Shortnames on this filesystem have tildas + * and the notification path does not have + * one, so we assume that it is a longname. + */ + goto normalize_it; + } + + grr = get_relative_longname(watch, wpath, wpath_len, + wpath_longname); + switch (grr) { + case GRR_NO_CONVERSION_NEEDED: /* use info buffer as is */ + break; + case GRR_HAVE_CONVERSION: + wpath = wpath_longname; + wpath_len = wcslen(wpath); + break; + default: + case GRR_SHUTDOWN: + goto force_shutdown; + } + } + +normalize_it: + if (normalize_path_in_utf8(wpath, wpath_len, &path) == -1) goto skip_this_path; t = fsmonitor_classify_path_workdir_relative(path.buf); - switch (t) { - case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: - /* special case cookie files within .git */ - - /* Use just the filename of the cookie file. */ - slash = find_last_dir_sep(path.buf); - string_list_append(&cookie_list, - slash ? slash + 1 : path.buf); - break; - - case IS_INSIDE_DOT_GIT: - /* ignore everything inside of "/.git/" */ - break; - - case IS_DOT_GIT: - /* "/.git" was deleted (or renamed away) */ - if ((info->Action == FILE_ACTION_REMOVED) || - (info->Action == FILE_ACTION_RENAMED_OLD_NAME)) { - trace2_data_string("fsmonitor", NULL, - "fsm-listen/dotgit", - "removed"); - goto force_shutdown; - } - break; - - case IS_WORKDIR_PATH: - /* queue normal pathname */ - if (!batch) - batch = fsmonitor_batch__new(); - fsmonitor_batch__add_path(batch, path.buf); - break; - - case IS_GITDIR: - case IS_INSIDE_GITDIR: - case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: - default: - BUG("unexpected path classification '%d' for '%s'", - t, path.buf); - } +process_it: + if (process_1_worktree_event(&cookie_list, &batch, &path, t, + info->Action)) + goto force_shutdown; skip_this_path: if (!info->NextEntryOffset) @@ -382,6 +626,9 @@ force_shutdown: * Note that we DO NOT get filesystem events on the external * itself (it is not inside something that we are watching). In particular, * we do not get an event if the external is deleted. + * + * Also, we do not care about shortnames within the external , since + * we never send these paths to clients. */ static int process_gitdir_events(struct fsmonitor_daemon_state *state) { @@ -403,8 +650,10 @@ static int process_gitdir_events(struct fsmonitor_daemon_state *state) const char *slash; enum fsmonitor_path_type t; - strbuf_reset(&path); - if (normalize_path_in_utf8(info, &path) == -1) + if (normalize_path_in_utf8( + info->FileName, + info->FileNameLength / sizeof(WCHAR), + &path) == -1) goto skip_this_path; t = fsmonitor_classify_path_gitdir_relative(path.buf); @@ -538,6 +787,8 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) if (!data->watch_worktree) goto failed; + check_for_shortnames(data->watch_worktree); + if (state->nr_paths_watching > 1) { data->watch_gitdir = create_watch(state, state->path_gitdir_watch.buf); diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index e62ec9aa3c..bb37504bec 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -123,6 +123,71 @@ test_expect_success 'implicit daemon stop (rename .git)' ' test_must_fail git -C test_implicit_2 fsmonitor--daemon status ' +# File systems on Windows may or may not have shortnames. +# This is a volume-specific setting on modern systems. +# "C:/" drives are required to have them enabled. Other +# hard drives default to disabled. +# +# This is a crude test to see if shortnames are enabled +# on the volume containing the test directory. It is +# crude, but it does not require elevation like `fsutil`. +# +test_lazy_prereq SHORTNAMES ' + mkdir .foo && + test -d "FOO~1" +' + +# Here we assume that the shortname of ".git" is "GIT~1". +test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~1)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_1s" && + + git init test_implicit_1s && + + start_daemon test_implicit_1s && + + # renaming the .git directory will implicitly stop the daemon. + # this moves {.git, GIT~1} to {.gitxyz, GITXYZ~1}. + # the rename-from FS Event will contain the shortname. + # + mv test_implicit_1s/GIT~1 test_implicit_1s/.gitxyz && + + sleep 1 && + # put it back so that our status will not crawl out to our + # parent directory. + # this moves {.gitxyz, GITXYZ~1} to {.git, GIT~1}. + mv test_implicit_1s/.gitxyz test_implicit_1s/.git && + + test_must_fail git -C test_implicit_1s fsmonitor--daemon status +' + +# Here we first create a file with LONGNAME of "GIT~1" before +# we create the repo. This will cause the shortname of ".git" +# to be "GIT~2". +test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~2)' ' + test_when_finished "stop_daemon_delete_repo test_implicit_1s2" && + + mkdir test_implicit_1s2 && + echo HELLO >test_implicit_1s2/GIT~1 && + git init test_implicit_1s2 && + + test_path_is_file test_implicit_1s2/GIT~1 && + test_path_is_dir test_implicit_1s2/GIT~2 && + + start_daemon test_implicit_1s2 && + + # renaming the .git directory will implicitly stop the daemon. + # the rename-from FS Event will contain the shortname. + # + mv test_implicit_1s2/GIT~2 test_implicit_1s2/.gitxyz && + + sleep 1 && + # put it back so that our status will not crawl out to our + # parent directory. + mv test_implicit_1s2/.gitxyz test_implicit_1s2/.git && + + test_must_fail git -C test_implicit_1s2 fsmonitor--daemon status +' + test_expect_success 'cannot start multiple daemons' ' test_when_finished "stop_daemon_delete_repo test_multiple" && From 3450d85902d45c232c5507bcf271e5653bd0ce1c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 30 Jun 2021 09:30:20 -0400 Subject: [PATCH 02/22] t7527: test FS event reporing on MacOS WRT case and Unicode Confirm that MacOS FS events are reported with a normalized spelling. APFS (and/or HFS+) is case-insensitive. This means that case-independent lookups ( [ -d .git ] and [ -d .GIT ] ) should both succeed. But that doesn't tell us how FS events are reported if we try "rm -rf .git" versus "rm -rf .GIT". Are the events reported using the on-disk spelling of the pathname or in the spelling used by the command. NEEDSWORK: I was only able to test case. It would be nice to add tests that use different Unicode spellings/normalizations and understand the differences between APFS and HFS+ in this area. We should confirm that the spelling of the workdir paths that the daemon sends to clients are always properly normalized. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index bb37504bec..03c6c68654 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -188,6 +188,36 @@ test_expect_success MINGW,SHORTNAMES 'implicit daemon stop (rename GIT~2)' ' test_must_fail git -C test_implicit_1s2 fsmonitor--daemon status ' +# Confirm that MacOS hides all of the Unicode normalization and/or +# case folding from the FS events. That is, are the pathnames in the +# FS events reported using the spelling on the disk or in the spelling +# used by the other process. +# +# Note that we assume that the filesystem is set to case insensitive. +# +# NEEDSWORK: APFS handles Unicode and Unicode normalization +# differently than HFS+. I only have an APFS partition, so +# more testing here would be helpful. +# + +# Rename .git using alternate spelling and confirm that the daemon +# sees the event using the correct spelling and shutdown. +test_expect_success UTF8_NFD_TO_NFC 'MacOS event spelling (rename .GIT)' ' + test_when_finished "stop_daemon_delete_repo test_apfs" && + + git init test_apfs && + start_daemon test_apfs && + + test_path_is_dir test_apfs/.git && + test_path_is_dir test_apfs/.GIT && + + mv test_apfs/.GIT test_apfs/.FOO && + sleep 1 && + mv test_apfs/.FOO test_apfs/.git && + + test_must_fail git -C test_apfs fsmonitor--daemon status +' + test_expect_success 'cannot start multiple daemons' ' test_when_finished "stop_daemon_delete_repo test_multiple" && From 8136a91215f9e3e42901f219001357de5538a697 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 12 Jul 2021 12:50:31 -0400 Subject: [PATCH 03/22] t7527: test builtin FSMonitor watching repos with unicode paths Create some test repos with UTF8 pathnames and verify that the builtin FSMonitor can watch them. This test is mainly for Windows where we need to avoid `*A()` routines. Signed-off-by: Jeff Hostetler --- t/t7527-builtin-fsmonitor.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 03c6c68654..08584d52d2 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -699,4 +699,27 @@ do done done +# Test Unicode UTF-8 characters in the pathname of the working +# directory. Use of "*A()" routines rather than "*W()" routines +# on Windows can sometimes lead to odd failures. +# +u1=$(printf "u_c3_a6__\xC3\xA6") +u2=$(printf "u_e2_99_ab__\xE2\x99\xAB") +u_values="$u1 $u2" +for u in $u_values +do + test_expect_success "Unicode path: $u" ' + test_when_finished "stop_daemon_delete_repo $u" && + + git init "$u" && + echo 1 >"$u"/file1 && + git -C "$u" add file1 && + git -C "$u" config core.useBuiltinFSMonitor true && + + start_daemon "$u" && + git -C "$u" status >actual && + grep "new file: file1" actual + ' +done + test_done From 0172fc9c435768b653e3aa87888417caf79ac5da Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 27 Jul 2021 14:06:33 -0400 Subject: [PATCH 04/22] t/helper/fsmonitor-client: create stress test Create a stress test to hammer on the fsmonitor daemon. Create a client-side thread pool of n threads and have each of them make m requests as fast as they can. NEEDSWORK: This is just the client-side thread pool and is useful for interactive testing and experimentation. We need to add a script test to drive this. Signed-off-by: Jeff Hostetler --- t/helper/test-fsmonitor-client.c | 105 +++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c index f7a5b3a32f..9dd2f9af55 100644 --- a/t/helper/test-fsmonitor-client.c +++ b/t/helper/test-fsmonitor-client.c @@ -7,6 +7,8 @@ #include "cache.h" #include "parse-options.h" #include "fsmonitor-ipc.h" +#include "thread-utils.h" +#include "trace2.h" #ifndef HAVE_FSMONITOR_DAEMON_BACKEND int cmd__fsmonitor_client(int argc, const char **argv) @@ -79,20 +81,120 @@ static int do_send_flush(void) return 0; } +struct hammer_thread_data +{ + pthread_t pthread_id; + int thread_nr; + + int nr_requests; + const char *token; + + int sum_successful; + int sum_errors; +}; + +static void *hammer_thread_proc(void *_hammer_thread_data) +{ + struct hammer_thread_data *data = _hammer_thread_data; + struct strbuf answer = STRBUF_INIT; + int k; + int ret; + + trace2_thread_start("hammer"); + + for (k = 0; k < data->nr_requests; k++) { + strbuf_reset(&answer); + + ret = fsmonitor_ipc__send_query(data->token, &answer); + if (ret < 0) + data->sum_errors++; + else + data->sum_successful++; + } + + strbuf_release(&answer); + trace2_thread_exit(); + return NULL; +} + +/* + * Start a pool of client threads that will each send a series of + * commands to the daemon. + * + * The goal is to overload the daemon with a sustained series of + * concurrent requests. + */ +static int do_hammer(const char *token, int nr_threads, int nr_requests) +{ + struct hammer_thread_data *data = NULL; + int k; + int sum_join_errors = 0; + int sum_commands = 0; + int sum_errors = 0; + + if (!token || !*token) + token = get_token_from_index(); + if (nr_threads < 1) + nr_threads = 1; + if (nr_requests < 1) + nr_requests = 1; + + CALLOC_ARRAY(data, nr_threads); + + for (k = 0; k < nr_threads; k++) { + struct hammer_thread_data *p = &data[k]; + p->thread_nr = k; + p->nr_requests = nr_requests; + p->token = token; + + if (pthread_create(&p->pthread_id, NULL, hammer_thread_proc, p)) { + warning("failed to create thread[%d] skipping remainder", k); + nr_threads = k; + break; + } + } + + for (k = 0; k < nr_threads; k++) { + struct hammer_thread_data *p = &data[k]; + + if (pthread_join(p->pthread_id, NULL)) + sum_join_errors++; + sum_commands += p->sum_successful; + sum_errors += p->sum_errors; + } + + fprintf(stderr, "HAMMER: [threads %d][requests %d] [ok %d][err %d][join %d]\n", + nr_threads, nr_requests, sum_commands, sum_errors, sum_join_errors); + + free(data); + + /* + * TODO Decide if/when to return an error or call die(). + */ + return 0; +} + int cmd__fsmonitor_client(int argc, const char **argv) { const char *subcmd; const char *token = NULL; + int nr_threads = 1; + int nr_requests = 1; const char * const fsmonitor_client_usage[] = { N_("test-helper fsmonitor-client query []"), N_("test-helper fsmonitor-client flush"), + N_("test-helper fsmonitor-client hammer [] [] []"), NULL, }; struct option options[] = { OPT_STRING(0, "token", &token, N_("token"), N_("command token to send to the server")), + + OPT_INTEGER(0, "threads", &nr_threads, N_("number of client threads")), + OPT_INTEGER(0, "requests", &nr_requests, N_("number of requests per thread")), + OPT_END() }; @@ -116,6 +218,9 @@ int cmd__fsmonitor_client(int argc, const char **argv) if (!strcmp(subcmd, "flush")) return !!do_send_flush(); + if (!strcmp(subcmd, "hammer")) + return !!do_hammer(token, nr_threads, nr_requests); + die("Unhandled subcommand: '%s'", subcmd); } #endif From 607b742e1e6c51c898c6d7deb3cd76cd45581528 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 23 Jun 2021 10:18:17 -0400 Subject: [PATCH 05/22] fsmonitor-settings: bare repos are incompatible with FSMonitor Bare repos do not have a worktree, so there is nothing for the daemon watch. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 11 +++++ builtin/update-index.c | 8 ++++ fsmonitor-settings.c | 83 +++++++++++++++++++++++++++++++++---- fsmonitor-settings.h | 11 +++++ t/t7519-status-fsmonitor.sh | 26 ++++++++++++ 5 files changed, 130 insertions(+), 9 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 6011fe42ee..899355c55a 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1424,6 +1424,17 @@ int cmd_fsmonitor__daemon(int argc, const char **argv, const char *prefix) die(_("invalid 'ipc-threads' value (%d)"), fsmonitor__ipc_threads); + prepare_repo_settings(the_repository); + fsm_settings__set_ipc(the_repository); + + if (fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_INCOMPATIBLE) { + struct strbuf buf_reason = STRBUF_INIT; + fsm_settings__get_reason(the_repository, &buf_reason); + error("%s '%s'", buf_reason.buf, xgetcwd()); + strbuf_release(&buf_reason); + return -1; + } + if (!strcmp(subcmd, "start")) return !!try_to_start_background_daemon(); diff --git a/builtin/update-index.c b/builtin/update-index.c index 79db3ff37e..3f1ed95549 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -1216,6 +1216,14 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) if (fsmonitor > 0) { enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); + if (fsm_mode == FSMONITOR_MODE_INCOMPATIBLE) { + struct strbuf buf_reason = STRBUF_INIT; + fsm_settings__get_reason(r, &buf_reason); + error("%s", buf_reason.buf); + strbuf_release(&buf_reason); + return -1; + } + if (fsm_mode == FSMONITOR_MODE_DISABLED) { warning(_("core.useBuiltinFSMonitor is unset; " "set it if you really want to enable the " diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 2770266f5e..5a803a41d5 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -9,19 +9,57 @@ */ struct fsmonitor_settings { enum fsmonitor_mode mode; + enum fsmonitor_reason reason; char *hook_path; }; -void fsm_settings__set_ipc(struct repository *r) +static void set_incompatible(struct repository *r, + enum fsmonitor_reason reason) { struct fsmonitor_settings *s = r->settings.fsmonitor; + s->mode = FSMONITOR_MODE_INCOMPATIBLE; + s->reason = reason; +} + +static int check_for_incompatible(struct repository *r) +{ + if (!r->worktree) { + /* + * Bare repositories don't have a working directory and + * therefore have nothing to watch. + */ + set_incompatible(r, FSMONITOR_REASON_BARE); + return 1; + } + + return 0; +} + +static struct fsmonitor_settings *s_init(struct repository *r) +{ + if (!r->settings.fsmonitor) + CALLOC_ARRAY(r->settings.fsmonitor, 1); + + return r->settings.fsmonitor; +} + +void fsm_settings__set_ipc(struct repository *r) +{ + struct fsmonitor_settings *s = s_init(r); + + if (check_for_incompatible(r)) + return; + s->mode = FSMONITOR_MODE_IPC; } void fsm_settings__set_hook(struct repository *r, const char *path) { - struct fsmonitor_settings *s = r->settings.fsmonitor; + struct fsmonitor_settings *s = s_init(r); + + if (check_for_incompatible(r)) + return; s->mode = FSMONITOR_MODE_HOOK; s->hook_path = strdup(path); @@ -29,9 +67,10 @@ void fsm_settings__set_hook(struct repository *r, const char *path) void fsm_settings__set_disabled(struct repository *r) { - struct fsmonitor_settings *s = r->settings.fsmonitor; + struct fsmonitor_settings *s = s_init(r); s->mode = FSMONITOR_MODE_DISABLED; + s->reason = FSMONITOR_REASON_ZERO; FREE_AND_NULL(s->hook_path); } @@ -65,12 +104,6 @@ static int check_for_hook(struct repository *r) static void lookup_fsmonitor_settings(struct repository *r) { - struct fsmonitor_settings *s; - - CALLOC_ARRAY(s, 1); - - r->settings.fsmonitor = s; - if (check_for_ipc(r)) return; @@ -95,3 +128,35 @@ const char *fsm_settings__get_hook_path(struct repository *r) return r->settings.fsmonitor->hook_path; } + +static void create_reason_message(struct repository *r, + struct strbuf *buf_reason) +{ + struct fsmonitor_settings *s = r->settings.fsmonitor; + + switch (s->reason) { + case FSMONITOR_REASON_ZERO: + return; + + case FSMONITOR_REASON_BARE: + strbuf_addstr(buf_reason, + _("bare repos are incompatible with fsmonitor")); + return; + + default: + BUG("Unhandled case in create_reason_message '%d'", s->reason); + } +} +enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, + struct strbuf *buf_reason) +{ + strbuf_reset(buf_reason); + + if (!r->settings.fsmonitor) + lookup_fsmonitor_settings(r); + + if (r->settings.fsmonitor->mode == FSMONITOR_MODE_INCOMPATIBLE) + create_reason_message(r, buf_reason); + + return r->settings.fsmonitor->reason; +} diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index 50b2923461..eb45524b1f 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -4,17 +4,28 @@ struct repository; enum fsmonitor_mode { + FSMONITOR_MODE_INCOMPATIBLE = -1, /* see _reason */ FSMONITOR_MODE_DISABLED = 0, FSMONITOR_MODE_HOOK = 1, /* core.fsmonitor */ FSMONITOR_MODE_IPC = 2, /* core.useBuiltinFSMonitor */ }; +/* + * Incompatibility reasons. + */ +enum fsmonitor_reason { + FSMONITOR_REASON_ZERO = 0, + FSMONITOR_REASON_BARE = 1, +}; + void fsm_settings__set_ipc(struct repository *r); void fsm_settings__set_hook(struct repository *r, const char *path); void fsm_settings__set_disabled(struct repository *r); enum fsmonitor_mode fsm_settings__get_mode(struct repository *r); const char *fsm_settings__get_hook_path(struct repository *r); +enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, + struct strbuf *buf_reason); struct fsmonitor_settings; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index a6308acf00..51b2218601 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -55,6 +55,32 @@ test_lazy_prereq UNTRACKED_CACHE ' test $ret -ne 1 ' +# Test that we detect and disallow repos that are incompatible with FSMonitor. +test_expect_success 'incompatible bare repo' ' + test_when_finished "rm -rf ./bare-clone actual expect" && + git init --bare bare-clone && + cat >expect <<-\EOF && + error: bare repos are incompatible with fsmonitor + EOF + + test_must_fail \ + git -C ./bare-clone -c core.fsmonitor=foo \ + update-index --fsmonitor 2>actual && + test_cmp expect actual && + + test_must_fail \ + git -C ./bare-clone -c core.usebuiltinfsmonitor=true \ + update-index --fsmonitor 2>actual && + test_cmp expect actual +' + +test_expect_success FSMONITOR_DAEMON 'run fsmonitor-daemon in bare repo' ' + test_when_finished "rm -rf ./bare-clone actual" && + git init --bare bare-clone && + test_must_fail git -C ./bare-clone fsmonitor--daemon run 2>actual && + grep "bare repos are incompatible with fsmonitor" actual +' + test_expect_success 'setup' ' mkdir -p .git/hooks && : >tracked && From 6be77ebade5272285f5af63f782ad94fbfdf246b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 29 Jul 2021 13:33:21 -0400 Subject: [PATCH 06/22] fsmonitor-settings: stub in platform-specific incompatibility checking Extend generic incompatibility checkout with platform-specific mechanism. Stub in Win32 version. In the existing fsmonitor-settings code we have a way to mark types of repos as incompatible with fsmonitor (whether via the hook and ipc APIs). For example, we do this for bare repos, since there are no files to watch. Extend this exclusion mechanism for platfor-specific reasons. This commit just creates the framework and adds a stub for Win32. Signed-off-by: Jeff Hostetler --- Makefile | 13 +++++++++++++ compat/fsmonitor/fsm-settings-win32.c | 9 +++++++++ config.mak.uname | 4 ++++ contrib/buildsystems/CMakeLists.txt | 3 +++ fsmonitor-settings.c | 12 ++++++++++++ fsmonitor-settings.h | 13 +++++++++++++ 6 files changed, 54 insertions(+) create mode 100644 compat/fsmonitor/fsm-settings-win32.c diff --git a/Makefile b/Makefile index 00a85a23a4..9324e707bb 100644 --- a/Makefile +++ b/Makefile @@ -469,6 +469,11 @@ all:: # `compat/fsmonitor/fsm-listen-.c` that implements the # `fsm_listen__*()` routines. # +# If your platform has os-specific ways to tell if a repo is incompatible with +# fsmonitor (whether the hook or ipc daemon version), set FSMONITOR_OS_SETTINGS +# to the "" of the corresponding `compat/fsmonitor/fsm-settings-.c` +# that implements the `fsm_os_settings__*()` routines. +# # Define DEVELOPER to enable more compiler warnings. Compiler version # and family are auto detected, but could be overridden by defining # COMPILER_FEATURES (see config.mak.dev). You can still set @@ -1949,6 +1954,11 @@ ifdef FSMONITOR_DAEMON_BACKEND COMPAT_OBJS += compat/fsmonitor/fsm-listen-$(FSMONITOR_DAEMON_BACKEND).o endif +ifdef FSMONITOR_OS_SETTINGS + COMPAT_CFLAGS += -DHAVE_FSMONITOR_OS_SETTINGS + COMPAT_OBJS += compat/fsmonitor/fsm-settings-$(FSMONITOR_OS_SETTINGS).o +endif + ifeq ($(TCLTK_PATH),) NO_TCLTK = NoThanks endif @@ -2871,6 +2881,9 @@ GIT-BUILD-OPTIONS: FORCE ifdef FSMONITOR_DAEMON_BACKEND @echo FSMONITOR_DAEMON_BACKEND=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_DAEMON_BACKEND)))'\' >>$@+ endif +ifdef FSMONITOR_OS_SETTINGS + @echo FSMONITOR_OS_SETTINGS=\''$(subst ','\'',$(subst ','\'',$(FSMONITOR_OS_SETTINGS)))'\' >>$@+ +endif ifdef TEST_OUTPUT_DIRECTORY @echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+ endif diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c new file mode 100644 index 0000000000..176a6f5726 --- /dev/null +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -0,0 +1,9 @@ +#include "cache.h" +#include "config.h" +#include "repository.h" +#include "fsmonitor-settings.h" + +enum fsmonitor_reason fsm_os__incompatible(struct repository *r) +{ + return FSMONITOR_REASON_ZERO; +} diff --git a/config.mak.uname b/config.mak.uname index d65a5eb807..9b9a8e7a1c 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -449,6 +449,8 @@ ifeq ($(uname_S),Windows) # These are always available, so we do not have to conditionally # support it. FSMONITOR_DAEMON_BACKEND = win32 + FSMONITOR_OS_SETTINGS = win32 + NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo @@ -639,6 +641,8 @@ ifeq ($(uname_S),MINGW) # These are always available, so we do not have to conditionally # support it. FSMONITOR_DAEMON_BACKEND = win32 + FSMONITOR_OS_SETTINGS = win32 + RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 7da31c38f4..8044db7e83 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -289,6 +289,9 @@ if(SUPPORTS_SIMPLE_IPC) if(CMAKE_SYSTEM_NAME STREQUAL "Windows") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) + + add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-win32.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 5a803a41d5..408c3f159d 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -33,6 +33,18 @@ static int check_for_incompatible(struct repository *r) return 1; } +#ifdef HAVE_FSMONITOR_OS_SETTINGS + { + enum fsmonitor_reason reason; + + reason = fsm_os__incompatible(r); + if (reason != FSMONITOR_REASON_ZERO) { + set_incompatible(r, reason); + return 1; + } + } +#endif + return 0; } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index eb45524b1f..813a540619 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -29,4 +29,17 @@ enum fsmonitor_reason fsm_settings__get_reason(struct repository *r, struct fsmonitor_settings; +#ifdef HAVE_FSMONITOR_OS_SETTINGS +/* + * Ask platform-specific code whether the repository is incompatible + * with fsmonitor (both hook and ipc modes). For example, if the working + * directory is on a remote volume and mounted via a technology that does + * not support notification events. + * + * fsm_os__* routines should considered private to fsm_settings__ + * routines. + */ +enum fsmonitor_reason fsm_os__incompatible(struct repository *r); +#endif /* HAVE_FSMONITOR_OS_SETTINGS */ + #endif /* FSMONITOR_SETTINGS_H */ From 376db8527d5f694ee8c153b2f4bfd6366a0d8e23 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 2 Aug 2021 15:40:36 -0400 Subject: [PATCH 07/22] fsmonitor-settings: virtual repos are incompatible with FSMonitor Virtual repos, such as GVFS (aka VFS for Git), are incompatible with FSMonitor. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-win32.c | 28 ++++++++++++++++++++++++++- fsmonitor-settings.c | 5 +++++ fsmonitor-settings.h | 1 + t/t7519-status-fsmonitor.sh | 9 +++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c index 176a6f5726..7caa79570a 100644 --- a/compat/fsmonitor/fsm-settings-win32.c +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -3,7 +3,33 @@ #include "repository.h" #include "fsmonitor-settings.h" -enum fsmonitor_reason fsm_os__incompatible(struct repository *r) +/* + * GVFS (aka VFS for Git) is incompatible with FSMonitor. + * + * Granted, core Git does not know anything about GVFS and we + * shouldn't make assumptions about a downstream feature, but users + * can install both versions. And this can lead to incorrect results + * from core Git commands. So, without bringing in any of the GVFS + * code, do a simple config test for a published config setting. (We + * do not look at the various *_TEST_* environment variables.) + */ +static enum fsmonitor_reason is_virtual(struct repository *r) { + const char *const_str; + + if (!repo_config_get_value(r, "core.virtualfilesystem", &const_str)) + return FSMONITOR_REASON_VIRTUAL; + + return FSMONITOR_REASON_ZERO; +} + +enum fsmonitor_reason fsm_os__incompatible(struct repository *r) +{ + enum fsmonitor_reason reason; + + reason = is_virtual(r); + if (reason) + return reason; + return FSMONITOR_REASON_ZERO; } diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 408c3f159d..41cfeea9e8 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -155,6 +155,11 @@ static void create_reason_message(struct repository *r, _("bare repos are incompatible with fsmonitor")); return; + case FSMONITOR_REASON_VIRTUAL: + strbuf_addstr(buf_reason, + _("virtual repos are incompatible with fsmonitor")); + return; + default: BUG("Unhandled case in create_reason_message '%d'", s->reason); } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index 813a540619..7c8da4e80a 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -16,6 +16,7 @@ enum fsmonitor_mode { enum fsmonitor_reason { FSMONITOR_REASON_ZERO = 0, FSMONITOR_REASON_BARE = 1, + FSMONITOR_REASON_VIRTUAL = 2, }; void fsm_settings__set_ipc(struct repository *r); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 51b2218601..0136dc5265 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -81,6 +81,15 @@ test_expect_success FSMONITOR_DAEMON 'run fsmonitor-daemon in bare repo' ' grep "bare repos are incompatible with fsmonitor" actual ' +test_expect_success MINGW,FSMONITOR_DAEMON 'run fsmonitor-daemon in virtual repo' ' + test_when_finished "rm -rf ./fake-virtual-clone actual" && + git init fake-virtual-clone && + test_must_fail git -C ./fake-virtual-clone \ + -c core.virtualfilesystem=true \ + fsmonitor--daemon run 2>actual && + grep "virtual repos are incompatible with fsmonitor" actual +' + test_expect_success 'setup' ' mkdir -p .git/hooks && : >tracked && From 19e6ec7d6ad9860cc19865889c7b7219ae8bc9f9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 4 Aug 2021 10:20:55 -0400 Subject: [PATCH 08/22] fsmonitor-settings: stub in platform-specific incompatibility checking on MacOS Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-darwin.c | 9 +++++++++ config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 compat/fsmonitor/fsm-settings-darwin.c diff --git a/compat/fsmonitor/fsm-settings-darwin.c b/compat/fsmonitor/fsm-settings-darwin.c new file mode 100644 index 0000000000..176a6f5726 --- /dev/null +++ b/compat/fsmonitor/fsm-settings-darwin.c @@ -0,0 +1,9 @@ +#include "cache.h" +#include "config.h" +#include "repository.h" +#include "fsmonitor-settings.h" + +enum fsmonitor_reason fsm_os__incompatible(struct repository *r) +{ + return FSMONITOR_REASON_ZERO; +} diff --git a/config.mak.uname b/config.mak.uname index 9b9a8e7a1c..2f2de8b242 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -163,6 +163,7 @@ ifeq ($(uname_S),Darwin) ifndef NO_PTHREADS ifndef NO_UNIX_SOCKETS FSMONITOR_DAEMON_BACKEND = darwin + FSMONITOR_OS_SETTINGS = darwin endif endif diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 8044db7e83..fb67a8a531 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -295,6 +295,9 @@ if(SUPPORTS_SIMPLE_IPC) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) + + add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-darwin.c) endif() endif() From 7d41dfc8324afd8a90324d605ff242da5da115c6 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 4 Aug 2021 16:22:10 -0400 Subject: [PATCH 09/22] fsmonitor-settings: remote repos on MacOS are incompatible with FSMonitor Teach Git to detect remote working directories on MacOS and mark them as incompatible with FSMonitor. With this, `git fsmonitor--daemon run` will error out with a message like it does for bare repos. Client commands, like `git status`, will not attempt to start the daemon. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-darwin.c | 66 ++++++++++++++++++++++++++ fsmonitor-settings.c | 5 ++ fsmonitor-settings.h | 1 + 3 files changed, 72 insertions(+) diff --git a/compat/fsmonitor/fsm-settings-darwin.c b/compat/fsmonitor/fsm-settings-darwin.c index 176a6f5726..c3b719d6fb 100644 --- a/compat/fsmonitor/fsm-settings-darwin.c +++ b/compat/fsmonitor/fsm-settings-darwin.c @@ -2,8 +2,74 @@ #include "config.h" #include "repository.h" #include "fsmonitor-settings.h" +#include "fsmonitor.h" +#include +#include + +/* + * Remote working directories are problematic for FSMonitor. + * + * The underlying file system on the server machine and/or the remote + * mount type (NFS, SAMBA, etc.) dictates whether notification events + * are available at all to remote client machines. + * + * Kernel differences between the server and client machines also + * dictate the how (buffering, frequency, de-dup) the events are + * delivered to client machine processes. + * + * A client machine (such as a laptop) may choose to suspend/resume + * and it is unclear (without lots of testing) whether the watcher can + * resync after a resume. We might be able to treat this as a normal + * "events were dropped by the kernel" event and do our normal "flush + * and resync" --or-- we might need to close the existing (zombie?) + * notification fd and create a new one. + * + * In theory, the above issues need to be addressed whether we are + * using the Hook or IPC API. + * + * For the builtin FSMonitor, we create the Unix domain socket for the + * IPC in the .git directory. If the working directory is remote, + * then the socket will be created on the remote file system. This + * can fail if the remote file system does not support UDS file types + * (e.g. smbfs to a Windows server) or if the remote kernel does not + * allow a non-local process to bind() the socket. (These problems + * could be fixed by moving the UDS out of the .git directory and to a + * well-known local directory on the client machine, but care should + * be taken to ensure that $HOME is actually local and not a managed + * file share.) + * + * So (for now at least), mark remote working directories as + * incompatible. + */ +static enum fsmonitor_reason is_remote(struct repository *r) +{ + struct statfs fs; + + if (statfs(r->worktree, &fs) == -1) { + int saved_errno = errno; + trace_printf_key(&trace_fsmonitor, "statfs('%s') failed: %s", + r->worktree, strerror(saved_errno)); + errno = saved_errno; + return FSMONITOR_REASON_ZERO; + } + + trace_printf_key(&trace_fsmonitor, + "statfs('%s') [type 0x%08x][flags 0x%08x] '%s'", + r->worktree, fs.f_type, fs.f_flags, fs.f_fstypename); + + if (!(fs.f_flags & MNT_LOCAL)) + return FSMONITOR_REASON_REMOTE; + + return FSMONITOR_REASON_ZERO; +} enum fsmonitor_reason fsm_os__incompatible(struct repository *r) { + enum fsmonitor_reason reason; + + reason = is_remote(r); + if (reason) + return reason; + return FSMONITOR_REASON_ZERO; } diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index 41cfeea9e8..1aa77b3193 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -160,6 +160,11 @@ static void create_reason_message(struct repository *r, _("virtual repos are incompatible with fsmonitor")); return; + case FSMONITOR_REASON_REMOTE: + strbuf_addstr(buf_reason, + _("remote repos are incompatible with fsmonitor")); + return; + default: BUG("Unhandled case in create_reason_message '%d'", s->reason); } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index 7c8da4e80a..da20378172 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -17,6 +17,7 @@ enum fsmonitor_reason { FSMONITOR_REASON_ZERO = 0, FSMONITOR_REASON_BARE = 1, FSMONITOR_REASON_VIRTUAL = 2, + FSMONITOR_REASON_REMOTE = 3, }; void fsm_settings__set_ipc(struct repository *r); From 20d84f8b80b0471e94c789ebe12ec91143f1d629 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 5 Aug 2021 09:07:48 -0400 Subject: [PATCH 10/22] fsmonitor-settings: remote repos on Windows are incompatible with FSMonitor Teach Git to detect remote working directories on Windows and mark them as incompatible with FSMonitor. With this `git fsmonitor--daemon run` will error out with a message like it does for bare repos. Client commands, such as `git status`, will not attempt to start the daemon. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-settings-win32.c | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/compat/fsmonitor/fsm-settings-win32.c b/compat/fsmonitor/fsm-settings-win32.c index 7caa79570a..77156c0ca2 100644 --- a/compat/fsmonitor/fsm-settings-win32.c +++ b/compat/fsmonitor/fsm-settings-win32.c @@ -2,6 +2,7 @@ #include "config.h" #include "repository.h" #include "fsmonitor-settings.h" +#include "fsmonitor.h" /* * GVFS (aka VFS for Git) is incompatible with FSMonitor. @@ -23,6 +24,103 @@ static enum fsmonitor_reason is_virtual(struct repository *r) return FSMONITOR_REASON_ZERO; } +/* + * Remote working directories are problematic for FSMonitor. + * + * The underlying file system on the server machine and/or the remote + * mount type dictates whether notification events are available at + * all to remote client machines. + * + * Kernel differences between the server and client machines also + * dictate the how (buffering, frequency, de-dup) the events are + * delivered to client machine processes. + * + * A client machine (such as a laptop) may choose to suspend/resume + * and it is unclear (without lots of testing) whether the watcher can + * resync after a resume. We might be able to treat this as a normal + * "events were dropped by the kernel" event and do our normal "flush + * and resync" --or-- we might need to close the existing (zombie?) + * notification fd and create a new one. + * + * In theory, the above issues need to be addressed whether we are + * using the Hook or IPC API. + * + * So (for now at least), mark remote working directories as + * incompatible. + * + * Notes for testing: + * + * (a) Windows allows a network share to be mapped to a drive letter. + * (This is the normal method to access it.) + * + * $ NET USE Z: \\server\share + * $ git -C Z:/repo status + * + * (b) Windows allows a network share to be referenced WITHOUT mapping + * it to drive letter. + * + * $ NET USE \\server\share\dir + * $ git -C //server/share/repo status + * + * (c) Windows allows "SUBST" to create a fake drive mapping to an + * arbitrary path (which may be remote) + * + * $ SUBST Q: Z:\repo + * $ git -C Q:/ status + * + * (d) Windows allows a directory symlink to be created on a local + * file system that points to a remote repo. + * + * $ mklink /d ./link //server/share/repo + * $ git -C ./link status + */ +static enum fsmonitor_reason is_remote(struct repository *r) +{ + wchar_t wpath[MAX_PATH]; + wchar_t wfullpath[MAX_PATH]; + size_t wlen; + UINT driveType; + + /* + * Do everything in wide chars because the drive letter might be + * a multi-byte sequence. See win32_has_dos_drive_prefix(). + */ + if (xutftowcs_path(wpath, r->worktree) < 0) + return FSMONITOR_REASON_ZERO; + + /* + * GetDriveTypeW() requires a final slash. We assume that the + * worktree pathname points to an actual directory. + */ + wlen = wcslen(wpath); + if (wpath[wlen - 1] != L'\\' && wpath[wlen - 1] != L'/') { + wpath[wlen++] = L'\\'; + wpath[wlen] = 0; + } + + /* + * Normalize the path. If nothing else, this converts forward + * slashes to backslashes. This is essential to get GetDriveTypeW() + * correctly handle some UNC "\\server\share\..." paths. + */ + if (!GetFullPathNameW(wpath, MAX_PATH, wfullpath, NULL)) + return FSMONITOR_REASON_ZERO; + + driveType = GetDriveTypeW(wfullpath); + trace_printf_key(&trace_fsmonitor, + "DriveType '%s' L'%ls' (%u)", + r->worktree, wfullpath, driveType); + + if (driveType == DRIVE_REMOTE) { + trace_printf_key(&trace_fsmonitor, + "is_remote('%s') true", + r->worktree); + return FSMONITOR_REASON_REMOTE; + } + + return FSMONITOR_REASON_ZERO; +} + enum fsmonitor_reason fsm_os__incompatible(struct repository *r) { enum fsmonitor_reason reason; @@ -31,5 +129,9 @@ enum fsmonitor_reason fsm_os__incompatible(struct repository *r) if (reason) return reason; + reason = is_remote(r); + if (reason) + return reason; + return FSMONITOR_REASON_ZERO; } From de8d3c9f3b2b3b16cfd81dda4946d6b51e0218b5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 23 Aug 2021 17:42:28 -0400 Subject: [PATCH 11/22] unpack-trees: initialize fsmonitor_has_run_once in o->result Initialize `o->result.fsmonitor_has_run_once` based upon value in `o->src_index->fsmonitor_has_run_once` to prevent a second fsmonitor query during the tree traversal and possibly getting a skewed view of the working directory. The checkout code has already talked to the fsmonitor and the traversal is updating the index as it traverses, so there is no need to query the fsmonitor. Signed-off-by: Jeff Hostetler --- unpack-trees.c | 1 + 1 file changed, 1 insertion(+) diff --git a/unpack-trees.c b/unpack-trees.c index 360844bda3..888cff81f9 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1772,6 +1772,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options o->result.fsmonitor_last_update = xstrdup_or_null(o->src_index->fsmonitor_last_update); + o->result.fsmonitor_has_run_once = o->src_index->fsmonitor_has_run_once; /* * Sparse checkout loop #1: set NEW_SKIP_WORKTREE on existing entries From aaf5e23225e6c02bf1c07b02bf96a8ee3d2659a5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 25 Aug 2021 15:11:11 -0400 Subject: [PATCH 12/22] compat/fsmonitor/fsm-listen-darwin: ignore FSEvents caused by xattr changes on MacOS Ignore FSEvents resulting from `xattr` changes. Git does not care about xattr's or changes to xattr's, so don't waste time collecting these events in the daemon nor transmitting them to clients. Various security tools add xattrs to files and/or directories, such as to mark them as having been downloaded. We should ignore these events since it doesn't affect the content of the file/directory or the normal meta-data that Git cares about. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 2aefdc14d8..79d08517d7 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -172,7 +172,7 @@ static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) if (flag & kFSEventStreamEventFlagItemCloned) strbuf_addstr(&msg, "ItemCloned|"); - trace_printf_key(&trace_fsmonitor, "fsevent: '%s', flags=%u %s", + trace_printf_key(&trace_fsmonitor, "fsevent: '%s', flags=0x%x %s", path, flag, msg.buf); strbuf_release(&msg); @@ -197,6 +197,31 @@ static int ef_is_dropped(const FSEventStreamEventFlags ef) ef & kFSEventStreamEventFlagUserDropped); } +/* + * If an `xattr` change is the only reason we received this event, + * then silently ignore it. Git doesn't care about xattr's. We + * have to be careful here because the kernel can combine multiple + * events for a single path. And because events always have certain + * bits set, such as `ItemIsFile` or `ItemIsDir`. + * + * Return 1 if we should ignore it. + */ +static int ef_ignore_xattr(const FSEventStreamEventFlags ef) +{ + static const FSEventStreamEventFlags mask = + kFSEventStreamEventFlagItemChangeOwner | + kFSEventStreamEventFlagItemCreated | + kFSEventStreamEventFlagItemFinderInfoMod | + kFSEventStreamEventFlagItemInodeMetaMod | + kFSEventStreamEventFlagItemModified | + kFSEventStreamEventFlagItemRemoved | + kFSEventStreamEventFlagItemRenamed | + kFSEventStreamEventFlagItemXattrMod | + kFSEventStreamEventFlagItemCloned; + + return ((ef & mask) == kFSEventStreamEventFlagItemXattrMod); +} + static void fsevent_callback(ConstFSEventStreamRef streamRef, void *ctx, size_t num_of_events, @@ -262,6 +287,13 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef, continue; } + if (ef_ignore_xattr(event_flags[k])) { + trace_printf_key(&trace_fsmonitor, + "ignore-xattr: '%s', flags=0x%x", + path_k, event_flags[k]); + continue; + } + switch (fsmonitor_classify_path_absolute(state, path_k)) { case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: From 93281883197dda65488759c96f14a8accd188b0b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 30 Aug 2021 13:03:57 -0400 Subject: [PATCH 13/22] fsmonitor--daemon: print start message only if fsmonitor.announceStartup Teach fsmonitor--daemon to print a startup message only when `fsmonitor.announceStartup` is true. This setting is false by default so that the output of client commands, like `git status`, is not changed if the daemon is implicitly started. The message is conditionally printed by "run" and "start" subcommands and is sent to stderr. It contains the path to the work tree root. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 899355c55a..dd0561cfc5 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -27,6 +27,9 @@ static int fsmonitor__ipc_threads = 8; #define FSMONITOR__START_TIMEOUT "fsmonitor.starttimeout" static int fsmonitor__start_timeout_sec = 60; +#define FSMONITOR__ANNOUNCE_STARTUP "fsmonitor.announcestartup" +static int fsmonitor__announce_startup = 0; + static int fsmonitor_config(const char *var, const char *value, void *cb) { if (!strcmp(var, FSMONITOR__IPC_THREADS)) { @@ -47,6 +50,16 @@ static int fsmonitor_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, FSMONITOR__ANNOUNCE_STARTUP)) { + int is_bool; + int i = git_config_bool_or_int(var, value, &is_bool); + if (i < 0) + return error(_("value of '%s' not bool or int: %d"), + var, i); + fsmonitor__announce_startup = i; + return 0; + } + return git_default_config(var, value, cb); } @@ -1307,9 +1320,11 @@ static int try_to_run_foreground_daemon(int free_console) die("fsmonitor--daemon is already running '%s'", the_repository->worktree); - printf(_("running fsmonitor-daemon in '%s'\n"), - the_repository->worktree); - fflush(stdout); + if (fsmonitor__announce_startup) { + fprintf(stderr, _("running fsmonitor-daemon in '%s'\n"), + the_repository->worktree); + fflush(stderr); + } #ifdef GIT_WINDOWS_NATIVE if (free_console) @@ -1360,9 +1375,11 @@ static int try_to_start_background_daemon(void) die("fsmonitor--daemon is already running '%s'", the_repository->worktree); - printf(_("starting fsmonitor-daemon in '%s'\n"), - the_repository->worktree); - fflush(stdout); + if (fsmonitor__announce_startup) { + fprintf(stderr, _("starting fsmonitor-daemon in '%s'\n"), + the_repository->worktree); + fflush(stderr); + } cp.git_cmd = 1; From 2fa93bb36e2fb8ba2c908e25756fe65a2e028097 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 2 Sep 2021 16:03:40 -0400 Subject: [PATCH 14/22] fsmonitor--daemon: cd out of worktree root Teach the fsmonitor--daemon to CD outside of the worktree before starting up. The common Git startup mechanism causes the CWD of the daemon process to be in the root of the worktree. On Windows, this causes the daemon process to hold a locked handle on the CWD and prevents other processes from moving or deleting the worktree while the daemon is running. CD to HOME before entering main event loops. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 32 +++++++++++++++++++++++++++-- compat/fsmonitor/fsm-listen-win32.c | 22 ++++++++++++++------ fsmonitor--daemon.h | 1 + 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index dd0561cfc5..e9b3f44d79 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1169,11 +1169,11 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) * before we need it. */ if (ipc_server_run_async(&state->ipc_server_data, - fsmonitor_ipc__get_path(), &ipc_opts, + state->path_ipc.buf, &ipc_opts, handle_client, state)) return error_errno( _("could not start IPC thread pool on '%s'"), - fsmonitor_ipc__get_path()); + state->path_ipc.buf); /* * Start the fsmonitor listener thread to collect filesystem @@ -1208,6 +1208,7 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) static int fsmonitor_run_daemon(void) { struct fsmonitor_daemon_state state; + const char *home; int err; memset(&state, 0, sizeof(state)); @@ -1277,6 +1278,15 @@ static int fsmonitor_run_daemon(void) strbuf_addch(&state.path_cookie_prefix, '/'); + /* + * We create a named-pipe or unix domain socket inside of the + * ".git" directory. (Well, on Windows, we base our named + * pipe in the NPFS on the absolute path of the git + * directory.) + */ + strbuf_init(&state.path_ipc, 0); + strbuf_addstr(&state.path_ipc, absolute_path(fsmonitor_ipc__get_path())); + /* * Confirm that we can create platform-specific resources for the * filesystem listener before we bother starting all the threads. @@ -1286,6 +1296,23 @@ static int fsmonitor_run_daemon(void) goto done; } + /* + * CD out of the worktree root directory. + * + * The common Git startup mechanism causes our CWD to be the + * root of the worktree. On Windows, this causes our process + * to hold a locked handle on the CWD. This prevents the + * worktree from being moved or deleted while the daemon is + * running. + * + * We assume that our FS and IPC listener threads have either + * opened all of the handles that they need or will do + * everything using absolute paths. + */ + home = getenv("HOME"); + if (home && *home && chdir(home)) + die_errno("could not cd home '%s'", home); + err = fsmonitor_run_daemon_1(&state); done: @@ -1298,6 +1325,7 @@ done: strbuf_release(&state.path_worktree_watch); strbuf_release(&state.path_gitdir_watch); strbuf_release(&state.path_cookie_prefix); + strbuf_release(&state.path_ipc); /* * NEEDSWORK: Consider "rm -rf /" diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index eb407b0748..6985903c95 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -398,12 +398,22 @@ static int recv_rdcw_watch(struct one_watch *watch) } /* - * NEEDSWORK: If an external is deleted, the above - * returns an error. I'm not sure that there's anything that - * we can do here other than failing -- the /.git - * link file would be broken anyway. We might try to check - * for that and return a better error message, but I'm not - * sure it is worth it. + * GetOverlappedResult() fails if the watched directory is + * deleted while we were waiting for an overlapped IO to + * complete. The documentation did not list specific errors, + * but I observed ERROR_ACCESS_DENIED (0x05) errors during + * testing. + * + * Note that we only get notificaiton events for events + * *within* the directory, not *on* the directory itself. + * (These might be properies of the parent directory, for + * example). + * + * NEEDSWORK: We might try to check for the deleted directory + * case and return a better error message, but I'm not sure it + * is worth it. + * + * Shutdown if we get any error. */ error("GetOverlappedResult failed on '%s' [GLE %ld]", diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index c16ef09568..a777c3a059 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -54,6 +54,7 @@ struct fsmonitor_daemon_state { struct fsmonitor_daemon_backend_data *backend_data; struct ipc_server_data *ipc_server_data; + struct strbuf path_ipc; }; /* From f710071cc8685ae18dc07b1b2c7ba1b3f2501e0d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 8 Sep 2021 16:11:02 -0400 Subject: [PATCH 15/22] fsmonitor--daemon: prepare for adding health thread Refactor daemon thread startup to make it easier to start a third thread class to monitor the health of the daemon. Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index e9b3f44d79..f42fb2ab62 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1162,6 +1162,8 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) */ .uds_disallow_chdir = 0 }; + int listener_started = 0; + int err = 0; /* * Start the IPC thread pool before the we've started the file @@ -1182,15 +1184,20 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) if (pthread_create(&state->listener_thread, NULL, fsm_listen__thread_proc, state) < 0) { ipc_server_stop_async(state->ipc_server_data); - ipc_server_await(state->ipc_server_data); - - return error(_("could not start fsmonitor listener thread")); + err = error(_("could not start fsmonitor listener thread")); + goto cleanup; } + listener_started = 1; /* * The daemon is now fully functional in background threads. + * Our primary thread should now just wait while the threads + * do all the work. + */ +cleanup: + /* * Wait for the IPC thread pool to shutdown (whether by client - * request or from filesystem activity). + * request, from filesystem activity, or an error). */ ipc_server_await(state->ipc_server_data); @@ -1199,10 +1206,16 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) * event from the IPC thread pool, but it doesn't hurt to tell * it again. And wait for it to shutdown. */ - fsm_listen__stop_async(state); - pthread_join(state->listener_thread, NULL); + if (listener_started) { + fsm_listen__stop_async(state); + pthread_join(state->listener_thread, NULL); + } - return state->error_code; + if (err) + return err; + if (state->error_code) + return state->error_code; + return 0; } static int fsmonitor_run_daemon(void) From d68a63122a41f53c71ba11be3a6b93867d049057 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 8 Sep 2021 16:46:42 -0400 Subject: [PATCH 16/22] fsmonitor--daemon: rename listener thread related variables Rename platform-specific listener thread related variables and data types as we prepare to add another backend thread type. [] `struct fsmonitor_daemon_backend_data` becomes `struct fsm_listen_data` [] `state->backend_data` becomes `state->listen_data` [] `state->error_code` becomes `state->listen_error_code` Signed-off-by: Jeff Hostetler --- builtin/fsmonitor--daemon.c | 6 +++--- compat/fsmonitor/fsm-listen-darwin.c | 30 ++++++++++++++-------------- compat/fsmonitor/fsm-listen-win32.c | 28 +++++++++++++------------- compat/fsmonitor/fsm-listen.h | 2 +- fsmonitor--daemon.h | 6 +++--- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index f42fb2ab62..f3fb865a9d 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -1213,8 +1213,8 @@ cleanup: if (err) return err; - if (state->error_code) - return state->error_code; + if (state->listen_error_code) + return state->listen_error_code; return 0; } @@ -1229,7 +1229,7 @@ static int fsmonitor_run_daemon(void) hashmap_init(&state.cookies, cookies_cmp, NULL, 0); pthread_mutex_init(&state.main_lock, NULL); pthread_cond_init(&state.cookies_cond, NULL); - state.error_code = 0; + state.listen_error_code = 0; state.current_token_data = fsmonitor_new_token_data(); /* Prepare to (recursively) watch the directory. */ diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 79d08517d7..87a8476b09 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -99,7 +99,7 @@ void FSEventStreamRelease(FSEventStreamRef stream); #include "fsm-listen.h" #include "fsmonitor--daemon.h" -struct fsmonitor_daemon_backend_data +struct fsm_listen_data { CFStringRef cfsr_worktree_path; CFStringRef cfsr_gitdir_path; @@ -230,7 +230,7 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef, const FSEventStreamEventId event_ids[]) { struct fsmonitor_daemon_state *state = ctx; - struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct fsm_listen_data *data = state->listen_data; char **paths = (char **)event_paths; struct fsmonitor_batch *batch = NULL; struct string_list cookie_list = STRING_LIST_INIT_DUP; @@ -419,11 +419,11 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) NULL, NULL }; - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; const void *dir_array[2]; CALLOC_ARRAY(data, 1); - state->backend_data = data; + state->listen_data = data; data->cfsr_worktree_path = CFStringCreateWithCString( NULL, state->path_worktree_watch.buf, kCFStringEncodingUTF8); @@ -455,18 +455,18 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) failed: error("Unable to create FSEventStream."); - FREE_AND_NULL(state->backend_data); + FREE_AND_NULL(state->listen_data); return -1; } void fsm_listen__dtor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; - if (!state || !state->backend_data) + if (!state || !state->listen_data) return; - data = state->backend_data; + data = state->listen_data; if (data->stream) { if (data->stream_started) @@ -476,14 +476,14 @@ void fsm_listen__dtor(struct fsmonitor_daemon_state *state) FSEventStreamRelease(data->stream); } - FREE_AND_NULL(state->backend_data); + FREE_AND_NULL(state->listen_data); } void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; - data = state->backend_data; + data = state->listen_data; data->shutdown_style = SHUTDOWN_EVENT; CFRunLoopStop(data->rl); @@ -491,9 +491,9 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) void fsm_listen__loop(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; - data = state->backend_data; + data = state->listen_data; data->rl = CFRunLoopGetCurrent(); @@ -510,7 +510,7 @@ void fsm_listen__loop(struct fsmonitor_daemon_state *state) switch (data->shutdown_style) { case FORCE_ERROR_STOP: - state->error_code = -1; + state->listen_error_code = -1; /* fall thru */ case FORCE_SHUTDOWN: ipc_server_stop_async(state->ipc_server_data); @@ -522,7 +522,7 @@ void fsm_listen__loop(struct fsmonitor_daemon_state *state) return; force_error_stop_without_loop: - state->error_code = -1; + state->listen_error_code = -1; ipc_server_stop_async(state->ipc_server_data); return; } diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 6985903c95..8406f8277d 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -54,7 +54,7 @@ struct one_watch wchar_t dotgit_shortname[16]; /* for 8.3 name */ }; -struct fsmonitor_daemon_backend_data +struct fsm_listen_data { struct one_watch *watch_worktree; struct one_watch *watch_gitdir; @@ -259,7 +259,7 @@ static enum get_relative_result get_relative_longname( void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) { - SetEvent(state->backend_data->hListener[LISTENER_SHUTDOWN]); + SetEvent(state->listen_data->hListener[LISTENER_SHUTDOWN]); } static struct one_watch *create_watch(struct fsmonitor_daemon_state *state, @@ -333,7 +333,7 @@ static void destroy_watch(struct one_watch *watch) free(watch); } -static int start_rdcw_watch(struct fsmonitor_daemon_backend_data *data, +static int start_rdcw_watch(struct fsm_listen_data *data, struct one_watch *watch) { DWORD dwNotifyFilter = @@ -512,7 +512,7 @@ static int process_1_worktree_event( */ static int process_worktree_events(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct fsm_listen_data *data = state->listen_data; struct one_watch *watch = data->watch_worktree; struct strbuf path = STRBUF_INIT; struct string_list cookie_list = STRING_LIST_INIT_DUP; @@ -642,7 +642,7 @@ force_shutdown: */ static int process_gitdir_events(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct fsm_listen_data *data = state->listen_data; struct one_watch *watch = data->watch_gitdir; struct strbuf path = STRBUF_INIT; struct string_list cookie_list = STRING_LIST_INIT_DUP; @@ -700,11 +700,11 @@ skip_this_path: void fsm_listen__loop(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data = state->backend_data; + struct fsm_listen_data *data = state->listen_data; DWORD dwWait; int result; - state->error_code = 0; + state->listen_error_code = 0; if (start_rdcw_watch(data, data->watch_worktree) == -1) goto force_error_stop; @@ -769,7 +769,7 @@ void fsm_listen__loop(struct fsmonitor_daemon_state *state) } force_error_stop: - state->error_code = -1; + state->listen_error_code = -1; force_shutdown: /* @@ -786,7 +786,7 @@ clean_shutdown: int fsm_listen__ctor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; CALLOC_ARRAY(data, 1); @@ -819,7 +819,7 @@ int fsm_listen__ctor(struct fsmonitor_daemon_state *state) data->nr_listener_handles++; } - state->backend_data = data; + state->listen_data = data; return 0; failed: @@ -832,16 +832,16 @@ failed: void fsm_listen__dtor(struct fsmonitor_daemon_state *state) { - struct fsmonitor_daemon_backend_data *data; + struct fsm_listen_data *data; - if (!state || !state->backend_data) + if (!state || !state->listen_data) return; - data = state->backend_data; + data = state->listen_data; CloseHandle(data->hEventShutdown); destroy_watch(data->watch_worktree); destroy_watch(data->watch_gitdir); - FREE_AND_NULL(state->backend_data); + FREE_AND_NULL(state->listen_data); } diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h index f0539349ba..41650bf897 100644 --- a/compat/fsmonitor/fsm-listen.h +++ b/compat/fsmonitor/fsm-listen.h @@ -33,7 +33,7 @@ void fsm_listen__dtor(struct fsmonitor_daemon_state *state); * do so if the listener thread receives a normal shutdown signal from * the IPC layer.) * - * It should set `state->error_code` to -1 if the daemon should exit + * It should set `state->listen_error_code` to -1 if the daemon should exit * with an error. */ void fsm_listen__loop(struct fsmonitor_daemon_state *state); diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index a777c3a059..f7de788251 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -33,7 +33,7 @@ void fsmonitor_batch__free_list(struct fsmonitor_batch *batch); */ void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, const char *path); -struct fsmonitor_daemon_backend_data; /* opaque platform-specific data */ +struct fsm_listen_data; /* opaque platform-specific data for listener thread */ struct fsmonitor_daemon_state { pthread_t listener_thread; @@ -50,8 +50,8 @@ struct fsmonitor_daemon_state { int cookie_seq; struct hashmap cookies; - int error_code; - struct fsmonitor_daemon_backend_data *backend_data; + int listen_error_code; + struct fsm_listen_data *listen_data; struct ipc_server_data *ipc_server_data; struct strbuf path_ipc; From 579a0a54fd58398b44bd35855a14dc27d67ad925 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Sep 2021 12:11:17 -0400 Subject: [PATCH 17/22] fsmonitor--daemon: stub in health thread Create another thread to watch over the daemon process and automatically shut it down if necessary. This commit creates the basic framework for a "health" thread to monitor the daemon and/or the file system. Later commits will add platform-specific code to do the actual work. The "health" thread is intended to monitor conditions that would be difficult to track inside the IPC thread pool and/or the file system listener threads. For example, when there are file system events outside of the watched worktree root or if we want to have an idle-timeout auto-shutdown feature. Signed-off-by: Jeff Hostetler --- Makefile | 6 ++- builtin/fsmonitor--daemon.c | 39 +++++++++++++++ compat/fsmonitor/fsm-health-darwin.c | 24 ++++++++++ compat/fsmonitor/fsm-health-win32.c | 72 ++++++++++++++++++++++++++++ compat/fsmonitor/fsm-health.h | 47 ++++++++++++++++++ contrib/buildsystems/CMakeLists.txt | 2 + fsmonitor--daemon.h | 4 ++ 7 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 compat/fsmonitor/fsm-health-darwin.c create mode 100644 compat/fsmonitor/fsm-health-win32.c create mode 100644 compat/fsmonitor/fsm-health.h diff --git a/Makefile b/Makefile index 9324e707bb..62e8a46983 100644 --- a/Makefile +++ b/Makefile @@ -466,8 +466,9 @@ all:: # # If your platform supports a built-in fsmonitor backend, set # FSMONITOR_DAEMON_BACKEND to the "" of the corresponding -# `compat/fsmonitor/fsm-listen-.c` that implements the -# `fsm_listen__*()` routines. +# `compat/fsmonitor/fsm-listen-.c` and +# `compat/fsmonitor/fsm-health-.c` files +# that implement the `fsm_listen__*()` and `fsm_health__*()` routines. # # If your platform has os-specific ways to tell if a repo is incompatible with # fsmonitor (whether the hook or ipc daemon version), set FSMONITOR_OS_SETTINGS @@ -1952,6 +1953,7 @@ endif ifdef FSMONITOR_DAEMON_BACKEND COMPAT_CFLAGS += -DHAVE_FSMONITOR_DAEMON_BACKEND COMPAT_OBJS += compat/fsmonitor/fsm-listen-$(FSMONITOR_DAEMON_BACKEND).o + COMPAT_OBJS += compat/fsmonitor/fsm-health-$(FSMONITOR_DAEMON_BACKEND).o endif ifdef FSMONITOR_OS_SETTINGS diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index f3fb865a9d..591433e897 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -3,6 +3,7 @@ #include "parse-options.h" #include "fsmonitor.h" #include "fsmonitor-ipc.h" +#include "compat/fsmonitor/fsm-health.h" #include "compat/fsmonitor/fsm-listen.h" #include "fsmonitor--daemon.h" #include "simple-ipc.h" @@ -1124,6 +1125,18 @@ void fsmonitor_publish(struct fsmonitor_daemon_state *state, pthread_mutex_unlock(&state->main_lock); } +static void *fsm_health__thread_proc(void *_state) +{ + struct fsmonitor_daemon_state *state = _state; + + trace2_thread_start("fsm-health"); + + fsm_health__loop(state); + + trace2_thread_exit(); + return NULL; +} + static void *fsm_listen__thread_proc(void *_state) { struct fsmonitor_daemon_state *state = _state; @@ -1162,6 +1175,7 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) */ .uds_disallow_chdir = 0 }; + int health_started = 0; int listener_started = 0; int err = 0; @@ -1189,6 +1203,17 @@ static int fsmonitor_run_daemon_1(struct fsmonitor_daemon_state *state) } listener_started = 1; + /* + * Start the health thread to watch over our process. + */ + if (pthread_create(&state->health_thread, NULL, + fsm_health__thread_proc, state) < 0) { + ipc_server_stop_async(state->ipc_server_data); + err = error(_("could not start fsmonitor health thread")); + goto cleanup; + } + health_started = 1; + /* * The daemon is now fully functional in background threads. * Our primary thread should now just wait while the threads @@ -1211,10 +1236,17 @@ cleanup: pthread_join(state->listener_thread, NULL); } + if (health_started) { + fsm_health__stop_async(state); + pthread_join(state->health_thread, NULL); + } + if (err) return err; if (state->listen_error_code) return state->listen_error_code; + if (state->health_error_code) + return state->health_error_code; return 0; } @@ -1230,6 +1262,7 @@ static int fsmonitor_run_daemon(void) pthread_mutex_init(&state.main_lock, NULL); pthread_cond_init(&state.cookies_cond, NULL); state.listen_error_code = 0; + state.health_error_code = 0; state.current_token_data = fsmonitor_new_token_data(); /* Prepare to (recursively) watch the directory. */ @@ -1309,6 +1342,11 @@ static int fsmonitor_run_daemon(void) goto done; } + if (fsm_health__ctor(&state)) { + err = error(_("could not initialize health thread")); + goto done; + } + /* * CD out of the worktree root directory. * @@ -1332,6 +1370,7 @@ done: pthread_cond_destroy(&state.cookies_cond); pthread_mutex_destroy(&state.main_lock); fsm_listen__dtor(&state); + fsm_health__dtor(&state); ipc_server_free(state.ipc_server_data); diff --git a/compat/fsmonitor/fsm-health-darwin.c b/compat/fsmonitor/fsm-health-darwin.c new file mode 100644 index 0000000000..b9f709e854 --- /dev/null +++ b/compat/fsmonitor/fsm-health-darwin.c @@ -0,0 +1,24 @@ +#include "cache.h" +#include "config.h" +#include "fsmonitor.h" +#include "fsm-health.h" +#include "fsmonitor--daemon.h" + +int fsm_health__ctor(struct fsmonitor_daemon_state *state) +{ + return 0; +} + +void fsm_health__dtor(struct fsmonitor_daemon_state *state) +{ + return; +} + +void fsm_health__loop(struct fsmonitor_daemon_state *state) +{ + return; +} + +void fsm_health__stop_async(struct fsmonitor_daemon_state *state) +{ +} diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c new file mode 100644 index 0000000000..94b1d020f2 --- /dev/null +++ b/compat/fsmonitor/fsm-health-win32.c @@ -0,0 +1,72 @@ +#include "cache.h" +#include "config.h" +#include "fsmonitor.h" +#include "fsm-health.h" +#include "fsmonitor--daemon.h" + +struct fsm_health_data +{ + HANDLE hEventShutdown; + + HANDLE hHandles[1]; /* the array does not own these handles */ +#define HEALTH_SHUTDOWN 0 + int nr_handles; /* number of active event handles */ +}; + +int fsm_health__ctor(struct fsmonitor_daemon_state *state) +{ + struct fsm_health_data *data; + + CALLOC_ARRAY(data, 1); + + data->hEventShutdown = CreateEvent(NULL, TRUE, FALSE, NULL); + + data->hHandles[HEALTH_SHUTDOWN] = data->hEventShutdown; + data->nr_handles++; + + state->health_data = data; + return 0; +} + +void fsm_health__dtor(struct fsmonitor_daemon_state *state) +{ + struct fsm_health_data *data; + + if (!state || !state->health_data) + return; + + data = state->health_data; + + CloseHandle(data->hEventShutdown); + + FREE_AND_NULL(state->health_data); +} + +void fsm_health__loop(struct fsmonitor_daemon_state *state) +{ + struct fsm_health_data *data = state->health_data; + + for (;;) { + DWORD dwWait = WaitForMultipleObjects(data->nr_handles, + data->hHandles, + FALSE, INFINITE); + + if (dwWait == WAIT_OBJECT_0 + HEALTH_SHUTDOWN) + goto clean_shutdown; + + error(_("health thread wait failed [GLE %ld]"), + GetLastError()); + goto force_error_stop; + } + +force_error_stop: + state->health_error_code = -1; + ipc_server_stop_async(state->ipc_server_data); +clean_shutdown: + return; +} + +void fsm_health__stop_async(struct fsmonitor_daemon_state *state) +{ + SetEvent(state->health_data->hHandles[HEALTH_SHUTDOWN]); +} diff --git a/compat/fsmonitor/fsm-health.h b/compat/fsmonitor/fsm-health.h new file mode 100644 index 0000000000..45547ba938 --- /dev/null +++ b/compat/fsmonitor/fsm-health.h @@ -0,0 +1,47 @@ +#ifndef FSM_HEALTH_H +#define FSM_HEALTH_H + +/* This needs to be implemented by each backend */ + +#ifdef HAVE_FSMONITOR_DAEMON_BACKEND + +struct fsmonitor_daemon_state; + +/* + * Initialize platform-specific data for the fsmonitor health thread. + * This will be called from the main thread PRIOR to staring the + * thread. + * + * Returns 0 if successful. + * Returns -1 otherwise. + */ +int fsm_health__ctor(struct fsmonitor_daemon_state *state); + +/* + * Cleanup platform-specific data for the health thread. + * This will be called from the main thread AFTER joining the thread. + */ +void fsm_health__dtor(struct fsmonitor_daemon_state *state); + +/* + * The main body of the platform-specific event loop to monitor the + * health of the daemon process. This will run in the health thread. + * + * The health thread should call `ipc_server_stop_async()` if it needs + * to cause a shutdown. (It should NOT do so if it receives a shutdown + * shutdown signal.) + * + * It should set `state->health_error_code` to -1 if the daemon should exit + * with an error. + */ +void fsm_health__loop(struct fsmonitor_daemon_state *state); + +/* + * Gently request that the health thread shutdown. + * It does not wait for it to stop. The caller should do a JOIN + * to wait for it. + */ +void fsm_health__stop_async(struct fsmonitor_daemon_state *state); + +#endif /* HAVE_FSMONITOR_DAEMON_BACKEND */ +#endif /* FSM_HEALTH_H */ diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index fb67a8a531..ecb3d366a1 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -289,12 +289,14 @@ if(SUPPORTS_SIMPLE_IPC) if(CMAKE_SYSTEM_NAME STREQUAL "Windows") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-win32.c) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-health-win32.c) add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-win32.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_FSMONITOR_DAEMON_BACKEND) list(APPEND compat_SOURCES compat/fsmonitor/fsm-listen-darwin.c) + list(APPEND compat_SOURCES compat/fsmonitor/fsm-health-darwin.c) add_compile_definitions(HAVE_FSMONITOR_OS_SETTINGS) list(APPEND compat_SOURCES compat/fsmonitor/fsm-settings-darwin.c) diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index f7de788251..716e0e4d28 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -34,9 +34,11 @@ void fsmonitor_batch__free_list(struct fsmonitor_batch *batch); void fsmonitor_batch__add_path(struct fsmonitor_batch *batch, const char *path); struct fsm_listen_data; /* opaque platform-specific data for listener thread */ +struct fsm_health_data; /* opaque platform-specific data for health thread */ struct fsmonitor_daemon_state { pthread_t listener_thread; + pthread_t health_thread; pthread_mutex_t main_lock; struct strbuf path_worktree_watch; @@ -51,7 +53,9 @@ struct fsmonitor_daemon_state { struct hashmap cookies; int listen_error_code; + int health_error_code; struct fsm_listen_data *listen_data; + struct fsm_health_data *health_data; struct ipc_server_data *ipc_server_data; struct strbuf path_ipc; From f1db862a428078042216a261dc4f034b16a8c3c5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Sep 2021 12:27:41 -0400 Subject: [PATCH 18/22] compat/fsmonitor/fsm-health-win32: add framework for periodically monitoring Create framework in Win32 version of the "health" thread to periodically inspect the system and shutdown if warranted. This version just include the setup for the timeout in WaitForMultipleObjects() and calls (currently empty) table of functions. A later commit will add functions to the table to actually inspect the system. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-health-win32.c | 54 ++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c index 94b1d020f2..3c3453369c 100644 --- a/compat/fsmonitor/fsm-health-win32.c +++ b/compat/fsmonitor/fsm-health-win32.c @@ -4,6 +4,40 @@ #include "fsm-health.h" #include "fsmonitor--daemon.h" +/* + * Every minute wake up and test our health. + */ +#define WAIT_FREQ_MS (60 * 1000) + +enum interval_fn_ctx { CTX_INIT = 0, CTX_TERM, CTX_TIMER }; + +typedef int (interval_fn)(struct fsmonitor_daemon_state *state, + enum interval_fn_ctx ctx); + +static interval_fn *table[] = { + NULL, /* must be last */ +}; + +/* + * Call all of the functions in the table. + * Shortcut and return first error. + * + * Return 0 if all succeeded. + */ +static int call_all(struct fsmonitor_daemon_state *state, + enum interval_fn_ctx ctx) +{ + int k; + + for (k = 0; table[k]; k++) { + int r = table[k](state, ctx); + if (r) + return r; + } + + return 0; +} + struct fsm_health_data { HANDLE hEventShutdown; @@ -45,15 +79,31 @@ void fsm_health__dtor(struct fsmonitor_daemon_state *state) void fsm_health__loop(struct fsmonitor_daemon_state *state) { struct fsm_health_data *data = state->health_data; + int r; + + r = call_all(state, CTX_INIT); + if (r < 0) + goto force_error_stop; + if (r > 0) + goto force_shutdown; for (;;) { DWORD dwWait = WaitForMultipleObjects(data->nr_handles, data->hHandles, - FALSE, INFINITE); + FALSE, WAIT_FREQ_MS); if (dwWait == WAIT_OBJECT_0 + HEALTH_SHUTDOWN) goto clean_shutdown; + if (dwWait == WAIT_TIMEOUT) { + r = call_all(state, CTX_TIMER); + if (r < 0) + goto force_error_stop; + if (r > 0) + goto force_shutdown; + continue; + } + error(_("health thread wait failed [GLE %ld]"), GetLastError()); goto force_error_stop; @@ -61,8 +111,10 @@ void fsm_health__loop(struct fsmonitor_daemon_state *state) force_error_stop: state->health_error_code = -1; +force_shutdown: ipc_server_stop_async(state->ipc_server_data); clean_shutdown: + call_all(state, CTX_TERM); return; } From af4ff8b62fe5f0aac7e8c9351eac8a5704489ddf Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Sep 2021 13:35:37 -0400 Subject: [PATCH 19/22] compat/fsmonitor/fsm-health-win32: force shutdown daemon if worktree root moves Force shutdown fsmonitor daemon if the worktree root directory is moved, renamed, or deleted. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-health-win32.c | 133 ++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c index 3c3453369c..2526ad9194 100644 --- a/compat/fsmonitor/fsm-health-win32.c +++ b/compat/fsmonitor/fsm-health-win32.c @@ -14,7 +14,10 @@ enum interval_fn_ctx { CTX_INIT = 0, CTX_TERM, CTX_TIMER }; typedef int (interval_fn)(struct fsmonitor_daemon_state *state, enum interval_fn_ctx ctx); +static interval_fn has_worktree_moved; + static interval_fn *table[] = { + has_worktree_moved, NULL, /* must be last */ }; @@ -45,6 +48,12 @@ struct fsm_health_data HANDLE hHandles[1]; /* the array does not own these handles */ #define HEALTH_SHUTDOWN 0 int nr_handles; /* number of active event handles */ + + struct wt_moved + { + wchar_t wpath[MAX_PATH + 1]; + BY_HANDLE_FILE_INFORMATION bhfi; + } wt_moved; }; int fsm_health__ctor(struct fsmonitor_daemon_state *state) @@ -76,6 +85,130 @@ void fsm_health__dtor(struct fsmonitor_daemon_state *state) FREE_AND_NULL(state->health_data); } +static int lookup_bhfi(wchar_t *wpath, + BY_HANDLE_FILE_INFORMATION *bhfi) +{ + DWORD desired_access = FILE_LIST_DIRECTORY; + DWORD share_mode = + FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; + HANDLE hDir; + + hDir = CreateFileW(wpath, desired_access, share_mode, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hDir == INVALID_HANDLE_VALUE) { + error(_("[GLE %ld] health thread could not open '%ls'"), + GetLastError(), wpath); + return -1; + } + + if (!GetFileInformationByHandle(hDir, bhfi)) { + error(_("[GLE %ld] health thread getting BHFI for '%ls'"), + GetLastError(), wpath); + CloseHandle(hDir); + return -1; + } + + CloseHandle(hDir); + return 0; +} + +static int bhfi_eq(const BY_HANDLE_FILE_INFORMATION *bhfi_1, + const BY_HANDLE_FILE_INFORMATION *bhfi_2) +{ + return (bhfi_1->dwVolumeSerialNumber == bhfi_2->dwVolumeSerialNumber && + bhfi_1->nFileIndexHigh == bhfi_2->nFileIndexHigh && + bhfi_1->nFileIndexLow == bhfi_2->nFileIndexLow); +} + +/* + * Shutdown if the original worktree root directory been deleted, + * moved, or renamed? + * + * Since the main thread did a "chdir(getenv($HOME))" and our CWD + * is not in the worktree root directory and because the listener + * thread added FILE_SHARE_DELETE to the watch handle, it is possible + * for the root directory to be moved or deleted while we are still + * watching it. We want to detect that here and force a shutdown. + * + * Granted, a delete MAY cause some operations to fail, such as + * GetOverlappedResult(), but it is not guaranteed. And because + * ReadDirectoryChangesW() only reports on changes *WITHIN* the + * directory, not changes *ON* the directory, our watch will not + * receive a delete event for it. + * + * A move/rename of the worktree root will also not generate an event. + * And since the listener thread already has an open handle, it may + * continue to receive events for events within the directory. + * However, the pathname of the named-pipe was constructed using the + * original location of the worktree root. (Remember named-pipes are + * stored in the NPFS and not in the actual file system.) Clients + * trying to talk to the worktree after the move/rename will not + * reach our daemon process, since we're still listening on the + * pipe with original path. + * + * Furthermore, if the user does something like: + * + * $ mv repo repo.old + * $ git init repo + * + * A new daemon cannot be started in the new instance of "repo" + * because the named-pipe is still being used by the daemon on + * the original instance. + * + * So, detect move/rename/delete and shutdown. This should also + * handle unsafe drive removal. + * + * We use the file system unique ID to distinguish the original + * directory instance from a new instance and force a shutdown + * if the unique ID changes. + * + * Since a worktree move/rename/delete/unmount doesn't happen + * that often (and we can't get an immediate event anyway), we + * use a timeout and periodically poll it. + */ +static int has_worktree_moved(struct fsmonitor_daemon_state *state, + enum interval_fn_ctx ctx) +{ + struct fsm_health_data *data = state->health_data; + BY_HANDLE_FILE_INFORMATION bhfi; + int r; + + switch (ctx) { + case CTX_TERM: + return 0; + + case CTX_INIT: + if (xutftowcs_path(data->wt_moved.wpath, + state->path_worktree_watch.buf) < 0) { + error(_("could not convert to wide characters: '%s'"), + state->path_worktree_watch.buf); + return -1; + } + + /* + * On the first call we lookup the unique sequence ID for + * the worktree root directory. + */ + return lookup_bhfi(data->wt_moved.wpath, &data->wt_moved.bhfi); + + case CTX_TIMER: + r = lookup_bhfi(data->wt_moved.wpath, &bhfi); + if (r) + return r; + if (!bhfi_eq(&data->wt_moved.bhfi, &bhfi)) { + error(_("BHFI changed '%ls'"), data->wt_moved.wpath); + return -1; + } + return 0; + + default: + die("unhandled case in 'has_worktree_moved': %d", + (int)ctx); + } + + return 0; +} + void fsm_health__loop(struct fsmonitor_daemon_state *state) { struct fsm_health_data *data = state->health_data; From 89986edf52dfbeada45b12584ad3150b92a18250 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Sep 2021 14:59:48 -0400 Subject: [PATCH 20/22] compat/fsmonitor/fsm-listen-darwin: shutdown daemon if worktree root is moved/renamed Teach the listener thread to shutdown the daemon if the spelling of the worktree root directory changes. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-listen-darwin.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 87a8476b09..d3afbbc53d 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -178,6 +178,11 @@ static void log_flags_set(const char *path, const FSEventStreamEventFlags flag) strbuf_release(&msg); } +static int ef_is_root_changed(const FSEventStreamEventFlags ef) +{ + return (ef & kFSEventStreamEventFlagRootChanged); +} + static int ef_is_root_delete(const FSEventStreamEventFlags ef) { return (ef & kFSEventStreamEventFlagItemIsDir && @@ -287,6 +292,26 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef, continue; } + if (ef_is_root_changed(event_flags[k])) { + /* + * The spelling of the pathname of the root directory + * has changed. This includes the name of the root + * directory itself of of any parent directory in the + * path. + * + * (There may be other conditions that throw this, + * but I couldn't find any information on it.) + * + * Force a shutdown now and avoid things getting + * out of sync. The Unix domain socket is inside + * the .git directory and a spelling change will make + * it hard for clients to rendezvous with us. + */ + trace_printf_key(&trace_fsmonitor, + "event: root changed"); + goto force_shutdown; + } + if (ef_ignore_xattr(event_flags[k])) { trace_printf_key(&trace_fsmonitor, "ignore-xattr: '%s', flags=0x%x", From b2ff7beef925e4a378af32c63433c0b9d6cc8b81 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 21 Sep 2021 10:49:19 -0400 Subject: [PATCH 21/22] fsmonitor: measure time taken to apply fsmonitor query result Signed-off-by: Jeff Hostetler --- fsmonitor.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fsmonitor.c b/fsmonitor.c index 3a63cda58f..2befde1ffd 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -403,6 +403,8 @@ apply_results: * information and that we should consider everything * invalid. We call this a trivial response. */ + trace2_region_enter("fsmonitor", "apply_results", istate->repo); + if (query_success && query_result.buf[bol] != '/') { /* * Mark all pathnames returned by the monitor as dirty. @@ -431,6 +433,9 @@ apply_results: if (count > fsmonitor_force_update_threshold) istate->cache_changed |= FSMONITOR_CHANGED; + trace2_data_intmax("fsmonitor", istate->repo, "apply_count", + count); + } else { /* * We received a trivial response, so invalidate everything. @@ -458,6 +463,8 @@ apply_results: if (istate->untracked) istate->untracked->use_fsmonitor = 0; } + trace2_region_leave("fsmonitor", "apply_results", istate->repo); + strbuf_release(&query_result); /* Now that we've updated istate, save the last_update_token */ From 8d4e3b0b3f2090c919c6e359c8a4bcf40201eb31 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 21 Sep 2021 15:06:33 -0400 Subject: [PATCH 22/22] fsmonitor: optimize processing of directory events Teach Git to perform binary search over the cache-entries for a directory notification and then linearly scan forward to find the immediate children. Previously, when the FSMonitor reported a modified directory Git would perform a linear search on the entire cache-entry array for all entries matching that directory prefix and invalidate them. Since the cache-entry array is already sorted, we can use a binary search to find the first matching entry and then only linearly walk forward and invalidate entries until the prefix changes. Also, the original code would invalidate anything having the same directory prefix. Since a directory event should only be received for items that are immediately within the directory (and not within sub-directories of it), only invalidate those entries and not the whole subtree. Signed-off-by: Jeff Hostetler --- fsmonitor.c | 73 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index 2befde1ffd..e866306fe5 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -198,30 +198,68 @@ int fsmonitor_is_trivial_response(const struct strbuf *query_result) static void fsmonitor_refresh_callback(struct index_state *istate, char *name) { int i, len = strlen(name); - if (name[len - 1] == '/') { + int pos = index_name_pos(istate, name, len); + trace_printf_key(&trace_fsmonitor, + "fsmonitor_refresh_callback '%s' (pos %d)", + name, pos); + + if (name[len - 1] == '/') { /* - * TODO We should binary search to find the first path with - * TODO this directory prefix. Then linearly update entries - * TODO while the prefix matches. Taking care to search without - * TODO the trailing slash -- because '/' sorts after a few - * TODO interesting special chars, like '.' and ' '. + * The daemon can decorate directory events, such as + * moves or renames, with a trailing slash if the OS + * FS Event contains sufficient information, such as + * MacOS. + * + * Use this to invalidate the entire cone under that + * directory. + * + * We do not expect an exact match because the index + * does not normally contain directory entries, so we + * start at the insertion point and scan. */ + if (pos < 0) + pos = -pos - 1; /* Mark all entries for the folder invalid */ - for (i = 0; i < istate->cache_nr; i++) { - if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID && - starts_with(istate->cache[i]->name, name)) - istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; + for (i = pos; i < istate->cache_nr; i++) { + if (!starts_with(istate->cache[i]->name, name)) + break; + istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; } - /* Need to remove the / from the path for the untracked cache */ - name[len - 1] = '\0'; - } else { - int pos = index_name_pos(istate, name, strlen(name)); - if (pos >= 0) { - struct cache_entry *ce = istate->cache[pos]; - ce->ce_flags &= ~CE_FSMONITOR_VALID; + /* + * We need to remove the traling "/" from the path + * for the untracked cache. + */ + name[len - 1] = '\0'; + } else if (pos >= 0) { + /* + * We have an exact match for this path and can just + * invalidate it. + */ + istate->cache[pos]->ce_flags &= ~CE_FSMONITOR_VALID; + } else { + /* + * The path is not a tracked file -or- it is a + * directory event on a platform that cannot + * distinguish between file and directory events in + * the event handler, such as Windows. + * + * Scan as if it is a directory and invalidate the + * cone under it. (But remember to ignore items + * between "name" and "name/", such as "name-" and + * "name.". + */ + pos = -pos - 1; + + for (i = pos; i < istate->cache_nr; i++) { + if (!starts_with(istate->cache[i]->name, name)) + break; + if ((unsigned char)istate->cache[i]->name[len] > '/') + break; + if (istate->cache[i]->name[len] == '/') + istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; } } @@ -229,7 +267,6 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) * Mark the untracked cache dirty even if it wasn't found in the index * as it could be a new untracked file. */ - trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback '%s'", name); untracked_cache_invalidate_path(istate, name, 0); }