From 11d1d91e02efb272819d425af9b52664284a380e Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sun, 24 Jul 2011 15:54:04 +0200 Subject: [PATCH 01/72] t9350: point out that refs are not updated correctly This happens only when the corresponding commits are not exported in the current fast-export run. This can happen either when the relevant commit is already marked, or when the commit is explicitly marked as UNINTERESTING with a negative ref by another argument. This breaks fast-export basec remote helpers. Signed-off-by: Sverre Rabbelier --- t/t9350-fast-export.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t9350-fast-export.sh b/t/t9350-fast-export.sh index 1372842559..e2d4e9c66f 100755 --- a/t/t9350-fast-export.sh +++ b/t/t9350-fast-export.sh @@ -747,4 +747,15 @@ test_expect_success 'merge commit gets exported with --import-marks' ' ) ' +cat > expected << EOF +reset refs/heads/master +from $(git rev-parse master) + +EOF + +test_expect_failure 'refs are updated even if no commits need to be exported' ' + git fast-export master..master > actual && + test_cmp expected actual +' + test_done From 861e22cf7c7ec04e0ba34d1d1d86657a012be5d5 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 28 Aug 2010 20:49:01 -0500 Subject: [PATCH 02/72] transport-helper: add trailing -- [PT: ensure we add an additional element to the argv array] Signed-off-by: Sverre Rabbelier Signed-off-by: Johannes Schindelin --- transport-helper.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transport-helper.c b/transport-helper.c index 5f6e0b3bd8..021f53fb8e 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -478,6 +478,8 @@ static int get_exporter(struct transport *transport, for (i = 0; i < revlist_args->nr; i++) strvec_push(&fastexport->args, revlist_args->items[i].string); + strvec_push(&fastexport->args, "--"); + fastexport->git_cmd = 1; return start_command(fastexport); } From f980ada47f941142b972a1edcf73e85785df52f2 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sun, 24 Jul 2011 00:06:00 +0200 Subject: [PATCH 03/72] remote-helper: check helper status after import/export Signed-off-by: Johannes Schindelin Signed-off-by: Sverre Rabbelier --- t/t5801-remote-helpers.sh | 2 +- transport-helper.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/t/t5801-remote-helpers.sh b/t/t5801-remote-helpers.sh index 0f04b6cddb..9148090e98 100755 --- a/t/t5801-remote-helpers.sh +++ b/t/t5801-remote-helpers.sh @@ -236,7 +236,7 @@ test_expect_success 'push update refs failure' ' echo "update fail" >>file && git commit -a -m "update fail" && git rev-parse --verify testgit/origin/heads/update >expect && - test_expect_code 1 env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ + test_must_fail env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ git push origin update && git rev-parse --verify testgit/origin/heads/update >actual && test_cmp expect actual diff --git a/transport-helper.c b/transport-helper.c index 021f53fb8e..e60f96fb15 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -484,6 +484,19 @@ static int get_exporter(struct transport *transport, return start_command(fastexport); } +static void check_helper_status(struct helper_data *data) +{ + int pid, status; + + pid = waitpid(data->helper->pid, &status, WNOHANG); + if (pid < 0) + die("Could not retrieve status of remote helper '%s'", + data->name); + if (pid > 0 && WIFEXITED(status)) + die("Remote helper '%s' died with %d", + data->name, WEXITSTATUS(status)); +} + static int fetch_with_import(struct transport *transport, int nr_heads, struct ref **to_fetch) { @@ -520,6 +533,7 @@ static int fetch_with_import(struct transport *transport, if (finish_command(&fastimport)) die(_("error while running fast-import")); + check_helper_status(data); /* * The fast-import stream of a remote helper that advertises @@ -1107,6 +1121,7 @@ static int push_refs_with_export(struct transport *transport, if (finish_command(&exporter)) die(_("error while running fast-export")); + check_helper_status(data); if (push_update_refs_status(data, remote_refs, flags)) return 1; From 892e24dd54c80a3091bde894b374ee91f16af0b0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 Apr 2017 12:09:08 +0200 Subject: [PATCH 04/72] mingw: demonstrate a problem with certain absolute paths On Windows, there are several categories of absolute paths. One such category starts with a backslash and is implicitly relative to the drive associated with the current working directory. Example: c: git clone https://github.com/git-for-windows/git \G4W should clone into C:\G4W. There is currently a problem with that, in that mingw_mktemp() does not expect the _wmktemp() function to prefix the absolute path with the drive prefix, and as a consequence, the resulting path does not fit into the originally-passed string buffer. The symptom is a "Result too large" error. Reported by Juan Carlos Arevalo Baeza. Signed-off-by: Johannes Schindelin --- t/t5580-unc-paths.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/t/t5580-unc-paths.sh b/t/t5580-unc-paths.sh index cf768b3a27..28505e5d2d 100755 --- a/t/t5580-unc-paths.sh +++ b/t/t5580-unc-paths.sh @@ -17,14 +17,11 @@ fi UNCPATH="$(winpwd)" case "$UNCPATH" in [A-Z]:*) + WITHOUTDRIVE="${UNCPATH#?:}" # Use administrative share e.g. \\localhost\C$\git-sdk-64\usr\src\git # (we use forward slashes here because MSYS2 and Git accept them, and # they are easier on the eyes) - UNCPATH="//localhost/${UNCPATH%%:*}\$/${UNCPATH#?:}" - test -d "$UNCPATH" || { - skip_all='could not access administrative share; skipping' - test_done - } + UNCPATH="//localhost/${UNCPATH%%:*}\$$WITHOUTDRIVE" ;; *) skip_all='skipping UNC path tests, cannot determine current path as UNC' @@ -32,6 +29,18 @@ case "$UNCPATH" in ;; esac +test_expect_failure 'clone into absolute path lacking a drive prefix' ' + USINGBACKSLASHES="$(echo "$WITHOUTDRIVE"/without-drive-prefix | + tr / \\\\)" && + git clone . "$USINGBACKSLASHES" && + test -f without-drive-prefix/.git/HEAD +' + +test -d "$UNCPATH" || { + skip_all='could not access administrative share; skipping' + test_done +} + test_expect_success setup ' test_commit initial ' From 89ce82b7dce218965b27712270e2e16d0079d1db Mon Sep 17 00:00:00 2001 From: Rohit Ashiwal Date: Fri, 15 Feb 2019 19:03:57 +0530 Subject: [PATCH 05/72] archive: replace write_or_die() calls with write_block_or_die() MinGit for Windows comes without `gzip` bundled inside, git-archive uses `gzip -cn` to compress tar files but for this to work, gzip needs to be present on the host system. In the next commit, we will change the gzip compression so that we no longer spawn `gzip` but let zlib perform the compression in the same process instead. In preparation for this, we consolidate all the block writes into a single function. This closes https://github.com/git-for-windows/git/issues/1970 Signed-off-by: Rohit Ashiwal Signed-off-by: Johannes Schindelin --- archive-tar.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/archive-tar.c b/archive-tar.c index a971fdc0f6..8b08fdd379 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -17,6 +17,8 @@ static unsigned long offset; static int tar_umask = 002; +static gzFile gzip; + static int write_tar_filter_archive(const struct archiver *ar, struct archiver_args *args); @@ -38,11 +40,21 @@ static int write_tar_filter_archive(const struct archiver *ar, #define USTAR_MAX_MTIME 077777777777ULL #endif +/* writes out the whole block, or dies if fails */ +static void write_block_or_die(const char *block) { + if (gzip) { + if (gzwrite(gzip, block, (unsigned) BLOCKSIZE) != BLOCKSIZE) + die(_("gzwrite failed")); + } else { + write_or_die(1, block, BLOCKSIZE); + } +} + /* writes out the whole block, but only if it is full */ static void write_if_needed(void) { if (offset == BLOCKSIZE) { - write_or_die(1, block, BLOCKSIZE); + write_block_or_die(block); offset = 0; } } @@ -66,7 +78,7 @@ static void do_write_blocked(const void *data, unsigned long size) write_if_needed(); } while (size >= BLOCKSIZE) { - write_or_die(1, buf, BLOCKSIZE); + write_block_or_die(buf); size -= BLOCKSIZE; buf += BLOCKSIZE; } @@ -101,10 +113,10 @@ static void write_trailer(void) { int tail = BLOCKSIZE - offset; memset(block + offset, 0, tail); - write_or_die(1, block, BLOCKSIZE); + write_block_or_die(block); if (tail < 2 * RECORDSIZE) { memset(block, 0, offset); - write_or_die(1, block, BLOCKSIZE); + write_block_or_die(block); } } From b68341ccfc9c217cd380f898c6b10709826d22f3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 9 Apr 2012 13:04:35 -0500 Subject: [PATCH 06/72] Always auto-gc after calling a fast-import transport After importing anything with fast-import, we should always let the garbage collector do its job, since the objects are written to disk inefficiently. This brings down an initial import of http://selenic.com/hg from about 230 megabytes to about 14. In the future, we may want to make this configurable on a per-remote basis, or maybe teach fast-import about it in the first place. Signed-off-by: Johannes Schindelin --- transport-helper.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/transport-helper.c b/transport-helper.c index e60f96fb15..a257ae8eb4 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -16,6 +16,8 @@ #include "protocol.h" static int debug; +/* TODO: put somewhere sensible, e.g. git_transport_options? */ +static int auto_gc = 1; struct helper_data { const char *name; @@ -567,6 +569,12 @@ static int fetch_with_import(struct transport *transport, } } strbuf_release(&buf); + if (auto_gc) { + const char *argv_gc_auto[] = { + "gc", "--auto", "--quiet", NULL, + }; + run_command_v_opt(argv_gc_auto, RUN_GIT_CMD); + } return 0; } From b4f019b34a39d118b99ac59a5028e163d5606290 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 15 Dec 2016 11:34:39 -0500 Subject: [PATCH 07/72] diffcore-rename: speed up register_rename_src Teach register_rename_src() to see if new file pair can simply be appended to the rename_src[] array before performing the binary search to find the proper insertion point. This is a performance optimization. This routine is called during run_diff_files in status and the caller is iterating over the sorted index, so we should expect to be able to append in the normal case. The existing insert logic is preserved so we don't have to assume that, but simply take advantage of it if possible. Signed-off-by: Jeff Hostetler --- diffcore-rename.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/diffcore-rename.c b/diffcore-rename.c index d367a6d244..dfbf69cfac 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -84,6 +84,18 @@ static struct diff_rename_src *register_rename_src(struct diff_filepair *p) first = 0; last = rename_src_nr; + + if (last > 0) { + struct diff_rename_src *src = &(rename_src[last-1]); + int cmp = strcmp(one->path, src->p->one->path); + if (!cmp) + return src; + if (cmp > 0) { + first = last; + goto append_it; + } + } + while (last > first) { int next = first + ((last - first) >> 1); struct diff_rename_src *src = &(rename_src[next]); @@ -97,6 +109,7 @@ static struct diff_rename_src *register_rename_src(struct diff_filepair *p) first = next+1; } +append_it: /* insert to make it at "first" */ ALLOC_GROW(rename_src, rename_src_nr + 1, rename_src_alloc); rename_src_nr++; From 944335d3fafed7173f7750e8b2613775acbf2e07 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 Apr 2017 12:38:30 +0200 Subject: [PATCH 08/72] mingw: allow absolute paths without drive prefix When specifying an absolute path without a drive prefix, we convert that path internally. Let's make sure that we handle that case properly, too ;-) This fixes the command git clone https://github.com/git-for-windows/git \G4W Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 +++++++++- t/t5580-unc-paths.sh | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index a43599841c..80158a0cae 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1007,11 +1007,19 @@ unsigned int sleep (unsigned int seconds) char *mingw_mktemp(char *template) { wchar_t wtemplate[MAX_PATH]; + int offset = 0; + if (xutftowcs_path(wtemplate, template) < 0) return NULL; + + if (is_dir_sep(template[0]) && !is_dir_sep(template[1]) && + iswalpha(wtemplate[0]) && wtemplate[1] == L':') { + /* We have an absolute path missing the drive prefix */ + offset = 2; + } if (!_wmktemp(wtemplate)) return NULL; - if (xwcstoutf(template, wtemplate, strlen(template) + 1) < 0) + if (xwcstoutf(template, wtemplate + offset, strlen(template) + 1) < 0) return NULL; return template; } diff --git a/t/t5580-unc-paths.sh b/t/t5580-unc-paths.sh index 28505e5d2d..efc4264903 100755 --- a/t/t5580-unc-paths.sh +++ b/t/t5580-unc-paths.sh @@ -29,7 +29,7 @@ case "$UNCPATH" in ;; esac -test_expect_failure 'clone into absolute path lacking a drive prefix' ' +test_expect_success 'clone into absolute path lacking a drive prefix' ' USINGBACKSLASHES="$(echo "$WITHOUTDRIVE"/without-drive-prefix | tr / \\\\)" && git clone . "$USINGBACKSLASHES" && From 01634cd1c0d344b6c6e01a0a649ac99e5c75001e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 11 Dec 2015 06:59:13 +0100 Subject: [PATCH 09/72] mingw: handle absolute paths in expand_user_path() On Windows, an absolute POSIX path needs to be turned into a Windows one. Signed-off-by: Johannes Schindelin --- path.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/path.c b/path.c index 7b385e5eb2..44462ec337 100644 --- a/path.c +++ b/path.c @@ -12,6 +12,7 @@ #include "packfile.h" #include "object-store.h" #include "lockfile.h" +#include "exec-cmd.h" static int get_st_mode_bits(const char *path, int *mode) { @@ -732,6 +733,10 @@ char *expand_user_path(const char *path, int real_home) if (path == NULL) goto return_null; +#ifdef __MINGW32__ + if (path[0] == '/') + return system_path(path + 1); +#endif if (path[0] == '~') { const char *first_slash = strchrnul(path, '/'); const char *username = path + 1; From 8430320f59a9ff63261d082621239cf1f0942893 Mon Sep 17 00:00:00 2001 From: Rohit Ashiwal Date: Tue, 19 Feb 2019 22:28:41 +0530 Subject: [PATCH 10/72] archive: avoid spawning `gzip` As we already link to the zlib library, we can perform the compression without even requiring gzip on the host machine. Signed-off-by: Rohit Ashiwal Signed-off-by: Johannes Schindelin --- archive-tar.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/archive-tar.c b/archive-tar.c index 8b08fdd379..04351b2d39 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -458,18 +458,34 @@ static int write_tar_filter_archive(const struct archiver *ar, filter.use_shell = 1; filter.in = -1; - if (start_command(&filter) < 0) - die_errno(_("unable to start '%s' filter"), argv[0]); - close(1); - if (dup2(filter.in, 1) < 0) - die_errno(_("unable to redirect descriptor")); - close(filter.in); + if (!strcmp("gzip -cn", ar->data)) { + char outmode[4] = "wb\0"; + + if (args->compression_level >= 0 && args->compression_level <= 9) + outmode[2] = '0' + args->compression_level; + + gzip = gzdopen(fileno(stdout), outmode); + if (!gzip) + die(_("Could not gzdopen stdout")); + } else { + if (start_command(&filter) < 0) + die_errno(_("unable to start '%s' filter"), argv[0]); + close(1); + if (dup2(filter.in, 1) < 0) + die_errno(_("unable to redirect descriptor")); + close(filter.in); + } r = write_tar_archive(ar, args); - close(1); - if (finish_command(&filter) != 0) - die(_("'%s' filter reported error"), argv[0]); + if (gzip) { + if (gzclose(gzip) != Z_OK) + die(_("gzclose failed")); + } else { + close(1); + if (finish_command(&filter) != 0) + die(_("'%s' filter reported error"), argv[0]); + } strbuf_release(&cmd); return r; From 77fa96309878c99806f3cf2ece2227faf6c0ce91 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 25 Apr 2019 12:05:29 -0400 Subject: [PATCH 11/72] gitk: prevent overly long command lines To avoid running into command line limitations, some of Git's commands support the `--stdin` option. Let's use exactly this option in the three rev-list/log invocations in gitk that would otherwise possibly run the danger of trying to invoke a too-long command line. While it is easy to redirect either stdin or stdout in Tcl/Tk scripts, what we need here is both. We need to capture the output, yet we also need to pipe in the revs/files arguments via stdin (because stdin does not have any limit, unlike the command line). To help this, we use the neat Tcl feature where you can capture stdout and at the same time feed a fixed string as stdin to the spawned process. One non-obvious aspect about this change is that the `--stdin` option allows to specify revs, the double-dash, and files, but *no* other options such as `--not`. This is addressed by prefixing the "negative" revs with `^` explicitly rather than relying on the `--not` option (thanks for coming up with that idea, Max!). This fixes https://github.com/git-for-windows/git/issues/1987 Analysis-and-initial-patch-by: Max Kirillov Signed-off-by: Johannes Schindelin --- gitk-git/gitk | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/gitk-git/gitk b/gitk-git/gitk index 23d9dd1fe0..42594da561 100755 --- a/gitk-git/gitk +++ b/gitk-git/gitk @@ -405,14 +405,16 @@ proc start_rev_list {view} { if {$revs eq {}} { return 0 } - set args [concat $vflags($view) $revs] + set args $vflags($view) } else { + set revs {} set args $vorigargs($view) } if {[catch { set fd [open [concat | git log --no-color -z --pretty=raw $show_notes \ - --parents --boundary $args "--" $files] r] + --parents --boundary $args --stdin \ + "<<[join [concat $revs "--" $files] "\\n"]"] r] } err]} { error_popup "[mc "Error executing git log:"] $err" return 0 @@ -554,13 +556,19 @@ proc updatecommits {} { set revs $newrevs set vposids($view) [lsort -unique [concat $oldpos $vposids($view)]] } - set args [concat $vflags($view) $revs --not $oldpos] + set args $vflags($view) + foreach r $oldpos { + lappend revs "^$r" + } } else { + set revs {} set args $vorigargs($view) } if {[catch { set fd [open [concat | git log --no-color -z --pretty=raw $show_notes \ - --parents --boundary $args "--" $vfilelimit($view)] r] + --parents --boundary $args --stdin \ + "<<[join [concat $revs "--" \ + $vfilelimit($view)] "\\n"]"] r] } err]} { error_popup "[mc "Error executing git log:"] $err" return @@ -10228,10 +10236,16 @@ proc getallcommits {} { foreach id $seeds { lappend ids "^$id" } + lappend ids "--" } } if {$ids ne {}} { - set fd [open [concat $cmd $ids] r] + if {$ids eq "--all"} { + set cmd [concat $cmd "--all"] + } else { + set cmd [concat $cmd --stdin "<<[join $ids "\\n"]"] + } + set fd [open $cmd r] fconfigure $fd -blocking 0 incr allcommits nowbusy allcommits From ba5f9c24fca5fa0682b058293ef05697a2905682 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 7 Dec 2018 13:39:30 +0100 Subject: [PATCH 12/72] clean: do not traverse mount points It seems to be not exactly rare on Windows to install NTFS junction points (the equivalent of "bind mounts" on Linux/Unix) in worktrees, e.g. to map some development tools into a subdirectory. In such a scenario, it is pretty horrible if `git clean -dfx` traverses into the mapped directory and starts to "clean up". Let's just not do that. Let's make sure before we traverse into a directory that it is not a mount point (or junction). This addresses https://github.com/git-for-windows/git/issues/607 Signed-off-by: Johannes Schindelin --- builtin/clean.c | 14 ++++++++++++++ cache.h | 1 + compat/mingw.c | 22 ++++++++++++++++++++++ compat/mingw.h | 3 +++ git-compat-util.h | 4 ++++ path.c | 39 +++++++++++++++++++++++++++++++++++++++ t/t7300-clean.sh | 9 +++++++++ 7 files changed, 92 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index 687ab473c2..592edc838c 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -34,6 +34,8 @@ static const char *msg_remove = N_("Removing %s\n"); static const char *msg_would_remove = N_("Would remove %s\n"); static const char *msg_skip_git_dir = N_("Skipping repository %s\n"); static const char *msg_would_skip_git_dir = N_("Would skip repository %s\n"); +static const char *msg_skip_mount_point = N_("Skipping mount point %s\n"); +static const char *msg_would_skip_mount_point = N_("Would skip mount point %s\n"); static const char *msg_warn_remove_failed = N_("failed to remove %s"); static const char *msg_warn_lstat_failed = N_("could not lstat %s\n"); @@ -171,6 +173,18 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, goto out; } + if (is_mount_point(path)) { + if (!quiet) { + quote_path_relative(path->buf, prefix, "ed); + printf(dry_run ? + _(msg_would_skip_mount_point) : + _(msg_skip_mount_point), quoted.buf); + } + *dir_gone = 0; + + goto out; + } + dir = opendir(path->buf); if (!dir) { /* an empty dir could be removed even if it is unreadble */ diff --git a/cache.h b/cache.h index 0a0b32feca..ff316eb2f0 100644 --- a/cache.h +++ b/cache.h @@ -1240,6 +1240,7 @@ int normalize_path_copy_len(char *dst, const char *src, int *prefix_len); int normalize_path_copy(char *dst, const char *src); int longest_ancestor_length(const char *path, struct string_list *prefixes); char *strip_path_suffix(const char *path, const char *suffix); +int is_mount_point_via_stat(struct strbuf *path); int daemon_avoid_alias(const char *path); /* diff --git a/compat/mingw.c b/compat/mingw.c index a43599841c..ff9b2a7e27 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2461,6 +2461,28 @@ pid_t waitpid(pid_t pid, int *status, int options) return -1; } +int mingw_is_mount_point(struct strbuf *path) +{ + WIN32_FIND_DATAW findbuf = { 0 }; + HANDLE handle; + wchar_t wfilename[MAX_PATH]; + int wlen = xutftowcs_path(wfilename, path->buf); + if (wlen < 0) + die(_("could not get long path for '%s'"), path->buf); + + /* remove trailing slash, if any */ + if (wlen > 0 && wfilename[wlen - 1] == L'/') + wfilename[--wlen] = L'\0'; + + handle = FindFirstFileW(wfilename, &findbuf); + if (handle == INVALID_HANDLE_VALUE) + return 0; + FindClose(handle); + + return (findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && + (findbuf.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT); +} + int xutftowcsn(wchar_t *wcs, const char *utfs, size_t wcslen, int utflen) { int upos = 0, wpos = 0; diff --git a/compat/mingw.h b/compat/mingw.h index af8eddd73e..5cc2ed3ff4 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -442,6 +442,9 @@ static inline void convert_slashes(char *path) if (*path == '\\') *path = '/'; } +struct strbuf; +int mingw_is_mount_point(struct strbuf *path); +#define is_mount_point mingw_is_mount_point #define PATH_SEP ';' char *mingw_query_user_email(void); #define query_user_email mingw_query_user_email diff --git a/git-compat-util.h b/git-compat-util.h index 7d3db43f11..a525ad078f 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -406,6 +406,10 @@ static inline int git_has_dir_sep(const char *path) #define has_dir_sep(path) git_has_dir_sep(path) #endif +#ifndef is_mount_point +#define is_mount_point is_mount_point_via_stat +#endif + #ifndef query_user_email #define query_user_email() NULL #endif diff --git a/path.c b/path.c index 7b385e5eb2..59931e3c60 100644 --- a/path.c +++ b/path.c @@ -1289,6 +1289,45 @@ char *strip_path_suffix(const char *path, const char *suffix) return offset == -1 ? NULL : xstrndup(path, offset); } +int is_mount_point_via_stat(struct strbuf *path) +{ + size_t len = path->len; + unsigned int current_dev; + struct stat st; + + if (!strcmp("/", path->buf)) + return 1; + + strbuf_addstr(path, "/."); + if (lstat(path->buf, &st)) { + /* + * If we cannot access the current directory, we cannot say + * that it is a bind mount. + */ + strbuf_setlen(path, len); + return 0; + } + current_dev = st.st_dev; + + /* Now look at the parent directory */ + strbuf_addch(path, '.'); + if (lstat(path->buf, &st)) { + /* + * If we cannot access the parent directory, we cannot say + * that it is a bind mount. + */ + strbuf_setlen(path, len); + return 0; + } + strbuf_setlen(path, len); + + /* + * If the device ID differs between current and parent directory, + * then it is a bind mount. + */ + return current_dev != st.st_dev; +} + int daemon_avoid_alias(const char *p) { int sl, ndot; diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index cb5e34d94c..71f450d139 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -746,4 +746,13 @@ test_expect_success 'clean untracked paths by pathspec' ' test_must_be_empty actual ' +test_expect_success MINGW 'clean does not traverse mount points' ' + mkdir target && + >target/dont-clean-me && + git init with-mountpoint && + cmd //c "mklink /j with-mountpoint\\mountpoint target" && + git -C with-mountpoint clean -dfx && + test_path_is_file target/dont-clean-me +' + test_done From fe6cf2c582625645bbe120e3cee747cd25dc761b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 6 Oct 2019 22:58:49 +0200 Subject: [PATCH 13/72] mingw: move Git for Windows' system config where users expect it Git for Windows' prefix is `/mingw64/` (or `/mingw32/` for 32-bit versions), therefore the system config is located at the clunky location `C:\Program Files\Git\mingw64\etc\gitconfig`. This moves the system config into a more logical location: the `mingw64` part of `C:\Program Files\Git\mingw64\etc\gitconfig` never made sense, as it is a mere implementation detail. Let's skip the `mingw64` part and move this to `C:\Program Files\Git\etc\gitconfig`. Side note: in the rare (and not recommended) case a user chooses to install 32-bit Git for Windows on a 64-bit system, the path will of course be `C:\Program Files (x86)\Git\etc\gitconfig`. Background: During the Git for Windows v1.x days, the system config was located at `C:\Program Files (x86)\Git\etc\gitconfig`. With Git for Windows v2.x, it moved to `C:\Program Files\Git\mingw64\gitconfig` (or `C:\Program Files (x86)\Git\mingw32\gitconfig`). Rather than fixing it back then, we tried to introduce a "Windows-wide" config, but that never caught on. Likewise, we move the system `gitattributes` into the same directory. Obviously, we are cautious to do this only for the known install locations `/mingw64` and `/mingw32`; If anybody wants to override that while building their version of Git (e.g. via `make prefix=$HOME`), we leave the default location of the system config and gitattributes alone. Signed-off-by: Johannes Schindelin --- config.mak.uname | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 198ab1e58f..17e61bd626 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -433,6 +433,11 @@ ifeq ($(uname_S),Windows) NO_POSIX_GOODIES = UnfortunatelyYes NATIVE_CRLF = YesPlease DEFAULT_HELP_FORMAT = html +ifeq (/mingw64,$(subst 32,64,$(prefix))) + # Move system config into top-level /etc/ + ETC_GITCONFIG = ../etc/gitconfig + ETC_GITATTRIBUTES = ../etc/gitattributes +endif CC = compat/vcbuild/scripts/clink.pl AR = compat/vcbuild/scripts/lib.pl @@ -667,6 +672,11 @@ else NO_LIBPCRE1_JIT = UnfortunatelyYes NO_CURL = USE_NED_ALLOCATOR = YesPlease + ifeq (/mingw64,$(subst 32,64,$(prefix))) + # Move system config into top-level /etc/ + ETC_GITCONFIG = ../etc/gitconfig + ETC_GITATTRIBUTES = ../etc/gitattributes + endif else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease From 51b36d9c6927ca5ee994d876526553e15dd88064 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 4 Sep 2017 11:59:45 +0200 Subject: [PATCH 14/72] mingw: change core.fsyncObjectFiles = 1 by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the documentation of said setting: This boolean will enable fsync() when writing object files. This is a total waste of time and effort on a filesystem that orders data writes properly, but can be useful for filesystems that do not use journalling (traditional UNIX filesystems) or that only journal metadata and not file contents (OS X’s HFS+, or Linux ext3 with "data=writeback"). The most common file system on Windows (NTFS) does not guarantee that order, therefore a sudden loss of power (or any other event causing an unclean shutdown) would cause corrupt files (i.e. files filled with NULs). Therefore we need to change the default. Note that the documentation makes it sound as if this causes really bad performance. In reality, writing loose objects is something that is done only rarely, and only a handful of files at a time. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 1 + 1 file changed, 1 insertion(+) diff --git a/compat/mingw.c b/compat/mingw.c index a43599841c..a1f3006cc9 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2841,6 +2841,7 @@ int wmain(int argc, const wchar_t **wargv) #endif maybe_redirect_std_handles(); + fsync_object_files = 1; /* determine size of argv and environ conversion buffer */ maxlen = wcslen(wargv[0]); From 191f8b7d0b47a72304e66d7517951cef8f0a704e Mon Sep 17 00:00:00 2001 From: Nico Rieck Date: Mon, 19 Aug 2019 20:46:12 +0200 Subject: [PATCH 15/72] gitk: Escape file paths before piping to git log Fixes https://github.com/git-for-windows/git/issues/2293 Signed-off-by: Nico Rieck --- gitk-git/gitk | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/gitk-git/gitk b/gitk-git/gitk index 42594da561..64678c0aa7 100755 --- a/gitk-git/gitk +++ b/gitk-git/gitk @@ -353,6 +353,16 @@ proc parseviewrevs {view revs} { return $ret } +# Escapes a list of filter paths to be passed to git log via stdin. Note that +# paths must not be quoted. +proc escape_filter_paths {paths} { + set escaped [list] + foreach path $paths { + lappend escaped [string map {\\ \\\\ "\ " "\\\ "} $path] + } + return $escaped +} + # Start off a git log process and arrange to read its output proc start_rev_list {view} { global startmsecs commitidx viewcomplete curview @@ -414,7 +424,8 @@ proc start_rev_list {view} { if {[catch { set fd [open [concat | git log --no-color -z --pretty=raw $show_notes \ --parents --boundary $args --stdin \ - "<<[join [concat $revs "--" $files] "\\n"]"] r] + "<<[join [concat $revs "--" \ + [escape_filter_paths $files]] "\\n"]"] r] } err]} { error_popup "[mc "Error executing git log:"] $err" return 0 @@ -568,7 +579,8 @@ proc updatecommits {} { set fd [open [concat | git log --no-color -z --pretty=raw $show_notes \ --parents --boundary $args --stdin \ "<<[join [concat $revs "--" \ - $vfilelimit($view)] "\\n"]"] r] + [escape_filter_paths \ + $vfilelimit($view)]] "\\n"]"] r] } err]} { error_popup "[mc "Error executing git log:"] $err" return From fd8a0a1e6b7107ce877bf8b618705f0dee3ab6e1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 11 Dec 2018 12:55:26 +0100 Subject: [PATCH 16/72] clean: remove mount points when possible Windows' equivalent to "bind mounts", NTFS junction points, can be unlinked without affecting the mount target. This is clearly what users expect to happen when they call `git clean -dfx` in a worktree that contains NTFS junction points: the junction should be removed, and the target directory of said junction should be left alone (unless it is inside the worktree). Signed-off-by: Johannes Schindelin --- builtin/clean.c | 15 ++++++++++++++- compat/mingw.h | 1 + t/t7300-clean.sh | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/builtin/clean.c b/builtin/clean.c index 592edc838c..230c6700df 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -34,8 +34,10 @@ static const char *msg_remove = N_("Removing %s\n"); static const char *msg_would_remove = N_("Would remove %s\n"); static const char *msg_skip_git_dir = N_("Skipping repository %s\n"); static const char *msg_would_skip_git_dir = N_("Would skip repository %s\n"); +#ifndef CAN_UNLINK_MOUNT_POINTS static const char *msg_skip_mount_point = N_("Skipping mount point %s\n"); static const char *msg_would_skip_mount_point = N_("Would skip mount point %s\n"); +#endif static const char *msg_warn_remove_failed = N_("failed to remove %s"); static const char *msg_warn_lstat_failed = N_("could not lstat %s\n"); @@ -174,13 +176,24 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, } if (is_mount_point(path)) { +#ifndef CAN_UNLINK_MOUNT_POINTS if (!quiet) { - quote_path_relative(path->buf, prefix, "ed); + quote_path(path->buf, prefix, "ed, 0); printf(dry_run ? _(msg_would_skip_mount_point) : _(msg_skip_mount_point), quoted.buf); } *dir_gone = 0; +#else + if (!dry_run && unlink(path->buf)) { + int saved_errno = errno; + quote_path(path->buf, prefix, "ed, 0); + errno = saved_errno; + warning_errno(_(msg_warn_remove_failed), quoted.buf); + *dir_gone = 0; + ret = -1; + } +#endif goto out; } diff --git a/compat/mingw.h b/compat/mingw.h index 5cc2ed3ff4..a93bf5ce02 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -445,6 +445,7 @@ static inline void convert_slashes(char *path) struct strbuf; int mingw_is_mount_point(struct strbuf *path); #define is_mount_point mingw_is_mount_point +#define CAN_UNLINK_MOUNT_POINTS 1 #define PATH_SEP ';' char *mingw_query_user_email(void); #define query_user_email mingw_query_user_email diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index 71f450d139..9584e0f3ac 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -752,6 +752,7 @@ test_expect_success MINGW 'clean does not traverse mount points' ' git init with-mountpoint && cmd //c "mklink /j with-mountpoint\\mountpoint target" && git -C with-mountpoint clean -dfx && + test_path_is_missing with-mountpoint/mountpoint && test_path_is_file target/dont-clean-me ' From a01bf5e434156111b8aa6b4cf1795c2d1a254c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlo=20Marcelo=20Arenas=20Bel=C3=B3n?= Date: Sat, 31 Aug 2019 06:26:10 -0700 Subject: [PATCH 17/72] config.mak.uname: PCRE1 cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no longer relevant after moving to PCRE2 Signed-off-by: Carlo Marcelo Arenas Belón --- config.mak.uname | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index 198ab1e58f..ebbc4ddaee 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -663,8 +663,7 @@ else HAVE_LIBCHARSET_H = YesPlease NO_GETTEXT = USE_GETTEXT_SCHEME = fallthrough - USE_LIBPCRE= YesPlease - NO_LIBPCRE1_JIT = UnfortunatelyYes + USE_LIBPCRE = YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease else From 4e83c5e94efe7eaf015834d98868c21e04281135 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 25 Oct 2019 01:02:33 +0200 Subject: [PATCH 18/72] config: normalize the path of the system gitconfig Git for Windows is compiled with a runtime prefix, and that runtime prefix is typically `C:/Program Files/Git/mingw64`. As we want the system gitconfig to live in the sibling directory `etc`, we define the relative path as `../etc/gitconfig`. However, as reported by Philip Oakley, the output of `git config --show-origin --system -l` looks rather ugly, as it shows the path as `file:C:/Program Files/Git/mingw64/../etc/gitconfig`, i.e. with the `mingw64/../` part. By normalizing the path, we get a prettier path. Signed-off-by: Johannes Schindelin --- config.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config.c b/config.c index 1137bd73af..29b5cbfd1a 100644 --- a/config.c +++ b/config.c @@ -1678,9 +1678,11 @@ static int git_config_from_blob_ref(config_fn_t fn, const char *git_etc_gitconfig(void) { - static const char *system_wide; - if (!system_wide) + static char *system_wide; + if (!system_wide) { system_wide = system_path(ETC_GITCONFIG); + normalize_path_copy(system_wide, system_wide); + } return system_wide; } From 75086bcb14bdbb1d14f85a8616dcbf2baf664bf2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 16 Feb 2015 14:06:59 +0100 Subject: [PATCH 19/72] mingw: include the Python parts in the build While Git for Windows does not _ship_ Python (in order to save on bandwidth), MSYS2 provides very fine Python interpreters that users can easily take advantage of, by using Git for Windows within its SDK. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 17e61bd626..d950b59c73 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -672,6 +672,7 @@ else NO_LIBPCRE1_JIT = UnfortunatelyYes NO_CURL = USE_NED_ALLOCATOR = YesPlease + NO_PYTHON = ifeq (/mingw64,$(subst 32,64,$(prefix))) # Move system config into top-level /etc/ ETC_GITCONFIG = ../etc/gitconfig From e3f1e9ef83c1fcb93501085ec987ed7b846664a6 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Thu, 8 May 2014 21:43:24 +0200 Subject: [PATCH 20/72] Config option to disable side-band-64k for transport Since commit 0c499ea60f the send-pack builtin uses the side-band-64k capability if advertised by the server. Unfortunately this breaks pushing over the dump git protocol if used over a network connection. The detailed reasons for this breakage are (by courtesy of Jeff Preshing, quoted from ttps://groups.google.com/d/msg/msysgit/at8D7J-h7mw/eaLujILGUWoJ): ---------------------------------------------------------------------------- MinGW wraps Windows sockets in CRT file descriptors in order to mimic the functionality of POSIX sockets. This causes msvcrt.dll to treat sockets as Installable File System (IFS) handles, calling ReadFile, WriteFile, DuplicateHandle and CloseHandle on them. This approach works well in simple cases on recent versions of Windows, but does not support all usage patterns. In particular, using this approach, any attempt to read & write concurrently on the same socket (from one or more processes) will deadlock in a scenario where the read waits for a response from the server which is only invoked after the write. This is what send_pack currently attempts to do in the use_sideband codepath. ---------------------------------------------------------------------------- The new config option "sendpack.sideband" allows to override the side-band-64k capability of the server, and thus makes the dump git protocol work. Other transportation methods like ssh and http/https still benefit from the sideband channel, therefore the default value of "sendpack.sideband" is still true. [jes: split out the documentation into Documentation/config/] Signed-off-by: Thomas Braun Signed-off-by: Johannes Schindelin Signed-off-by: Oliver Schneider --- Documentation/config.txt | 2 ++ Documentation/config/sendpack.txt | 5 +++++ send-pack.c | 13 ++++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 Documentation/config/sendpack.txt diff --git a/Documentation/config.txt b/Documentation/config.txt index 6ba50b1104..2c2dbd967e 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -440,6 +440,8 @@ include::config/reset.txt[] include::config/sendemail.txt[] +include::config/sendpack.txt[] + include::config/sequencer.txt[] include::config/showbranch.txt[] diff --git a/Documentation/config/sendpack.txt b/Documentation/config/sendpack.txt new file mode 100644 index 0000000000..e306f657fb --- /dev/null +++ b/Documentation/config/sendpack.txt @@ -0,0 +1,5 @@ +sendpack.sideband:: + Allows to disable the side-band-64k capability for send-pack even + when it is advertised by the server. Makes it possible to work + around a limitation in the git for windows implementation together + with the dump git protocol. Defaults to true. diff --git a/send-pack.c b/send-pack.c index 9045f8a082..55682f6558 100644 --- a/send-pack.c +++ b/send-pack.c @@ -39,6 +39,16 @@ int option_parse_push_signed(const struct option *opt, die("bad %s argument: %s", opt->long_name, arg); } +static int config_use_sideband = 1; + +static int send_pack_config(const char *var, const char *value, void *unused) +{ + if (!strcmp("sendpack.sideband", var)) + config_use_sideband = git_config_bool(var, value); + + return 0; +} + static void feed_object(const struct object_id *oid, FILE *fh, int negative) { if (negative && @@ -437,6 +447,7 @@ int send_pack(struct send_pack_args *args, const char *push_cert_nonce = NULL; struct packet_reader reader; + git_config(send_pack_config, NULL); git_config_get_bool("transfer.advertisesid", &advertise_sid); /* Does the other end support the reporting? */ @@ -448,7 +459,7 @@ int send_pack(struct send_pack_args *args, allow_deleting_refs = 1; if (server_supports("ofs-delta")) args->use_ofs_delta = 1; - if (server_supports("side-band-64k")) + if (config_use_sideband && server_supports("side-band-64k")) use_sideband = 1; if (server_supports("quiet")) quiet_supported = 1; From 7a783f6e55dbe047c71c5ed16fc151de25f4f4f8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 14 Nov 2019 20:09:23 +0100 Subject: [PATCH 21/72] mingw: make sure `errno` is set correctly when socket operations fail The winsock2 library provides functions that work on different data types than file descriptors, therefore we wrap them. But that is not the only difference: they also do not set `errno` but expect the callers to enquire about errors via `WSAGetLastError()`. Let's translate that into appropriate `errno` values whenever the socket operations fail so that Git's code base does not have to change its expectations. This closes https://github.com/git-for-windows/git/issues/2404 Helped-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/mingw.c | 157 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 147 insertions(+), 10 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index a43599841c..0b634478d1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1973,18 +1973,150 @@ static void ensure_socket_initialization(void) initialized = 1; } +static int winsock_error_to_errno(DWORD err) +{ + switch (err) { + case WSAEINTR: return EINTR; + case WSAEBADF: return EBADF; + case WSAEACCES: return EACCES; + case WSAEFAULT: return EFAULT; + case WSAEINVAL: return EINVAL; + case WSAEMFILE: return EMFILE; + case WSAEWOULDBLOCK: return EWOULDBLOCK; + case WSAEINPROGRESS: return EINPROGRESS; + case WSAEALREADY: return EALREADY; + case WSAENOTSOCK: return ENOTSOCK; + case WSAEDESTADDRREQ: return EDESTADDRREQ; + case WSAEMSGSIZE: return EMSGSIZE; + case WSAEPROTOTYPE: return EPROTOTYPE; + case WSAENOPROTOOPT: return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: return EOPNOTSUPP; + case WSAEAFNOSUPPORT: return EAFNOSUPPORT; + case WSAEADDRINUSE: return EADDRINUSE; + case WSAEADDRNOTAVAIL: return EADDRNOTAVAIL; + case WSAENETDOWN: return ENETDOWN; + case WSAENETUNREACH: return ENETUNREACH; + case WSAENETRESET: return ENETRESET; + case WSAECONNABORTED: return ECONNABORTED; + case WSAECONNRESET: return ECONNRESET; + case WSAENOBUFS: return ENOBUFS; + case WSAEISCONN: return EISCONN; + case WSAENOTCONN: return ENOTCONN; + case WSAETIMEDOUT: return ETIMEDOUT; + case WSAECONNREFUSED: return ECONNREFUSED; + case WSAELOOP: return ELOOP; + case WSAENAMETOOLONG: return ENAMETOOLONG; + case WSAEHOSTUNREACH: return EHOSTUNREACH; + case WSAENOTEMPTY: return ENOTEMPTY; + /* No errno equivalent; default to EIO */ + case WSAESOCKTNOSUPPORT: + case WSAEPFNOSUPPORT: + case WSAESHUTDOWN: + case WSAETOOMANYREFS: + case WSAEHOSTDOWN: + case WSAEPROCLIM: + case WSAEUSERS: + case WSAEDQUOT: + case WSAESTALE: + case WSAEREMOTE: + case WSASYSNOTREADY: + case WSAVERNOTSUPPORTED: + case WSANOTINITIALISED: + case WSAEDISCON: + case WSAENOMORE: + case WSAECANCELLED: + case WSAEINVALIDPROCTABLE: + case WSAEINVALIDPROVIDER: + case WSAEPROVIDERFAILEDINIT: + case WSASYSCALLFAILURE: + case WSASERVICE_NOT_FOUND: + case WSATYPE_NOT_FOUND: + case WSA_E_NO_MORE: + case WSA_E_CANCELLED: + case WSAEREFUSED: + case WSAHOST_NOT_FOUND: + case WSATRY_AGAIN: + case WSANO_RECOVERY: + case WSANO_DATA: + case WSA_QOS_RECEIVERS: + case WSA_QOS_SENDERS: + case WSA_QOS_NO_SENDERS: + case WSA_QOS_NO_RECEIVERS: + case WSA_QOS_REQUEST_CONFIRMED: + case WSA_QOS_ADMISSION_FAILURE: + case WSA_QOS_POLICY_FAILURE: + case WSA_QOS_BAD_STYLE: + case WSA_QOS_BAD_OBJECT: + case WSA_QOS_TRAFFIC_CTRL_ERROR: + case WSA_QOS_GENERIC_ERROR: + case WSA_QOS_ESERVICETYPE: + case WSA_QOS_EFLOWSPEC: + case WSA_QOS_EPROVSPECBUF: + case WSA_QOS_EFILTERSTYLE: + case WSA_QOS_EFILTERTYPE: + case WSA_QOS_EFILTERCOUNT: + case WSA_QOS_EOBJLENGTH: + case WSA_QOS_EFLOWCOUNT: +#ifndef _MSC_VER + case WSA_QOS_EUNKNOWNPSOBJ: +#endif + case WSA_QOS_EPOLICYOBJ: + case WSA_QOS_EFLOWDESC: + case WSA_QOS_EPSFLOWSPEC: + case WSA_QOS_EPSFILTERSPEC: + case WSA_QOS_ESDMODEOBJ: + case WSA_QOS_ESHAPERATEOBJ: + case WSA_QOS_RESERVED_PETYPE: + default: return EIO; + } +} + +/* + * On Windows, `errno` is a global macro to a function call. + * This makes it difficult to debug and single-step our mappings. + */ +static inline void set_wsa_errno(void) +{ + DWORD wsa = WSAGetLastError(); + int e = winsock_error_to_errno(wsa); + errno = e; + +#ifdef DEBUG_WSA_ERRNO + fprintf(stderr, "winsock error: %d -> %d\n", wsa, e); + fflush(stderr); +#endif +} + +static inline int winsock_return(int ret) +{ + if (ret < 0) + set_wsa_errno(); + + return ret; +} + +#define WINSOCK_RETURN(x) do { return winsock_return(x); } while (0) + #undef gethostname int mingw_gethostname(char *name, int namelen) { - ensure_socket_initialization(); - return gethostname(name, namelen); + ensure_socket_initialization(); + WINSOCK_RETURN(gethostname(name, namelen)); } #undef gethostbyname struct hostent *mingw_gethostbyname(const char *host) { + struct hostent *ret; + ensure_socket_initialization(); - return gethostbyname(host); + + ret = gethostbyname(host); + if (!ret) + set_wsa_errno(); + + return ret; } #undef getaddrinfo @@ -1992,7 +2124,7 @@ int mingw_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { ensure_socket_initialization(); - return getaddrinfo(node, service, hints, res); + WINSOCK_RETURN(getaddrinfo(node, service, hints, res)); } int mingw_socket(int domain, int type, int protocol) @@ -2012,7 +2144,7 @@ int mingw_socket(int domain, int type, int protocol) * in errno so that _if_ someone looks up the code somewhere, * then it is at least the number that are usually listed. */ - errno = WSAGetLastError(); + set_wsa_errno(); return -1; } /* convert into a file descriptor */ @@ -2028,35 +2160,35 @@ int mingw_socket(int domain, int type, int protocol) int mingw_connect(int sockfd, struct sockaddr *sa, size_t sz) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return connect(s, sa, sz); + WINSOCK_RETURN(connect(s, sa, sz)); } #undef bind int mingw_bind(int sockfd, struct sockaddr *sa, size_t sz) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return bind(s, sa, sz); + WINSOCK_RETURN(bind(s, sa, sz)); } #undef setsockopt int mingw_setsockopt(int sockfd, int lvl, int optname, void *optval, int optlen) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return setsockopt(s, lvl, optname, (const char*)optval, optlen); + WINSOCK_RETURN(setsockopt(s, lvl, optname, (const char*)optval, optlen)); } #undef shutdown int mingw_shutdown(int sockfd, int how) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return shutdown(s, how); + WINSOCK_RETURN(shutdown(s, how)); } #undef listen int mingw_listen(int sockfd, int backlog) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return listen(s, backlog); + WINSOCK_RETURN(listen(s, backlog)); } #undef accept @@ -2067,6 +2199,11 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) SOCKET s1 = (SOCKET)_get_osfhandle(sockfd1); SOCKET s2 = accept(s1, sa, sz); + if (s2 == INVALID_SOCKET) { + set_wsa_errno(); + return -1; + } + /* convert into a file descriptor */ if ((sockfd2 = _open_osfhandle(s2, O_RDWR|O_BINARY)) < 0) { int err = errno; From be6b1e250c19edd0c9d3733f0f5d4f111a6eaa3d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 1 Jan 2020 21:07:22 +0100 Subject: [PATCH 22/72] mingw: do resolve symlinks in `getcwd()` As pointed out in https://github.com/git-for-windows/git/issues/1676, the `git rev-parse --is-inside-work-tree` command currently fails when the current directory's path contains symbolic links. The underlying reason for this bug is that `getcwd()` is supposed to resolve symbolic links, but our `mingw_getcwd()` implementation did not. We do have all the building blocks for that, though: the `GetFinalPathByHandleW()` function will resolve symbolic links. However, we only called that function if `GetLongPathNameW()` failed, for historical reasons: the latter function was supported for a long time, but the former API function was introduced only with Windows Vista, and we used to support also Windows XP. With that support having been dropped, we are free to call the symbolic link-resolving function right away. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index a43599841c..aa2abca0a7 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1078,18 +1078,16 @@ char *mingw_getcwd(char *pointer, int len) { wchar_t cwd[MAX_PATH], wpointer[MAX_PATH]; DWORD ret = GetCurrentDirectoryW(ARRAY_SIZE(cwd), cwd); + HANDLE hnd; if (!ret || ret >= ARRAY_SIZE(cwd)) { errno = ret ? ENAMETOOLONG : err_win_to_posix(GetLastError()); return NULL; } - ret = GetLongPathNameW(cwd, wpointer, ARRAY_SIZE(wpointer)); - if (!ret && GetLastError() == ERROR_ACCESS_DENIED) { - HANDLE hnd = CreateFileW(cwd, 0, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, - OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - if (hnd == INVALID_HANDLE_VALUE) - return NULL; + hnd = CreateFileW(cwd, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd != INVALID_HANDLE_VALUE) { ret = GetFinalPathNameByHandleW(hnd, wpointer, ARRAY_SIZE(wpointer), 0); CloseHandle(hnd); if (!ret || ret >= ARRAY_SIZE(wpointer)) @@ -1098,9 +1096,7 @@ char *mingw_getcwd(char *pointer, int len) return NULL; return pointer; } - if (!ret || ret >= ARRAY_SIZE(wpointer)) - return NULL; - if (xwcstoutf(pointer, wpointer, len) < 0) + if (xwcstoutf(pointer, cwd, len) < 0) return NULL; convert_slashes(pointer); return pointer; From b91cc7a3aee44f266cd2ddfb7554881b44f7338e Mon Sep 17 00:00:00 2001 From: Bjoern Mueller Date: Wed, 22 Jan 2020 13:49:13 +0100 Subject: [PATCH 23/72] mingw: fix fatal error working on mapped network drives on Windows In 1e64d18 (mingw: do resolve symlinks in `getcwd()`) a problem was introduced that causes git for Windows to stop working with certain mapped network drives (in particular, drives that are mapped to locations with long path names). Error message was "fatal: Unable to read current working directory: No such file or directory". Present change fixes this issue as discussed in https://github.com/git-for-windows/git/issues/2480 Signed-off-by: Bjoern Mueller --- compat/mingw.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index a43599841c..b49f7a0e7d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1092,8 +1092,13 @@ char *mingw_getcwd(char *pointer, int len) return NULL; ret = GetFinalPathNameByHandleW(hnd, wpointer, ARRAY_SIZE(wpointer), 0); CloseHandle(hnd); - if (!ret || ret >= ARRAY_SIZE(wpointer)) - return NULL; + if (!ret || ret >= ARRAY_SIZE(wpointer)) { + ret = GetLongPathNameW(cwd, wpointer, ARRAY_SIZE(wpointer)); + if (!ret || ret >= ARRAY_SIZE(wpointer)) { + errno = ret ? ENAMETOOLONG : err_win_to_posix(GetLastError()); + return NULL; + } + } if (xwcstoutf(pointer, normalize_ntpath(wpointer), len) < 0) return NULL; return pointer; From f198100c32cbc886809b172f7b72be46fdd6f08c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 30 Jan 2020 14:22:27 -0500 Subject: [PATCH 24/72] clink.pl: fix MSVC compile script to handle libcurl-d.lib Update clink.pl to link with either libcurl.lib or libcurl-d.lib depending on whether DEBUG=1 is set. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/vcbuild/scripts/clink.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 3bd824154b..c4c99d1a11 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -56,7 +56,8 @@ while (@ARGV) { # need to use that instead? foreach my $flag (@lflags) { if ($flag =~ /^-LIBPATH:(.*)/) { - foreach my $l ("libcurl_imp.lib", "libcurl.lib") { + my $libcurl = $is_debug ? "libcurl-d.lib" : "libcurl.lib"; + foreach my $l ("libcurl_imp.lib", $libcurl) { if (-f "$1/$l") { $lib = $l; last; From 61c2424bf33d2413e9b61ea76261ddeb9e841792 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 Feb 2017 13:28:58 +0100 Subject: [PATCH 25/72] mingw: ensure valid CTYPE A change between versions 2.4.1 and 2.6.0 of the MSYS2 runtime modified how Cygwin's runtime (and hence Git for Windows' MSYS2 runtime derivative) handles locales: d16a56306d (Consolidate wctomb/mbtowc calls for POSIX-1.2008, 2016-07-20). An unintended side-effect is that "cold-calling" into the POSIX emulation will start with a locale based on the current code page, something that Git for Windows is very ill-prepared for, as it expects to be able to pass a command-line containing non-ASCII characters to the shell without having those characters munged. One symptom of this behavior: when `git clone` or `git fetch` shell out to call `git-upload-pack` with a path that contains non-ASCII characters, the shell tried to interpret the entire command-line (including command-line parameters) as executable path, which obviously must fail. This fixes https://github.com/git-for-windows/git/issues/1036 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index a43599841c..dde400ac69 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2597,6 +2597,9 @@ static void setup_windows_environment(void) if (!tmp && (tmp = getenv("USERPROFILE"))) setenv("HOME", tmp, 1); } + + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) + setenv("LC_CTYPE", "C.UTF-8", 1); } int is_valid_win32_path(const char *path, int allow_literal_nul) From 600c5dcf80969b3cb1cf3c16f6c19b34e8e6b79d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Jan 2020 12:02:47 +0100 Subject: [PATCH 26/72] mingw: demonstrate a `git add` issue with NTFS junctions NTFS junctions are somewhat similar in spirit to Unix bind mounts: they point to a different directory and are resolved by the filesystem driver. As such, they appear to `lstat()` as if they are directories, not as if they are symbolic links. _Any_ user can create junctions, while symbolic links can only be created by non-administrators in Developer Mode on Windows 10. Hence NTFS junctions are much more common "in the wild" than NTFS symbolic links. It was reported in https://github.com/git-for-windows/git/issues/2481 that adding files via an absolute path that traverses an NTFS junction: since 1e64d18 (mingw: do resolve symlinks in `getcwd()`), we resolve not only symbolic links but also NTFS junctions when determining the absolute path of the current directory. The same is not true for `git add `, where symbolic links are resolved in ``, but not NTFS junctions. Signed-off-by: Johannes Schindelin --- t/t3700-add.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t3700-add.sh b/t/t3700-add.sh index b7d4ba608c..46593d0364 100755 --- a/t/t3700-add.sh +++ b/t/t3700-add.sh @@ -423,4 +423,15 @@ test_expect_success CASE_INSENSITIVE_FS 'path is case-insensitive' ' git add "$downcased" ' +test_expect_failure MINGW 'can add files via NTFS junctions' ' + test_when_finished "cmd //c rmdir junction && rm -rf target" && + test_create_repo target && + cmd //c "mklink /j junction target" && + >target/via-junction && + git -C junction add "$(pwd)/junction/via-junction" && + echo via-junction >expect && + git -C target diff --cached --name-only >actual && + test_cmp expect actual +' + test_done From dc451210b4c113b3820d82cabd633be5e5e20b9e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 1 Feb 2020 00:31:16 +0100 Subject: [PATCH 27/72] mingw: allow `git.exe` to be used instead of the "Git wrapper" Git for Windows wants to add `git.exe` to the users' `PATH`, without cluttering the latter with unnecessary executables such as `wish.exe`. To that end, it invented the concept of its "Git wrapper", i.e. a tiny executable located in `C:\Program Files\Git\cmd\git.exe` (originally a CMD script) whose sole purpose is to set up a couple of environment variables and then spawn the _actual_ `git.exe` (which nowadays lives in `C:\Program Files\Git\mingw64\bin\git.exe` for 64-bit, and the obvious equivalent for 32-bit installations). Currently, the following environment variables are set unless already initialized: - `MSYSTEM`, to make sure that the MSYS2 Bash and the MSYS2 Perl interpreter behave as expected, and - `PLINK_PROTOCOL`, to force PuTTY's `plink.exe` to use the SSH protocol instead of Telnet, - `PATH`, to make sure that the `bin` folder in the user's home directory, as well as the `/mingw64/bin` and the `/usr/bin` directories are included. The trick here is that the `/mingw64/bin/` and `/usr/bin/` directories are relative to the top-level installation directory of Git for Windows (which the included Bash interprets as `/`, i.e. as the MSYS pseudo root directory). Using the absence of `MSYSTEM` as a tell-tale, we can detect in `git.exe` whether these environment variables have been initialized properly. Therefore we can call `C:\Program Files\Git\mingw64\bin\git` in-place after this change, without having to call Git through the Git wrapper. Obviously, above-mentioned directories must be _prepended_ to the `PATH` variable, otherwise we risk picking up executables from unrelated Git installations. We do that by constructing the new `PATH` value from scratch, appending `$HOME/bin` (if `HOME` is set), then the MSYS2 system directories, and then appending the original `PATH`. Side note: this modification of the `PATH` variable is independent of the modification necessary to reach the executables and scripts in `/mingw64/libexec/git-core/`, i.e. the `GIT_EXEC_PATH`. That modification is still performed by Git, elsewhere, long after making the changes described above. While we _still_ cannot simply hard-link `mingw64\bin\git.exe` to `cmd` (because the former depends on a couple of `.dll` files that are only in `mingw64\bin`, i.e. calling `...\cmd\git.exe` would fail to load due to missing dependencies), at least we can now avoid that extra process of running the Git wrapper (which then has to wait for the spawned `git.exe` to finish) by calling `...\mingw64\bin\git.exe` directly, via its absolute path. Testing this is in Git's test suite tricky: we set up a "new" MSYS pseudo-root and copy the `git.exe` file into the appropriate location, then verify that `MSYSTEM` is set properly, and also that the `PATH` is modified so that scripts can be found in `$HOME/bin`, `/mingw64/bin/` and `/usr/bin/`. This addresses https://github.com/git-for-windows/git/issues/2283 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 69 +++++++++++++++++++++++++++++++++++++++++++ config.mak.uname | 4 +-- t/t0060-path-utils.sh | 21 +++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index dde400ac69..e91ba01b6f 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2546,6 +2546,47 @@ int xwcstoutf(char *utf, const wchar_t *wcs, size_t utflen) return -1; } +#ifdef ENSURE_MSYSTEM_IS_SET +static size_t append_system_bin_dirs(char *path, size_t size) +{ +#if !defined(RUNTIME_PREFIX) || !defined(HAVE_WPGMPTR) + return 0; +#else + char prefix[32768]; + const char *slash; + size_t len = xwcstoutf(prefix, _wpgmptr, sizeof(prefix)), off = 0; + + if (len == 0 || len >= sizeof(prefix) || + !(slash = find_last_dir_sep(prefix))) + return 0; + /* strip trailing `git.exe` */ + len = slash - prefix; + + /* strip trailing `cmd` or `mingw64\bin` or `mingw32\bin` or `bin` or `libexec\git-core` */ + if (strip_suffix_mem(prefix, &len, "\\mingw64\\libexec\\git-core") || + strip_suffix_mem(prefix, &len, "\\mingw64\\bin")) + off += xsnprintf(path + off, size - off, + "%.*s\\mingw64\\bin;", (int)len, prefix); + else if (strip_suffix_mem(prefix, &len, "\\mingw32\\libexec\\git-core") || + strip_suffix_mem(prefix, &len, "\\mingw32\\bin")) + off += xsnprintf(path + off, size - off, + "%.*s\\mingw32\\bin;", (int)len, prefix); + else if (strip_suffix_mem(prefix, &len, "\\cmd") || + strip_suffix_mem(prefix, &len, "\\bin") || + strip_suffix_mem(prefix, &len, "\\libexec\\git-core")) + off += xsnprintf(path + off, size - off, + "%.*s\\mingw%d\\bin;", (int)len, prefix, + (int)(sizeof(void *) * 8)); + else + return 0; + + off += xsnprintf(path + off, size - off, + "%.*s\\usr\\bin;", (int)len, prefix); + return off; +#endif +} +#endif + static void setup_windows_environment(void) { char *tmp = getenv("TMPDIR"); @@ -2598,6 +2639,34 @@ static void setup_windows_environment(void) setenv("HOME", tmp, 1); } + if (!getenv("PLINK_PROTOCOL")) + setenv("PLINK_PROTOCOL", "ssh", 0); + +#ifdef ENSURE_MSYSTEM_IS_SET + if (!(tmp = getenv("MSYSTEM")) || !tmp[0]) { + const char *home = getenv("HOME"), *path = getenv("PATH"); + char buf[32768]; + size_t off = 0; + + xsnprintf(buf, sizeof(buf), + "MINGW%d", (int)(sizeof(void *) * 8)); + setenv("MSYSTEM", buf, 1); + + if (home) + off += xsnprintf(buf + off, sizeof(buf) - off, + "%s\\bin;", home); + off += append_system_bin_dirs(buf + off, sizeof(buf) - off); + if (path) + off += xsnprintf(buf + off, sizeof(buf) - off, + "%s", path); + else if (off > 0) + buf[off - 1] = '\0'; + else + buf[0] = '\0'; + setenv("PATH", buf, 1); + } +#endif + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) setenv("LC_CTYPE", "C.UTF-8", 1); } diff --git a/config.mak.uname b/config.mak.uname index 198ab1e58f..7c07d3b15f 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -443,7 +443,7 @@ ifeq ($(uname_S),Windows) compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/trace2_win32_process_info.o \ compat/win32/dirent.o - COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" + COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DENSURE_MSYSTEM_IS_SET -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -ENTRY:wmainCRTStartup -SUBSYSTEM:CONSOLE # invalidcontinue.obj allows Git's source code to close the same file # handle twice, or to access the osfhandle of an already-closed stdout @@ -656,7 +656,7 @@ else endif CC = gcc COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \ - -fstack-protector-strong + -DENSURE_MSYSTEM_IS_SET -fstack-protector-strong EXTLIBS += -lntdll INSTALL = /bin/install INTERNAL_QSORT = YesPlease diff --git a/t/t0060-path-utils.sh b/t/t0060-path-utils.sh index 56db5c8aba..4820c6834a 100755 --- a/t/t0060-path-utils.sh +++ b/t/t0060-path-utils.sh @@ -495,4 +495,25 @@ test_expect_success MINGW 'is_valid_path() on Windows' ' "PRN./abc" ' +test_expect_success MINGW 'MSYSTEM/PATH is adjusted if necessary' ' + mkdir -p "$HOME"/bin pretend/mingw64/bin \ + pretend/mingw64/libexec/git-core pretend/usr/bin && + cp "$GIT_EXEC_PATH"/git.exe pretend/mingw64/bin/ && + cp "$GIT_EXEC_PATH"/git.exe pretend/mingw64/libexec/git-core/ && + echo "env | grep MSYSTEM=" | write_script "$HOME"/bin/git-test-home && + echo "echo mingw64" | write_script pretend/mingw64/bin/git-test-bin && + echo "echo usr" | write_script pretend/usr/bin/git-test-bin2 && + + ( + MSYSTEM= && + GIT_EXEC_PATH= && + pretend/mingw64/libexec/git-core/git.exe test-home >actual && + pretend/mingw64/libexec/git-core/git.exe test-bin >>actual && + pretend/mingw64/bin/git.exe test-bin2 >>actual + ) && + test_write_lines MSYSTEM=$MSYSTEM mingw64 usr >expect && + test_cmp expect actual +' +' + test_done From 2f7e629293de7d5120dc5ddbe19caa31b95d67f6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Jan 2020 11:44:31 +0100 Subject: [PATCH 28/72] strbuf_realpath(): use platform-dependent API if available Some platforms (e.g. Windows) provide API functions to resolve paths much quicker. Let's offer a way to short-cut `strbuf_realpath()` on those platforms. Signed-off-by: Johannes Schindelin --- abspath.c | 3 +++ git-compat-util.h | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/abspath.c b/abspath.c index 6f15a418bb..8a14e61c2c 100644 --- a/abspath.c +++ b/abspath.c @@ -95,6 +95,9 @@ char *strbuf_realpath(struct strbuf *resolved, const char *path, goto error_out; } + if (platform_strbuf_realpath(resolved, path)) + return resolved->buf; + strbuf_addstr(&remaining, path); get_root_part(resolved, &remaining); diff --git a/git-compat-util.h b/git-compat-util.h index 7d3db43f11..811e2509be 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -410,6 +410,10 @@ static inline int git_has_dir_sep(const char *path) #define query_user_email() NULL #endif +#ifndef platform_strbuf_realpath +#define platform_strbuf_realpath(resolved, path) NULL +#endif + #ifdef __TANDEM #include #include From 9e686f82a739aff06362f221b2b39208843cdea8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Aug 2020 12:13:26 +0200 Subject: [PATCH 29/72] mingw: ignore HOMEDRIVE/HOMEPATH if it points to Windows' system directory Internally, Git expects the environment variable `HOME` to be set, and to point to the current user's home directory. This environment variable is not set by default on Windows, and therefore Git tries its best to construct one if it finds `HOME` unset. There are actually two different approaches Git tries: first, it looks at `HOMEDRIVE`/`HOMEPATH` because this is widely used in corporate environments with roaming profiles, and a user generally wants their global Git settings to be in a roaming profile. Only when `HOMEDRIVE`/`HOMEPATH` is either unset or does not point to a valid location, Git will fall back to using `USERPROFILE` instead. However, starting with Windows Vista, for secondary logons and services, the environment variables `HOMEDRIVE`/`HOMEPATH` point to Windows' system directory (usually `C:\Windows\system32`). That is undesirable, and that location is usually write-protected anyway. So let's verify that the `HOMEDRIVE`/`HOMEPATH` combo does not point to Windows' system directory before using it, falling back to `USERPROFILE` if it does. This fixes git-for-windows#2709 Initial-Path-by: Ivan Pozdeev Signed-off-by: Johannes Schindelin --- compat/mingw.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index e91ba01b6f..d8eec38f68 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2587,6 +2587,18 @@ static size_t append_system_bin_dirs(char *path, size_t size) } #endif +static int is_system32_path(const char *path) +{ + WCHAR system32[MAX_LONG_PATH], wpath[MAX_LONG_PATH]; + + if (xutftowcs_long_path(wpath, path) < 0 || + !GetSystemDirectoryW(system32, ARRAY_SIZE(system32)) || + _wcsicmp(system32, wpath)) + return 0; + + return 1; +} + static void setup_windows_environment(void) { char *tmp = getenv("TMPDIR"); @@ -2627,7 +2639,8 @@ static void setup_windows_environment(void) strbuf_addstr(&buf, tmp); if ((tmp = getenv("HOMEPATH"))) { strbuf_addstr(&buf, tmp); - if (is_directory(buf.buf)) + if (!is_system32_path(buf.buf) && + is_directory(buf.buf)) setenv("HOME", buf.buf, 1); else tmp = NULL; /* use $USERPROFILE */ From d413e8216e4bc88dd9e41c6e2d89bd48711932fd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Jan 2020 11:49:04 +0100 Subject: [PATCH 30/72] mingw: implement a platform-specific `strbuf_realpath()` There is a Win32 API function to resolve symbolic links, and we can use that instead of resolving them manually. Even better, this function also resolves NTFS junction points (which are somewhat similar to bind mounts). This fixes https://github.com/git-for-windows/git/issues/2481. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 2 ++ t/t3700-add.sh | 2 +- t/t5601-clone.sh | 7 ++++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index a43599841c..20b6d60337 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1074,6 +1074,69 @@ struct tm *localtime_r(const time_t *timep, struct tm *result) return NULL; } +char *mingw_strbuf_realpath(struct strbuf *resolved, const char *path) +{ + wchar_t wpath[MAX_PATH]; + HANDLE h; + DWORD ret; + int len; + const char *last_component = NULL; + + if (xutftowcs_path(wpath, path) < 0) + return NULL; + + h = CreateFileW(wpath, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + /* + * strbuf_realpath() allows the last path component to not exist. If + * that is the case, now it's time to try without last component. + */ + if (h == INVALID_HANDLE_VALUE && + GetLastError() == ERROR_FILE_NOT_FOUND) { + /* cut last component off of `wpath` */ + wchar_t *p = wpath + wcslen(wpath); + + while (p != wpath) + if (*(--p) == L'/' || *p == L'\\') + break; /* found start of last component */ + + if (p != wpath && (last_component = find_last_dir_sep(path))) { + last_component++; /* skip directory separator */ + *p = L'\0'; + h = CreateFileW(wpath, 0, FILE_SHARE_READ | + FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL); + } + } + + if (h == INVALID_HANDLE_VALUE) + return NULL; + + ret = GetFinalPathNameByHandleW(h, wpath, ARRAY_SIZE(wpath), 0); + CloseHandle(h); + if (!ret || ret >= ARRAY_SIZE(wpath)) + return NULL; + + len = wcslen(wpath) * 3; + strbuf_grow(resolved, len); + len = xwcstoutf(resolved->buf, normalize_ntpath(wpath), len); + if (len < 0) + return NULL; + resolved->len = len; + + if (last_component) { + /* Use forward-slash, like `normalize_ntpath()` */ + strbuf_addch(resolved, '/'); + strbuf_addstr(resolved, last_component); + } + + return resolved->buf; + +} + char *mingw_getcwd(char *pointer, int len) { wchar_t cwd[MAX_PATH], wpointer[MAX_PATH]; diff --git a/compat/mingw.h b/compat/mingw.h index af8eddd73e..7f043cd29e 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -445,6 +445,8 @@ static inline void convert_slashes(char *path) #define PATH_SEP ';' char *mingw_query_user_email(void); #define query_user_email mingw_query_user_email +char *mingw_strbuf_realpath(struct strbuf *resolved, const char *path); +#define platform_strbuf_realpath mingw_strbuf_realpath #if !defined(__MINGW64_VERSION_MAJOR) && (!defined(_MSC_VER) || _MSC_VER < 1800) #define PRIuMAX "I64u" #define PRId64 "I64d" diff --git a/t/t3700-add.sh b/t/t3700-add.sh index 46593d0364..d12aebd233 100755 --- a/t/t3700-add.sh +++ b/t/t3700-add.sh @@ -423,7 +423,7 @@ test_expect_success CASE_INSENSITIVE_FS 'path is case-insensitive' ' git add "$downcased" ' -test_expect_failure MINGW 'can add files via NTFS junctions' ' +test_expect_success MINGW 'can add files via NTFS junctions' ' test_when_finished "cmd //c rmdir junction && rm -rf target" && test_create_repo target && cmd //c "mklink /j junction target" && diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 7df3c5373a..34c8d36fc7 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -68,6 +68,13 @@ test_expect_success 'clone respects GIT_WORK_TREE' ' ' +test_expect_success CASE_INSENSITIVE_FS 'core.worktree is not added due to path case' ' + + mkdir UPPERCASE && + git clone src "$(pwd)/uppercase" && + test "unset" = "$(git -C UPPERCASE config --default unset core.worktree)" +' + test_expect_success 'clone from hooks' ' test_create_repo r0 && From 4e5433467458ba141d4b3f18ba8f2b9f9123293a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 5 Feb 2020 13:01:33 +0100 Subject: [PATCH 31/72] tests: exercise the RUNTIME_PREFIX feature Originally, we refrained from adding a regression test in 7b6c6496374 (system_path(): Add prefix computation at runtime if RUNTIME_PREFIX set, 2008-08-10), and in 226c0ddd0d6 (exec_cmd: RUNTIME_PREFIX on some POSIX systems, 2018-04-10). The reason was that it was deemed too tricky to test. Turns out that it is not tricky to test at all: we simply create a pseudo-root, copy the `git` executable into the `git/` subdirectory of that pseudo-root, then copy a script into the `libexec/git-core/` directory and expect that to be picked up. As long as the trash directory is in a location where binaries can be executed, this works. Signed-off-by: Johannes Schindelin --- Makefile | 5 +++++ t/t0060-path-utils.sh | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/Makefile b/Makefile index 4128b457e1..7bbc32cc9e 100644 --- a/Makefile +++ b/Makefile @@ -2772,6 +2772,11 @@ ifdef GIT_TEST_INDEX_VERSION endif ifdef GIT_TEST_PERL_FATAL_WARNINGS @echo GIT_TEST_PERL_FATAL_WARNINGS=\''$(subst ','\'',$(subst ','\'',$(GIT_TEST_PERL_FATAL_WARNINGS)))'\' >>$@+ +endif +ifdef RUNTIME_PREFIX + @echo RUNTIME_PREFIX=\'true\' >>$@+ +else + @echo RUNTIME_PREFIX=\'false\' >>$@+ endif @if cmp $@+ $@ >/dev/null 2>&1; then $(RM) $@+; else mv $@+ $@; fi diff --git a/t/t0060-path-utils.sh b/t/t0060-path-utils.sh index 4820c6834a..ada3e5b72d 100755 --- a/t/t0060-path-utils.sh +++ b/t/t0060-path-utils.sh @@ -514,6 +514,23 @@ test_expect_success MINGW 'MSYSTEM/PATH is adjusted if necessary' ' test_write_lines MSYSTEM=$MSYSTEM mingw64 usr >expect && test_cmp expect actual ' + +test_lazy_prereq RUNTIME_PREFIX ' + test true = "$RUNTIME_PREFIX" +' + +test_lazy_prereq CAN_EXEC_IN_PWD ' + cp "$GIT_EXEC_PATH"/git$X ./ && + ./git rev-parse +' + +test_expect_success RUNTIME_PREFIX,CAN_EXEC_IN_PWD 'RUNTIME_PREFIX works' ' + mkdir -p pretend/git pretend/libexec/git-core && + echo "echo HERE" | write_script pretend/libexec/git-core/git-here && + cp "$GIT_EXEC_PATH"/git$X pretend/git/ && + GIT_EXEC_PATH= ./pretend/git/git here >actual && + echo HERE >expect && + test_cmp expect actual ' test_done From b087eac6acb01ca72a8cef79de56bc043b35dcc1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 4 Mar 2020 21:55:28 +0100 Subject: [PATCH 32/72] http: use new "best effort" strategy for Secure Channel revoke checking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native Windows HTTPS backend is based on Secure Channel which lets the caller decide how to handle revocation checking problems caused by missing information in the certificate or offline CRL distribution points. Unfortunately, cURL chose to handle these problems differently than OpenSSL by default: while OpenSSL happily ignores those problems (essentially saying "¯\_(ツ)_/¯"), the Secure Channel backend will error out instead. As a remedy, the "no revoke" mode was introduced, which turns off revocation checking altogether. This is a bit heavy-handed. We support this via the `http.schannelCheckRevoke` setting. In https://github.com/curl/curl/pull/4981, we contributed an opt-in "best effort" strategy that emulates what OpenSSL seems to do. In Git for Windows, we actually want this to be the default. This patch makes it so, introducing it as a new value for the `http.schannelCheckRevoke" setting, which now becmes a tristate: it accepts the values "false", "true" or "best-effort" (defaulting to the last one). Signed-off-by: Johannes Schindelin --- Documentation/config/http.txt | 12 +++++++----- http.c | 26 ++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/Documentation/config/http.txt b/Documentation/config/http.txt index 7003661c0d..24f5187939 100644 --- a/Documentation/config/http.txt +++ b/Documentation/config/http.txt @@ -173,11 +173,13 @@ http.sslBackend:: http.schannelCheckRevoke:: Used to enforce or disable certificate revocation checks in cURL - when http.sslBackend is set to "schannel". Defaults to `true` if - unset. Only necessary to disable this if Git consistently errors - and the message is about checking the revocation status of a - certificate. This option is ignored if cURL lacks support for - setting the relevant SSL option at runtime. + when http.sslBackend is set to "schannel" via "true" and "false", + respectively. Another accepted value is "best-effort" (the default) + in which case revocation checks are performed, but errors due to + revocation list distribution points that are offline are silently + ignored, as well as errors due to certificates missing revocation + list distribution points. This option is ignored if cURL lacks + support for setting the relevant SSL option at runtime. http.schannelUseSSLCAInfo:: As of cURL v7.60.0, the Secure Channel backend can use the diff --git a/http.c b/http.c index 8b23a546af..2dd616aab6 100644 --- a/http.c +++ b/http.c @@ -165,7 +165,13 @@ static char *cached_accept_language; static char *http_ssl_backend; -static int http_schannel_check_revoke = 1; +static int http_schannel_check_revoke_mode = +#ifdef CURLSSLOPT_REVOKE_BEST_EFFORT + CURLSSLOPT_REVOKE_BEST_EFFORT; +#else + CURLSSLOPT_NO_REVOKE; +#endif + /* * With the backend being set to `schannel`, setting sslCAinfo would override * the Certificate Store in cURL v7.60.0 and later, which is not what we want @@ -330,7 +336,19 @@ static int http_options(const char *var, const char *value, void *cb) } if (!strcmp("http.schannelcheckrevoke", var)) { - http_schannel_check_revoke = git_config_bool(var, value); + if (value && !strcmp(value, "best-effort")) { + http_schannel_check_revoke_mode = +#ifdef CURLSSLOPT_REVOKE_BEST_EFFORT + CURLSSLOPT_REVOKE_BEST_EFFORT; +#else + CURLSSLOPT_NO_REVOKE; + warning(_("%s=%s unsupported by current cURL"), + var, value); +#endif + } else + http_schannel_check_revoke_mode = + (git_config_bool(var, value) ? + 0 : CURLSSLOPT_NO_REVOKE); return 0; } @@ -903,9 +921,9 @@ static CURL *get_curl_handle(void) #endif if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) && - !http_schannel_check_revoke) { + http_schannel_check_revoke_mode) { #if LIBCURL_VERSION_NUM >= 0x072c00 - curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE); + curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, http_schannel_check_revoke_mode); #else warning(_("CURLSSLOPT_NO_REVOKE not supported with cURL < 7.44.0")); #endif From f544b28ee7c4d9fd7c8239ac69573d0ae50a3063 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 9 May 2020 14:08:36 +0200 Subject: [PATCH 33/72] vcxproj: unclash project directories with build outputs It already caused problems with the test suite that the directory containing `git.vcxproj` is called the same as the Git executable without its file extension: `./git` is ambiguous, it could refer both to the directory `git/` as well as to `git.exe`. Now there is one more problem: when our GitHub workflow runs on the `vs/master` branch, it fails in all but the Windows builds, as they want to write the file `git` but there is already a directory in the way. Let's just go ahead and append `.proj` to all of those directories, e.g. `git.proj/` instead of `git/`. Signed-off-by: Johannes Schindelin --- config.mak.uname | 8 ++++---- contrib/buildsystems/Generators/Vcxproj.pm | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index 198ab1e58f..9a7cf45459 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -697,7 +697,7 @@ vcxproj: # Make .vcxproj files and add them unset QUIET_GEN QUIET_BUILT_IN; \ perl contrib/buildsystems/generate -g Vcxproj - git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj + git add -f git.sln {*,*/lib.proj,t/helper/*}/*.vcxproj # Generate the LinkOrCopyBuiltins.targets and LinkOrCopyRemoteHttp.targets file (echo '' && \ @@ -707,7 +707,7 @@ vcxproj: echo ' '; \ done && \ echo ' ' && \ - echo '') >git/LinkOrCopyBuiltins.targets + echo '') >git.proj/LinkOrCopyBuiltins.targets (echo '' && \ echo ' ' && \ for name in $(REMOTE_CURL_ALIASES); \ @@ -715,8 +715,8 @@ vcxproj: echo ' '; \ done && \ echo ' ' && \ - echo '') >git-remote-http/LinkOrCopyRemoteHttp.targets - git add -f git/LinkOrCopyBuiltins.targets git-remote-http/LinkOrCopyRemoteHttp.targets + echo '') >git-remote-http.proj/LinkOrCopyRemoteHttp.targets + git add -f git.proj/LinkOrCopyBuiltins.targets git-remote-http.proj/LinkOrCopyRemoteHttp.targets # Add command-list.h and config-list.h $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 config-list.h command-list.h diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index d2584450ba..dd1a20e91b 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -58,8 +58,8 @@ sub createProject { my $uuid = generate_guid($name); $$build_structure{"$prefix${target}_GUID"} = $uuid; my $vcxproj = $target; - $vcxproj =~ s/(.*\/)?(.*)/$&\/$2.vcxproj/; - $vcxproj =~ s/([^\/]*)(\/lib)\/(lib.vcxproj)/$1$2\/$1_$3/; + $vcxproj =~ s/(.*\/)?(.*)/$&.proj\/$2.vcxproj/; + $vcxproj =~ s/([^\/]*)(\/lib\.proj)\/(lib.vcxproj)/$1$2\/$1_$3/; $$build_structure{"$prefix${target}_VCXPROJ"} = $vcxproj; my @srcs = sort(map("$rel_dir\\$_", @{$$build_structure{"$prefix${name}_SOURCES"}})); @@ -89,7 +89,9 @@ sub createProject { $defines =~ s/>/>/g; $defines =~ s/\'//g; - die "Could not create the directory $target for $label project!\n" unless (-d "$target" || mkdir "$target"); + my $dir = $vcxproj; + $dir =~ s/\/[^\/]*$//; + die "Could not create the directory $dir for $label project!\n" unless (-d "$dir" || mkdir "$dir"); open F, ">$vcxproj" or die "Could not open $vcxproj for writing!\n"; binmode F, ":crlf :utf8"; @@ -236,14 +238,14 @@ EOM print F << "EOM"; - + $uuid_libgit false EOM if (!($name =~ 'xdiff')) { print F << "EOM"; - + $uuid_xdiff_lib false @@ -252,7 +254,7 @@ EOM if ($name =~ /(test-(line-buffer|svn-fe)|^git-remote-testsvn)\.exe$/) { my $uuid_vcs_svn_lib = $$build_structure{"LIBS_vcs-svn/lib_GUID"}; print F << "EOM"; - + $uuid_vcs_svn_lib false @@ -329,7 +331,7 @@ sub createGlueProject { my $vcxproj = $build_structure{"APPS_${appname}_VCXPROJ"}; $vcxproj =~ s/\//\\/g; $appname =~ s/.*\///; - print F "\"${appname}\", \"${vcxproj}\", \"${uuid}\""; + print F "\"${appname}.proj\", \"${vcxproj}\", \"${uuid}\""; print F "$SLN_POST"; } foreach (@libs) { @@ -339,7 +341,7 @@ sub createGlueProject { my $vcxproj = $build_structure{"LIBS_${libname}_VCXPROJ"}; $vcxproj =~ s/\//\\/g; $libname =~ s/\//_/g; - print F "\"${libname}\", \"${vcxproj}\", \"${uuid}\""; + print F "\"${libname}.proj\", \"${vcxproj}\", \"${uuid}\""; print F "$SLN_POST"; } From a9fa8ba0e897a999988893079558aebcc6771773 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 9 May 2020 16:19:06 +0200 Subject: [PATCH 34/72] t5505/t5516: allow running without `.git/branches/` in the templates When we commit the template directory as part of `make vcxproj`, the `branches/` directory is not actually commited, as it is empty. Two tests were not prepared for that situation. This developer tried to get rid of the support for `.git/branches/` a long time ago, but that effort did not bear fruit, so the best we can do is work around in these here tests. Signed-off-by: Johannes Schindelin --- t/t5505-remote.sh | 2 ++ t/t5516-fetch-push.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index ecadf02d64..c06b019b2f 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -923,6 +923,7 @@ test_expect_success 'migrate a remote from named file in $GIT_DIR/branches' ' ( cd six && git remote rm origin && + mkdir -p .git/branches && echo "$origin_url#main" >.git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && @@ -937,6 +938,7 @@ test_expect_success 'migrate a remote from named file in $GIT_DIR/branches (2)' ( cd seven && git remote rm origin && + mkdir -p .git/branches && echo "quux#foom" > .git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 3ed121d0ce..c0af71a779 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -872,6 +872,7 @@ test_expect_success 'fetch with branches' ' mk_empty testrepo && git branch second $the_first_commit && git checkout second && + mkdir -p testrepo/.git/branches && echo ".." > testrepo/.git/branches/branch1 && ( cd testrepo && @@ -885,6 +886,7 @@ test_expect_success 'fetch with branches' ' test_expect_success 'fetch with branches containing #' ' mk_empty testrepo && + mkdir -p testrepo/.git/branches && echo "..#second" > testrepo/.git/branches/branch2 && ( cd testrepo && @@ -899,6 +901,7 @@ test_expect_success 'fetch with branches containing #' ' test_expect_success 'push with branches' ' mk_empty testrepo && git checkout second && + mkdir -p .git/branches && echo "testrepo" > .git/branches/branch1 && git push branch1 && ( @@ -911,6 +914,7 @@ test_expect_success 'push with branches' ' test_expect_success 'push with branches containing #' ' mk_empty testrepo && + mkdir -p .git/branches && echo "testrepo#branch3" > .git/branches/branch2 && git push branch2 && ( From b9543bdb05f85488cfea5664171cbe69aab43408 Mon Sep 17 00:00:00 2001 From: Kelly Heller Date: Wed, 27 May 2015 14:51:43 -0700 Subject: [PATCH 35/72] Allow `add -p` and `add -i` with a large number of files This fixes https://github.com/msysgit/git/issues/182. Inspired by Pull Request 218 using code from @PhilipDavis. [jes: simplified code quite a bit] Signed-off-by: Kelly Heller Signed-off-by: Johannes Schindelin --- git-add--interactive.perl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/git-add--interactive.perl b/git-add--interactive.perl index bc3a1e8eff..3a4fba1b62 100755 --- a/git-add--interactive.perl +++ b/git-add--interactive.perl @@ -174,6 +174,24 @@ sub run_cmd_pipe { die "$^O does not support: @invalid\n" if @invalid; my @args = map { m/ /o ? "\"$_\"": $_ } @_; return qx{@args}; + } elsif (($^O eq 'MSWin32' || $^O eq 'msys') && (scalar @_ > 200) && + grep $_ eq '--', @_) { + use File::Temp qw(tempfile); + my ($fhargs, $filename) = + tempfile('git-args-XXXXXX', UNLINK => 1); + + my $cmd = 'cat '.$filename.' | xargs -0 -s 20000 '; + while ($_[0] ne '--') { + $cmd = $cmd . shift(@_) . ' '; + } + + shift(@_); + print $fhargs join("\0", @_); + close($fhargs); + + my $fh = undef; + open($fh, '-|', $cmd) or die; + return <$fh>; } else { my $fh = undef; open($fh, '-|', @_) or die; From dd1616ace30a5e38f2362315aca96a8986003621 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 9 May 2020 19:24:23 +0200 Subject: [PATCH 36/72] t5505/t5516: fix white-space around redirectors The convention in Git project's shell scripts is to have white-space _before_, but not _after_ the `>` (or `<`). Signed-off-by: Johannes Schindelin --- t/t5505-remote.sh | 6 +++--- t/t5516-fetch-push.sh | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index c06b019b2f..c0403a80ef 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -734,8 +734,8 @@ test_expect_success '"remote show" does not show symbolic refs' ' ( cd three && git remote show origin >output && - ! grep "^ *HEAD$" < output && - ! grep -i stale < output + ! grep "^ *HEAD$" .git/branches/origin && + echo "quux#foom" >.git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && test "$(git config remote.origin.url)" = "quux" && diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index c0af71a779..da37870e0b 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -873,7 +873,7 @@ test_expect_success 'fetch with branches' ' git branch second $the_first_commit && git checkout second && mkdir -p testrepo/.git/branches && - echo ".." > testrepo/.git/branches/branch1 && + echo ".." >testrepo/.git/branches/branch1 && ( cd testrepo && git fetch branch1 && @@ -887,7 +887,7 @@ test_expect_success 'fetch with branches' ' test_expect_success 'fetch with branches containing #' ' mk_empty testrepo && mkdir -p testrepo/.git/branches && - echo "..#second" > testrepo/.git/branches/branch2 && + echo "..#second" >testrepo/.git/branches/branch2 && ( cd testrepo && git fetch branch2 && @@ -902,7 +902,7 @@ test_expect_success 'push with branches' ' mk_empty testrepo && git checkout second && mkdir -p .git/branches && - echo "testrepo" > .git/branches/branch1 && + echo "testrepo" >.git/branches/branch1 && git push branch1 && ( cd testrepo && @@ -915,7 +915,7 @@ test_expect_success 'push with branches' ' test_expect_success 'push with branches containing #' ' mk_empty testrepo && mkdir -p .git/branches && - echo "testrepo#branch3" > .git/branches/branch2 && + echo "testrepo#branch3" >.git/branches/branch2 && git push branch2 && ( cd testrepo && @@ -1440,7 +1440,7 @@ EOF git init no-thin && git --git-dir=no-thin/.git config receive.unpacklimit 0 && git push no-thin/.git refs/heads/master:refs/heads/foo && - echo modified >> path1 && + echo modified >>path1 && git commit -am modified && git repack -adf && rcvpck="git receive-pack --reject-thin-pack-for-testing" && From 534934aab40a95262c2f5899504160b50a4deeb8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 12 Sep 2015 12:25:47 +0200 Subject: [PATCH 37/72] t3701: verify that we can add *lots* of files interactively Signed-off-by: Johannes Schindelin --- t/t3701-add-interactive.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/t/t3701-add-interactive.sh b/t/t3701-add-interactive.sh index cc3f434a97..13b113feb6 100755 --- a/t/t3701-add-interactive.sh +++ b/t/t3701-add-interactive.sh @@ -944,6 +944,27 @@ test_expect_success 'checkout -p patch editing of added file' ' ) ' +test_expect_success EXPENSIVE 'add -i with a lot of files' ' + git reset --hard && + x160=0123456789012345678901234567890123456789 && + x160=$x160$x160$x160$x160 && + y= && + i=0 && + while test $i -le 200 + do + name=$(printf "%s%03d" $x160 $i) && + echo $name >$name && + git add -N $name && + y="${y}y$LF" && + i=$(($i+1)) || + break + done && + echo "$y" | git add -p -- . && + git diff --cached >staged && + test_line_count = 1407 staged && + git reset --hard +' + test_expect_success 'show help from add--helper' ' git reset --hard && cat >expect <<-EOF && From 0ec7ee5715392b7cfa85edea3b76d6fe1b4ec7da Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 2 Jul 2020 16:35:05 +0200 Subject: [PATCH 38/72] git add -i: handle CR/LF line endings in the interactive input As of Git for Windows v2.27.0, there is an option to use Windows' newly-introduced Pseudo Console support. When running an interactive add operation with this support enabled, Git will receive CR/LF line endings. Therefore, let's not pretend that we are expecting Unix line endings. This fixes https://github.com/git-for-windows/git/issues/2729 Signed-off-by: Johannes Schindelin --- prompt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompt.c b/prompt.c index 5ded21a017..460661dbfb 100644 --- a/prompt.c +++ b/prompt.c @@ -80,7 +80,7 @@ int git_read_line_interactively(struct strbuf *line) int ret; fflush(stdout); - ret = strbuf_getline_lf(line, stdin); + ret = strbuf_getline(line, stdin); if (ret != EOF) strbuf_trim_trailing_newline(line); From 2872be23246be3080b726d4977b5ae9c291d3443 Mon Sep 17 00:00:00 2001 From: Luke Bonanomi Date: Wed, 24 Jun 2020 07:45:52 -0400 Subject: [PATCH 39/72] commit: accept "scissors" with CR/LF line endings This change enhances `git commit --cleanup=scissors` by detecting scissors lines ending in either LF (UNIX-style) or CR/LF (DOS-style). Regression tests are included to specifically test for trailing comments after a CR/LF-terminated scissors line. Signed-off-by: Luke Bonanomi Signed-off-by: Johannes Schindelin --- t/t7502-commit-porcelain.sh | 42 +++++++++++++++++++++++++++++++++++++ wt-status.c | 13 +++++++++--- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/t/t7502-commit-porcelain.sh b/t/t7502-commit-porcelain.sh index 14c92e4c25..83874914d5 100755 --- a/t/t7502-commit-porcelain.sh +++ b/t/t7502-commit-porcelain.sh @@ -271,6 +271,48 @@ test_expect_success 'cleanup commit messages (scissors option,-F,-e, scissors on test_must_be_empty actual ' +test_expect_success 'helper-editor' ' + + write_script lf-to-crlf.sh <<-\EOF + sed "s/\$/Q/" <"$1" | tr Q "\\015" >"$1".new && + mv -f "$1".new "$1" + EOF +' + +test_expect_success 'cleanup commit messages (scissors option,-F,-e, CR/LF line endings)' ' + + test_config core.editor "\"$PWD/lf-to-crlf.sh\"" && + scissors="# ------------------------ >8 ------------------------" && + + test_write_lines >text \ + "# Keep this comment" "" " $scissors" \ + "# Keep this comment, too" "$scissors" \ + "# Remove this comment" "$scissors" \ + "Remove this comment, too" && + + test_write_lines >expect \ + "# Keep this comment" "" " $scissors" \ + "# Keep this comment, too" && + + git commit --cleanup=scissors -e -F text --allow-empty && + git cat-file -p HEAD >raw && + sed -e "1,/^\$/d" raw >actual && + test_cmp expect actual +' + +test_expect_success 'cleanup commit messages (scissors option,-F,-e, scissors on first line, CR/LF line endings)' ' + + scissors="# ------------------------ >8 ------------------------" && + test_write_lines >text \ + "$scissors" \ + "# Remove this comment and any following lines" && + cp text /tmp/test2-text && + git commit --cleanup=scissors -e -F text --allow-empty --allow-empty-message && + git cat-file -p HEAD >raw && + sed -e "1,/^\$/d" raw >actual && + test_must_be_empty actual +' + test_expect_success 'cleanup commit messages (strip option,-F)' ' echo >>negative && diff --git a/wt-status.c b/wt-status.c index 40b59be478..cddfc5f3e9 100644 --- a/wt-status.c +++ b/wt-status.c @@ -22,7 +22,7 @@ #define AB_DELAY_WARNING_IN_MS (2 * 1000) static const char cut_line[] = -"------------------------ >8 ------------------------\n"; +"------------------------ >8 ------------------------"; static char default_wt_status_colors[][COLOR_MAXLEN] = { GIT_COLOR_NORMAL, /* WT_STATUS_HEADER */ @@ -999,15 +999,22 @@ conclude: status_printf_ln(s, GIT_COLOR_NORMAL, "%s", ""); } +static inline int starts_with_newline(const char *p) +{ + return *p == '\n' || (*p == '\r' && p[1] == '\n'); +} + size_t wt_status_locate_end(const char *s, size_t len) { const char *p; struct strbuf pattern = STRBUF_INIT; strbuf_addf(&pattern, "\n%c %s", comment_line_char, cut_line); - if (starts_with(s, pattern.buf + 1)) + if (starts_with(s, pattern.buf + 1) && + starts_with_newline(s + pattern.len - 1)) len = 0; - else if ((p = strstr(s, pattern.buf))) + else if ((p = strstr(s, pattern.buf)) && + starts_with_newline(p + pattern.len)) len = p - s + 1; strbuf_release(&pattern); return len; From b4a0944c35cbebefe069c776ea6820c75aae35fb Mon Sep 17 00:00:00 2001 From: Jens Glathe Date: Tue, 2 Jun 2020 12:12:25 +0200 Subject: [PATCH 40/72] t0014: fix indentation For some reason, this test case was indented with 4 spaces instead of 1 horizontal tab. The other test cases in the same test script are fine. Signed-off-by: Jens Glathe Signed-off-by: Johannes Schindelin --- t/t0014-alias.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t/t0014-alias.sh b/t/t0014-alias.sh index 8d3d9144c0..288e08299a 100755 --- a/t/t0014-alias.sh +++ b/t/t0014-alias.sh @@ -38,10 +38,10 @@ test_expect_success 'looping aliases - internal execution' ' #' test_expect_success 'run-command formats empty args properly' ' - test_must_fail env GIT_TRACE=1 git frotz a "" b " " c 2>actual.raw && - sed -ne "/run_command:/s/.*trace: run_command: //p" actual.raw >actual && - echo "git-frotz a '\'''\'' b '\'' '\'' c" >expect && - test_cmp expect actual + test_must_fail env GIT_TRACE=1 git frotz a "" b " " c 2>actual.raw && + sed -ne "/run_command:/s/.*trace: run_command: //p" actual.raw >actual && + echo "git-frotz a '\'''\'' b '\'' '\'' c" >expect && + test_cmp expect actual ' test_done From 50107fbd7e295ef71955889b1eeff6458f2fc00e Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 24 Nov 2020 04:16:42 +0000 Subject: [PATCH 41/72] maintenance: extract platform-specific scheduling The existing schedule mechanism using 'cron' is supported by POSIX platforms, but not Windows. It also works slightly differently on macOS to significant detriment of the user experience. To allow for new implementations on these platforms, extract a method that performs the platform-specific scheduling mechanism. This will be swapped at compile time with new implementations on specialized platforms. As we add this generality, rename GIT_TEST_CRONTAB to GIT_TEST_MAINT_SCHEDULER. Further, this variable is now parsed as ":" so we can test platform-specific scheduling logic even when not on the correct platform. By specifying the in this string, we will be able to test all three sets of Git logic from a Linux machine. Co-authored-by: Eric Sunshine Signed-off-by: Eric Sunshine Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/gc.c | 72 ++++++++++++++++++++++++++---------------- t/t7900-maintenance.sh | 10 +++--- t/test-lib.sh | 7 ++-- 3 files changed, 53 insertions(+), 36 deletions(-) diff --git a/builtin/gc.c b/builtin/gc.c index 64f2b52d6e..ca87d3b917 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1496,35 +1496,23 @@ static int maintenance_unregister(void) #define BEGIN_LINE "# BEGIN GIT MAINTENANCE SCHEDULE" #define END_LINE "# END GIT MAINTENANCE SCHEDULE" -static int update_background_schedule(int run_maintenance) +static int crontab_update_schedule(int run_maintenance, int fd, const char *cmd) { int result = 0; int in_old_region = 0; struct child_process crontab_list = CHILD_PROCESS_INIT; struct child_process crontab_edit = CHILD_PROCESS_INIT; FILE *cron_list, *cron_in; - const char *crontab_name; struct strbuf line = STRBUF_INIT; - struct lock_file lk; - char *lock_path = xstrfmt("%s/schedule", the_repository->objects->odb->path); - if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) - return error(_("another process is scheduling background maintenance")); - - crontab_name = getenv("GIT_TEST_CRONTAB"); - if (!crontab_name) - crontab_name = "crontab"; - - strvec_split(&crontab_list.args, crontab_name); + strvec_split(&crontab_list.args, cmd); strvec_push(&crontab_list.args, "-l"); crontab_list.in = -1; - crontab_list.out = dup(get_lock_file_fd(&lk)); + crontab_list.out = dup(fd); crontab_list.git_cmd = 0; - if (start_command(&crontab_list)) { - result = error(_("failed to run 'crontab -l'; your system might not support 'cron'")); - goto cleanup; - } + if (start_command(&crontab_list)) + return error(_("failed to run 'crontab -l'; your system might not support 'cron'")); /* Ignore exit code, as an empty crontab will return error. */ finish_command(&crontab_list); @@ -1533,17 +1521,15 @@ static int update_background_schedule(int run_maintenance) * Read from the .lock file, filtering out the old * schedule while appending the new schedule. */ - cron_list = fdopen(get_lock_file_fd(&lk), "r"); + cron_list = fdopen(fd, "r"); rewind(cron_list); - strvec_split(&crontab_edit.args, crontab_name); + strvec_split(&crontab_edit.args, cmd); crontab_edit.in = -1; crontab_edit.git_cmd = 0; - if (start_command(&crontab_edit)) { - result = error(_("failed to run 'crontab'; your system might not support 'cron'")); - goto cleanup; - } + if (start_command(&crontab_edit)) + return error(_("failed to run 'crontab'; your system might not support 'cron'")); cron_in = fdopen(crontab_edit.in, "w"); if (!cron_in) { @@ -1587,14 +1573,44 @@ static int update_background_schedule(int run_maintenance) close(crontab_edit.in); done_editing: - if (finish_command(&crontab_edit)) { + if (finish_command(&crontab_edit)) result = error(_("'crontab' died")); - goto cleanup; - } - fclose(cron_list); + else + fclose(cron_list); + return result; +} + +static const char platform_scheduler[] = "crontab"; + +static int update_background_schedule(int enable) +{ + int result; + const char *scheduler = platform_scheduler; + const char *cmd = scheduler; + char *testing; + struct lock_file lk; + char *lock_path = xstrfmt("%s/schedule", the_repository->objects->odb->path); + + testing = xstrdup_or_null(getenv("GIT_TEST_MAINT_SCHEDULER")); + if (testing) { + char *sep = strchr(testing, ':'); + if (!sep) + die("GIT_TEST_MAINT_SCHEDULER unparseable: %s", testing); + *sep = '\0'; + scheduler = testing; + cmd = sep + 1; + } + + if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) + return error(_("another process is scheduling background maintenance")); + + if (!strcmp(scheduler, "crontab")) + result = crontab_update_schedule(enable, lk.tempfile->fd, cmd); + else + die("unknown background scheduler: %s", scheduler); -cleanup: rollback_lock_file(&lk); + free(testing); return result; } diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 99bf0c7582..fa05e75e4e 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -419,7 +419,7 @@ test_expect_success !MINGW 'register and unregister with regex metacharacters' ' ' test_expect_success 'start from empty cron table' ' - GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance start && + GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance start && # start registers the repo git config --get --global --fixed-value maintenance.repo "$(pwd)" && @@ -430,19 +430,19 @@ test_expect_success 'start from empty cron table' ' ' test_expect_success 'stop from existing schedule' ' - GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance stop && + GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance stop && # stop does not unregister the repo git config --get --global --fixed-value maintenance.repo "$(pwd)" && # Operation is idempotent - GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance stop && + GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance stop && test_must_be_empty cron.txt ' test_expect_success 'start preserves existing schedule' ' echo "Important information!" >cron.txt && - GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance start && + GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance start && grep "Important information!" cron.txt ' @@ -457,7 +457,7 @@ test_expect_success 'magic markers are correct' ' test_expect_success 'stop preserves surrounding schedule' ' echo "Crucial information!" >>cron.txt && - GIT_TEST_CRONTAB="test-tool crontab cron.txt" git maintenance stop && + GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt" git maintenance stop && grep "Important information!" cron.txt && grep "Crucial information!" cron.txt ' diff --git a/t/test-lib.sh b/t/test-lib.sh index 9fa7c1d0f6..03c1c0836f 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1713,7 +1713,8 @@ test_lazy_prereq REBASE_P ' ' # Ensure that no test accidentally triggers a Git command -# that runs 'crontab', affecting a user's cron schedule. -# Tests that verify the cron integration must set this locally +# that runs the actual maintenance scheduler, affecting a user's +# system permanently. +# Tests that verify the scheduler integration must set this locally # to avoid errors. -GIT_TEST_CRONTAB="exit 1" +GIT_TEST_MAINT_SCHEDULER="none:exit 1" From 441a25d5735aaff022e16c3c955f708a9fb7eae8 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 24 Nov 2020 04:16:43 +0000 Subject: [PATCH 42/72] maintenance: include 'cron' details in docs Advanced and expert users may want to know how 'git maintenance start' schedules background maintenance in order to customize their own schedules beyond what the maintenance.* config values allow. Start a new set of sections in git-maintenance.txt that describe how 'cron' is used to run these tasks. This is particularly valuable for users who want to inspect what Git is doing or for users who want to customize the schedule further. Having a baseline can provide a way forward for users who have never worked with cron schedules. Helped-by: Eric Sunshine Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/git-maintenance.txt | 54 +++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/Documentation/git-maintenance.txt b/Documentation/git-maintenance.txt index 6fe1e5e105..17044c4922 100644 --- a/Documentation/git-maintenance.txt +++ b/Documentation/git-maintenance.txt @@ -218,6 +218,60 @@ Further, the `git gc` command should not be combined with but does not take the lock in the same way as `git maintenance run`. If possible, use `git maintenance run --task=gc` instead of `git gc`. +The following sections describe the mechanisms put in place to run +background maintenance by `git maintenance start` and how to customize +them. + +BACKGROUND MAINTENANCE ON POSIX SYSTEMS +--------------------------------------- + +The standard mechanism for scheduling background tasks on POSIX systems +is cron(8). This tool executes commands based on a given schedule. The +current list of user-scheduled tasks can be found by running `crontab -l`. +The schedule written by `git maintenance start` is similar to this: + +----------------------------------------------------------------------- +# BEGIN GIT MAINTENANCE SCHEDULE +# The following schedule was created by Git +# Any edits made in this region might be +# replaced in the future by a Git command. + +0 1-23 * * * "//git" --exec-path="/" for-each-repo --config=maintenance.repo maintenance run --schedule=hourly +0 0 * * 1-6 "//git" --exec-path="/" for-each-repo --config=maintenance.repo maintenance run --schedule=daily +0 0 * * 0 "//git" --exec-path="/" for-each-repo --config=maintenance.repo maintenance run --schedule=weekly + +# END GIT MAINTENANCE SCHEDULE +----------------------------------------------------------------------- + +The comments are used as a region to mark the schedule as written by Git. +Any modifications within this region will be completely deleted by +`git maintenance stop` or overwritten by `git maintenance start`. + +The `crontab` entry specifies the full path of the `git` executable to +ensure that the executed `git` command is the same one with which +`git maintenance start` was issued independent of `PATH`. If the same user +runs `git maintenance start` with multiple Git executables, then only the +latest executable is used. + +These commands use `git for-each-repo --config=maintenance.repo` to run +`git maintenance run --schedule=` on each repository listed in +the multi-valued `maintenance.repo` config option. These are typically +loaded from the user-specific global config. The `git maintenance` process +then determines which maintenance tasks are configured to run on each +repository with each `` using the `maintenance..schedule` +config options. These values are loaded from the global or repository +config values. + +If the config values are insufficient to achieve your desired background +maintenance schedule, then you can create your own schedule. If you run +`crontab -e`, then an editor will load with your user-specific `cron` +schedule. In that editor, you can add your own schedule lines. You could +start by adapting the default schedule listed earlier, or you could read +the crontab(5) documentation for advanced scheduling techniques. Please +do use the full path and `--exec-path` techniques from the default +schedule to ensure you are executing the correct binaries in your +schedule. + GIT --- From 4b236199cf30354d0a4cc40e7d42fb378767ab11 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 5 Jan 2021 13:08:27 +0000 Subject: [PATCH 43/72] maintenance: use launchctl on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing mechanism for scheduling background maintenance is done through cron. The 'crontab -e' command allows updating the schedule while cron itself runs those commands. While this is technically supported by macOS, it has some significant deficiencies: 1. Every run of 'crontab -e' must request elevated privileges through the user interface. When running 'git maintenance start' from the Terminal app, it presents a dialog box saying "Terminal.app would like to administer your computer. Administration can include modifying passwords, networking, and system settings." This is more alarming than what we are hoping to achieve. If this alert had some information about how "git" is trying to run "crontab" then we would have some reason to believe that this dialog might be fine. However, it also doesn't help that some scenarios just leave Git waiting for a response without presenting anything to the user. I experienced this when executing the command from a Bash terminal view inside Visual Studio Code. 2. While cron initializes a user environment enough for "git config --global --show-origin" to show the correct config file information, it does not set up the environment enough for Git Credential Manager Core to load credentials during a 'prefetch' task. My prefetches against private repositories required re-authenticating through UI pop-ups in a way that should not be required. The solution is to switch from cron to the Apple-recommended [1] 'launchd' tool. [1] https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPSystemStartup/Chapters/ScheduledJobs.html The basics of this tool is that we need to create XML-formatted "plist" files inside "~/Library/LaunchAgents/" and then use the 'launchctl' tool to make launchd aware of them. The plist files include all of the scheduling information, along with the command-line arguments split across an array of tags. For example, here is my plist file for the weekly scheduled tasks: Labelorg.git-scm.git.weekly ProgramArguments /usr/local/libexec/git-core/git --exec-path=/usr/local/libexec/git-core for-each-repo --config=maintenance.repo maintenance run --schedule=weekly StartCalendarInterval Day0 Hour0 Minute0 The schedules for the daily and hourly tasks are more complicated since we need to use an array for the StartCalendarInterval with an entry for each of the six days other than the 0th day (to avoid colliding with the weekly task), and each of the 23 hours other than the 0th hour (to avoid colliding with the daily task). The "Label" value is currently filled with "org.git-scm.git.X" where X is the frequency. We need a different plist file for each frequency. The launchctl command needs to be aligned with a user id in order to initialize the command environment. This must be done using the 'launchctl bootstrap' subcommand. This subcommand is new as of macOS 10.11, which was released in September 2015. Before that release the 'launchctl load' subcommand was recommended. The best source of information on this transition I have seen is available at [2]. The current design does not preclude a future version that detects the available fatures of 'launchctl' to use the older commands. However, it is best to rely on the newest version since Apple might completely remove the deprecated version on short notice. [2] https://babodee.wordpress.com/2016/04/09/launchctl-2-0-syntax/ To remove a schedule, we must run 'launchctl bootout' with a valid plist file. We also need to 'bootout' a task before the 'bootstrap' subcommand will succeed, if such a task already exists. The need for a user id requires us to run 'id -u' which works on POSIX systems but not Windows. Further, the need for fully-qualitifed path names including $HOME behaves differently in the Git internals and the external test suite. The $HOME variable starts with "C:\..." instead of the "/c/..." that is provided by Git in these subcommands. The test therefore has a prerequisite that we are not on Windows. The cross- platform logic still allows us to test the macOS logic on a Linux machine. We can verify the commands that were run by 'git maintenance start' and 'git maintenance stop' by injecting a script that writes the command-line arguments into GIT_TEST_MAINT_SCHEDULER. An earlier version of this patch accidentally had an opening "" tag when it should have had a closing "" tag. This was caught during manual testing with actual 'launchctl' commands, but we do not want to update developers' tasks when running tests. It appears that macOS includes the "xmllint" tool which can verify the XML format. This is useful for any system that might contain the tool, so use it whenever it is available. We strive to make these tests work on all platforms, but Windows caused some headaches. In particular, the value of getuid() called by the C code is not guaranteed to be the same as `$(id -u)` invoked by a test. This is because `git.exe` is a native Windows program, whereas the utility programs run by the test script mostly utilize the MSYS2 runtime, which emulates a POSIX-like environment. Since the purpose of the test is to check that the input to the hook is well-formed, the actual user ID is immaterial, thus we can work around the problem by making the the test UID-agnostic. Another subtle issue is the $HOME environment variable being a Windows-style path instead of a Unix-style path. We can be more flexible here instead of expecting exact path matches. Helped-by: Ævar Arnfjörð Bjarmason Co-authored-by: Eric Sunshine Signed-off-by: Eric Sunshine Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/git-maintenance.txt | 40 +++++++ builtin/gc.c | 188 +++++++++++++++++++++++++++++- t/t7900-maintenance.sh | 59 ++++++++++ 3 files changed, 286 insertions(+), 1 deletion(-) diff --git a/Documentation/git-maintenance.txt b/Documentation/git-maintenance.txt index 17044c4922..56f1cd2c75 100644 --- a/Documentation/git-maintenance.txt +++ b/Documentation/git-maintenance.txt @@ -273,6 +273,46 @@ schedule to ensure you are executing the correct binaries in your schedule. +BACKGROUND MAINTENANCE ON MACOS SYSTEMS +--------------------------------------- + +While macOS technically supports `cron`, using `crontab -e` requires +elevated privileges and the executed process does not have a full user +context. Without a full user context, Git and its credential helpers +cannot access stored credentials, so some maintenance tasks are not +functional. + +Instead, `git maintenance start` interacts with the `launchctl` tool, +which is the recommended way to schedule timed jobs in macOS. Scheduling +maintenance through `git maintenance (start|stop)` requires some +`launchctl` features available only in macOS 10.11 or later. + +Your user-specific scheduled tasks are stored as XML-formatted `.plist` +files in `~/Library/LaunchAgents/`. You can see the currently-registered +tasks using the following command: + +----------------------------------------------------------------------- +$ ls ~/Library/LaunchAgents/org.git-scm.git* +org.git-scm.git.daily.plist +org.git-scm.git.hourly.plist +org.git-scm.git.weekly.plist +----------------------------------------------------------------------- + +One task is registered for each `--schedule=` option. To +inspect how the XML format describes each schedule, open one of these +`.plist` files in an editor and inspect the `` element following +the `StartCalendarInterval` element. + +`git maintenance start` will overwrite these files and register the +tasks again with `launchctl`, so any customizations should be done by +creating your own `.plist` files with distinct names. Similarly, the +`git maintenance stop` command will unregister the tasks with `launchctl` +and delete the `.plist` files. + +To create more advanced customizations to your background tasks, see +launchctl.plist(5) for more information. + + GIT --- Part of the linkgit:git[1] suite diff --git a/builtin/gc.c b/builtin/gc.c index ca87d3b917..6f1d692d0d 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1493,6 +1493,186 @@ static int maintenance_unregister(void) return run_command(&config_unset); } +static const char *get_frequency(enum schedule_priority schedule) +{ + switch (schedule) { + case SCHEDULE_HOURLY: + return "hourly"; + case SCHEDULE_DAILY: + return "daily"; + case SCHEDULE_WEEKLY: + return "weekly"; + default: + BUG("invalid schedule %d", schedule); + } +} + +static char *launchctl_service_name(const char *frequency) +{ + struct strbuf label = STRBUF_INIT; + strbuf_addf(&label, "org.git-scm.git.%s", frequency); + return strbuf_detach(&label, NULL); +} + +static char *launchctl_service_filename(const char *name) +{ + char *expanded; + struct strbuf filename = STRBUF_INIT; + strbuf_addf(&filename, "~/Library/LaunchAgents/%s.plist", name); + + expanded = expand_user_path(filename.buf, 1); + if (!expanded) + die(_("failed to expand path '%s'"), filename.buf); + + strbuf_release(&filename); + return expanded; +} + +static char *launchctl_get_uid(void) +{ + return xstrfmt("gui/%d", getuid()); +} + +static int launchctl_boot_plist(int enable, const char *filename, const char *cmd) +{ + int result; + struct child_process child = CHILD_PROCESS_INIT; + char *uid = launchctl_get_uid(); + + strvec_split(&child.args, cmd); + if (enable) + strvec_push(&child.args, "bootstrap"); + else + strvec_push(&child.args, "bootout"); + strvec_push(&child.args, uid); + strvec_push(&child.args, filename); + + child.no_stderr = 1; + child.no_stdout = 1; + + if (start_command(&child)) + die(_("failed to start launchctl")); + + result = finish_command(&child); + + free(uid); + return result; +} + +static int launchctl_remove_plist(enum schedule_priority schedule, const char *cmd) +{ + const char *frequency = get_frequency(schedule); + char *name = launchctl_service_name(frequency); + char *filename = launchctl_service_filename(name); + int result = launchctl_boot_plist(0, filename, cmd); + unlink(filename); + free(filename); + free(name); + return result; +} + +static int launchctl_remove_plists(const char *cmd) +{ + return launchctl_remove_plist(SCHEDULE_HOURLY, cmd) || + launchctl_remove_plist(SCHEDULE_DAILY, cmd) || + launchctl_remove_plist(SCHEDULE_WEEKLY, cmd); +} + +static int launchctl_schedule_plist(const char *exec_path, enum schedule_priority schedule, const char *cmd) +{ + FILE *plist; + int i; + const char *preamble, *repeat; + const char *frequency = get_frequency(schedule); + char *name = launchctl_service_name(frequency); + char *filename = launchctl_service_filename(name); + + if (safe_create_leading_directories(filename)) + die(_("failed to create directories for '%s'"), filename); + plist = xfopen(filename, "w"); + + preamble = "\n" + "\n" + "" + "\n" + "Label%s\n" + "ProgramArguments\n" + "\n" + "%s/git\n" + "--exec-path=%s\n" + "for-each-repo\n" + "--config=maintenance.repo\n" + "maintenance\n" + "run\n" + "--schedule=%s\n" + "\n" + "StartCalendarInterval\n" + "\n"; + fprintf(plist, preamble, name, exec_path, exec_path, frequency); + + switch (schedule) { + case SCHEDULE_HOURLY: + repeat = "\n" + "Hour%d\n" + "Minute0\n" + "\n"; + for (i = 1; i <= 23; i++) + fprintf(plist, repeat, i); + break; + + case SCHEDULE_DAILY: + repeat = "\n" + "Day%d\n" + "Hour0\n" + "Minute0\n" + "\n"; + for (i = 1; i <= 6; i++) + fprintf(plist, repeat, i); + break; + + case SCHEDULE_WEEKLY: + fprintf(plist, + "\n" + "Day0\n" + "Hour0\n" + "Minute0\n" + "\n"); + break; + + default: + /* unreachable */ + break; + } + fprintf(plist, "\n\n\n"); + fclose(plist); + + /* bootout might fail if not already running, so ignore */ + launchctl_boot_plist(0, filename, cmd); + if (launchctl_boot_plist(1, filename, cmd)) + die(_("failed to bootstrap service %s"), filename); + + free(filename); + free(name); + return 0; +} + +static int launchctl_add_plists(const char *cmd) +{ + const char *exec_path = git_exec_path(); + + return launchctl_schedule_plist(exec_path, SCHEDULE_HOURLY, cmd) || + launchctl_schedule_plist(exec_path, SCHEDULE_DAILY, cmd) || + launchctl_schedule_plist(exec_path, SCHEDULE_WEEKLY, cmd); +} + +static int launchctl_update_schedule(int run_maintenance, int fd, const char *cmd) +{ + if (run_maintenance) + return launchctl_add_plists(cmd); + else + return launchctl_remove_plists(cmd); +} + #define BEGIN_LINE "# BEGIN GIT MAINTENANCE SCHEDULE" #define END_LINE "# END GIT MAINTENANCE SCHEDULE" @@ -1580,7 +1760,11 @@ done_editing: return result; } +#if defined(__APPLE__) +static const char platform_scheduler[] = "launchctl"; +#else static const char platform_scheduler[] = "crontab"; +#endif static int update_background_schedule(int enable) { @@ -1604,7 +1788,9 @@ static int update_background_schedule(int enable) if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) return error(_("another process is scheduling background maintenance")); - if (!strcmp(scheduler, "crontab")) + if (!strcmp(scheduler, "launchctl")) + result = launchctl_update_schedule(enable, lk.tempfile->fd, cmd); + else if (!strcmp(scheduler, "crontab")) result = crontab_update_schedule(enable, lk.tempfile->fd, cmd); else die("unknown background scheduler: %s", scheduler); diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index fa05e75e4e..430b580963 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -7,6 +7,19 @@ test_description='git maintenance builtin' GIT_TEST_COMMIT_GRAPH=0 GIT_TEST_MULTI_PACK_INDEX=0 +test_lazy_prereq XMLLINT ' + xmllint --version +' + +test_xmllint () { + if test_have_prereq XMLLINT + then + xmllint --noout "$@" + else + true + fi +} + test_expect_success 'help text' ' test_expect_code 129 git maintenance -h 2>err && test_i18ngrep "usage: git maintenance " err && @@ -462,6 +475,52 @@ test_expect_success 'stop preserves surrounding schedule' ' grep "Crucial information!" cron.txt ' +test_expect_success 'start and stop macOS maintenance' ' + # ensure $HOME can be compared against hook arguments on all platforms + pfx=$(cd "$HOME" && pwd) && + + write_script print-args <<-\EOF && + echo $* | sed "s:gui/[0-9][0-9]*:gui/[UID]:" >>args + EOF + + rm -f args && + GIT_TEST_MAINT_SCHEDULER=launchctl:./print-args git maintenance start && + + # start registers the repo + git config --get --global maintenance.repo "$(pwd)" && + + ls "$HOME/Library/LaunchAgents" >actual && + cat >expect <<-\EOF && + org.git-scm.git.daily.plist + org.git-scm.git.hourly.plist + org.git-scm.git.weekly.plist + EOF + test_cmp expect actual && + + rm -f expect && + for frequency in hourly daily weekly + do + PLIST="$pfx/Library/LaunchAgents/org.git-scm.git.$frequency.plist" && + test_xmllint "$PLIST" && + grep schedule=$frequency "$PLIST" && + echo "bootout gui/[UID] $PLIST" >>expect && + echo "bootstrap gui/[UID] $PLIST" >>expect || return 1 + done && + test_cmp expect args && + + rm -f args && + GIT_TEST_MAINT_SCHEDULER=launchctl:./print-args git maintenance stop && + + # stop does not unregister the repo + git config --get --global maintenance.repo "$(pwd)" && + + printf "bootout gui/[UID] $pfx/Library/LaunchAgents/org.git-scm.git.%s.plist\n" \ + hourly daily weekly >expect && + test_cmp expect args && + ls "$HOME/Library/LaunchAgents" >actual && + test_line_count = 0 actual +' + test_expect_success 'register preserves existing strategy' ' git config maintenance.strategy none && git maintenance register && From d1501e5c6ecb4632629449731db602b261e11e42 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 5 Jan 2021 13:08:28 +0000 Subject: [PATCH 44/72] maintenance: use Windows scheduled tasks Git's background maintenance uses cron by default, but this is not available on Windows. Instead, integrate with Task Scheduler. Tasks can be scheduled using the 'schtasks' command. There are several command-line options that can allow for some advanced scheduling, but unfortunately these seem to all require authenticating using a password. Instead, use the "/xml" option to pass an XML file that contains the configuration for the necessary schedule. These XML files are based on some that I exported after constructing a schedule in the Task Scheduler GUI. These options only run background maintenance when the user is logged in, and more fields are populated with the current username and SID at run-time by 'schtasks'. Since the GIT_TEST_MAINT_SCHEDULER environment variable allows us to specify 'schtasks' as the scheduler, we can test the Windows-specific logic on other platforms. Thus, add a check that the XML file written by Git is valid when xmllint exists on the system. Since we use a temporary file for the XML files sent to 'schtasks', we prefix the random characters with the frequency so it is easier to examine the proper file during tests. Instead of an exact match on the 'args' file, we 'grep' for the arguments other than the filename. There is a deficiency in the current design. Windows has two kinds of applications: GUI applications that start by "winmain()" and console applications that start by "main()". Console applications are attached to a new Console window if they are not already associated with a GUI application. This means that every hour the scheudled task launches a command window for the scheduled tasks. Not only is this visually obtrusive, but it also takes focus from whatever else the user is doing! A simple fix would be to insert a GUI application that acts as a shim between the scheduled task and Git. This is currently possible in Git for Windows by setting the tag equal to C:\Program Files\Git\git-bash.exe with options "--hide --no-needs-console --command=cmd\git.exe" followed by the arguments currently used. Since git-bash.exe is not included in Windows builds of core Git, I chose to leave out this feature. My plan is to submit a small patch to Git for Windows that converts the use of git.exe with this use of git-bash.exe in the short term. In the long term, we can consider creating this GUI shim application within core Git, perhaps in contrib/. Co-authored-by: Eric Sunshine Signed-off-by: Eric Sunshine Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/git-maintenance.txt | 22 ++++ builtin/gc.c | 168 +++++++++++++++++++++++++++++- t/t7900-maintenance.sh | 37 +++++++ 3 files changed, 226 insertions(+), 1 deletion(-) diff --git a/Documentation/git-maintenance.txt b/Documentation/git-maintenance.txt index 56f1cd2c75..3b432171d6 100644 --- a/Documentation/git-maintenance.txt +++ b/Documentation/git-maintenance.txt @@ -313,6 +313,28 @@ To create more advanced customizations to your background tasks, see launchctl.plist(5) for more information. +BACKGROUND MAINTENANCE ON WINDOWS SYSTEMS +----------------------------------------- + +Windows does not support `cron` and instead has its own system for +scheduling background tasks. The `git maintenance start` command uses +the `schtasks` command to submit tasks to this system. You can inspect +all background tasks using the Task Scheduler application. The tasks +added by Git have names of the form `Git Maintenance ()`. +The Task Scheduler GUI has ways to inspect these tasks, but you can also +export the tasks to XML files and view the details there. + +Note that since Git is a console application, these background tasks +create a console window visible to the current user. This can be changed +manually by selecting the "Run whether user is logged in or not" option +in Task Scheduler. This change requires a password input, which is why +`git maintenance start` does not select it by default. + +If you want to customize the background tasks, please rename the tasks +so future calls to `git maintenance (start|stop)` do not overwrite your +custom tasks. + + GIT --- Part of the linkgit:git[1] suite diff --git a/builtin/gc.c b/builtin/gc.c index 6f1d692d0d..f01f2fddfe 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1591,7 +1591,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit die(_("failed to create directories for '%s'"), filename); plist = xfopen(filename, "w"); - preamble = "\n" + preamble = "\n" "\n" "" "\n" @@ -1673,6 +1673,168 @@ static int launchctl_update_schedule(int run_maintenance, int fd, const char *cm return launchctl_remove_plists(cmd); } +static char *schtasks_task_name(const char *frequency) +{ + struct strbuf label = STRBUF_INIT; + strbuf_addf(&label, "Git Maintenance (%s)", frequency); + return strbuf_detach(&label, NULL); +} + +static int schtasks_remove_task(enum schedule_priority schedule, const char *cmd) +{ + int result; + struct strvec args = STRVEC_INIT; + const char *frequency = get_frequency(schedule); + char *name = schtasks_task_name(frequency); + + strvec_split(&args, cmd); + strvec_pushl(&args, "/delete", "/tn", name, "/f", NULL); + + result = run_command_v_opt(args.v, 0); + + strvec_clear(&args); + free(name); + return result; +} + +static int schtasks_remove_tasks(const char *cmd) +{ + return schtasks_remove_task(SCHEDULE_HOURLY, cmd) || + schtasks_remove_task(SCHEDULE_DAILY, cmd) || + schtasks_remove_task(SCHEDULE_WEEKLY, cmd); +} + +static int schtasks_schedule_task(const char *exec_path, enum schedule_priority schedule, const char *cmd) +{ + int result; + struct child_process child = CHILD_PROCESS_INIT; + const char *xml; + struct tempfile *tfile; + const char *frequency = get_frequency(schedule); + char *name = schtasks_task_name(frequency); + struct strbuf tfilename = STRBUF_INIT; + + strbuf_addf(&tfilename, "%s/schedule_%s_XXXXXX", + get_git_common_dir(), frequency); + tfile = xmks_tempfile(tfilename.buf); + strbuf_release(&tfilename); + + if (!fdopen_tempfile(tfile, "w")) + die(_("failed to create temp xml file")); + + xml = "\n" + "\n" + "\n" + "\n"; + fputs(xml, tfile->fp); + + switch (schedule) { + case SCHEDULE_HOURLY: + fprintf(tfile->fp, + "2020-01-01T01:00:00\n" + "true\n" + "\n" + "1\n" + "\n" + "\n" + "PT1H\n" + "PT23H\n" + "false\n" + "\n"); + break; + + case SCHEDULE_DAILY: + fprintf(tfile->fp, + "2020-01-01T00:00:00\n" + "true\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "1\n" + "\n"); + break; + + case SCHEDULE_WEEKLY: + fprintf(tfile->fp, + "2020-01-01T00:00:00\n" + "true\n" + "\n" + "\n" + "\n" + "\n" + "1\n" + "\n"); + break; + + default: + break; + } + + xml = "\n" + "\n" + "\n" + "\n" + "InteractiveToken\n" + "LeastPrivilege\n" + "\n" + "\n" + "\n" + "IgnoreNew\n" + "true\n" + "true\n" + "true\n" + "false\n" + "PT72H\n" + "7\n" + "\n" + "\n" + "\n" + "\"%s\\git.exe\"\n" + "--exec-path=\"%s\" for-each-repo --config=maintenance.repo maintenance run --schedule=%s\n" + "\n" + "\n" + "\n"; + fprintf(tfile->fp, xml, exec_path, exec_path, frequency); + strvec_split(&child.args, cmd); + strvec_pushl(&child.args, "/create", "/tn", name, "/f", "/xml", + get_tempfile_path(tfile), NULL); + close_tempfile_gently(tfile); + + child.no_stdout = 1; + child.no_stderr = 1; + + if (start_command(&child)) + die(_("failed to start schtasks")); + result = finish_command(&child); + + delete_tempfile(&tfile); + free(name); + return result; +} + +static int schtasks_schedule_tasks(const char *cmd) +{ + const char *exec_path = git_exec_path(); + + return schtasks_schedule_task(exec_path, SCHEDULE_HOURLY, cmd) || + schtasks_schedule_task(exec_path, SCHEDULE_DAILY, cmd) || + schtasks_schedule_task(exec_path, SCHEDULE_WEEKLY, cmd); +} + +static int schtasks_update_schedule(int run_maintenance, int fd, const char *cmd) +{ + if (run_maintenance) + return schtasks_schedule_tasks(cmd); + else + return schtasks_remove_tasks(cmd); +} + #define BEGIN_LINE "# BEGIN GIT MAINTENANCE SCHEDULE" #define END_LINE "# END GIT MAINTENANCE SCHEDULE" @@ -1762,6 +1924,8 @@ done_editing: #if defined(__APPLE__) static const char platform_scheduler[] = "launchctl"; +#elif defined(GIT_WINDOWS_NATIVE) +static const char platform_scheduler[] = "schtasks"; #else static const char platform_scheduler[] = "crontab"; #endif @@ -1790,6 +1954,8 @@ static int update_background_schedule(int enable) if (!strcmp(scheduler, "launchctl")) result = launchctl_update_schedule(enable, lk.tempfile->fd, cmd); + else if (!strcmp(scheduler, "schtasks")) + result = schtasks_update_schedule(enable, lk.tempfile->fd, cmd); else if (!strcmp(scheduler, "crontab")) result = crontab_update_schedule(enable, lk.tempfile->fd, cmd); else diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 430b580963..2e0c8a4c31 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -521,6 +521,43 @@ test_expect_success 'start and stop macOS maintenance' ' test_line_count = 0 actual ' +test_expect_success 'start and stop Windows maintenance' ' + write_script print-args <<-\EOF && + echo $* >>args + while test $# -gt 0 + do + case "$1" in + /xml) shift; xmlfile=$1; break ;; + *) shift ;; + esac + done + test -z "$xmlfile" || cp "$xmlfile" "$xmlfile.xml" + EOF + + rm -f args && + GIT_TEST_MAINT_SCHEDULER="schtasks:./print-args" git maintenance start && + + # start registers the repo + git config --get --global maintenance.repo "$(pwd)" && + + for frequency in hourly daily weekly + do + grep "/create /tn Git Maintenance ($frequency) /f /xml" args && + file=$(ls .git/schedule_${frequency}*.xml) && + test_xmllint "$file" || return 1 + done && + + rm -f args && + GIT_TEST_MAINT_SCHEDULER="schtasks:./print-args" git maintenance stop && + + # stop does not unregister the repo + git config --get --global maintenance.repo "$(pwd)" && + + printf "/delete /tn Git Maintenance (%s) /f\n" \ + hourly daily weekly >expect && + test_cmp expect args +' + test_expect_success 'register preserves existing strategy' ' git config maintenance.strategy none && git maintenance register && From 9aba25b8ce7bfd34c5902112fc8cffb3a7e60a08 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 3 Dec 2020 22:44:24 +0100 Subject: [PATCH 45/72] win32: add a helper to run `git.exe` without a foreground window On Windows, there are two kinds of executables, console ones and non-console ones. Git's executables are all console ones. When launching the former e.g. in a scheduled task, a CMD window pops up. This is not what we want for the tasks installed via the `git maintenance` command. To work around this, let's introduce `headless-git.exe`, which is a non-console program that does _not_ pop up any window. All it does is to re-launch `git.exe`, suppressing that console window, passing through all command-line arguments as-are. Signed-off-by: Johannes Schindelin Signed-off-by: Derrick Stolee --- Makefile | 7 ++ compat/win32/headless.c | 114 ++++++++++++++++++++++++++++ config.mak.uname | 3 + contrib/buildsystems/CMakeLists.txt | 3 + 4 files changed, 127 insertions(+) create mode 100644 compat/win32/headless.c diff --git a/Makefile b/Makefile index 4128b457e1..211ac79c1a 100644 --- a/Makefile +++ b/Makefile @@ -2509,6 +2509,13 @@ compat/nedmalloc/nedmalloc.sp compat/nedmalloc/nedmalloc.o: EXTRA_CPPFLAGS = \ compat/nedmalloc/nedmalloc.sp: SP_EXTRA_FLAGS += -Wno-non-pointer-null endif +headless-git.o: compat/win32/headless.c + $(QUIET_CC)$(CC) $(ALL_CFLAGS) $(COMPAT_CFLAGS) \ + -fno-stack-protector -o $@ -c -Wall -Wwrite-strings $< + +headless-git$X: headless-git.o git.res + $(QUIET_LINK)$(CC) $(ALL_LDFLAGS) -mwindows $(COMPAT_CFLAGS) -o $@ $^ + git-%$X: %.o GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) diff --git a/compat/win32/headless.c b/compat/win32/headless.c new file mode 100644 index 0000000000..d04ddc0795 --- /dev/null +++ b/compat/win32/headless.c @@ -0,0 +1,114 @@ +/* + * headless Git - run Git without opening a console window on Windows + */ + +#define STRICT +#define WIN32_LEAN_AND_MEAN +#define UNICODE +#define _UNICODE +#include +#include +#include +#include + +/* + * If `dir` contains the path to a Git exec directory, extend `PATH` to + * include the corresponding `bin/` directory (which is where all those + * `.dll` files needed by `git.exe` are, on Windows). + */ +static int extend_path(wchar_t *dir, size_t dir_len) +{ + const wchar_t *suffix = L"\\libexec\\git-core"; + size_t suffix_len = wcslen(suffix); + wchar_t *env; + DWORD len; + + if (dir_len < suffix_len) + return 0; + + dir_len -= suffix_len; + if (memcmp(dir + dir_len, suffix, suffix_len * sizeof(wchar_t))) + return 0; + + len = GetEnvironmentVariableW(L"PATH", NULL, 0); + if (!len) + return 0; + + env = _alloca((dir_len + 5 + len) * sizeof(wchar_t)); + wcsncpy(env, dir, dir_len); + wcscpy(env + dir_len, L"\\bin;"); + if (!GetEnvironmentVariableW(L"PATH", env + dir_len + 5, len)) + return 0; + + SetEnvironmentVariableW(L"PATH", env); + return 1; +} + +int WINAPI wWinMain(HINSTANCE instance, HINSTANCE previous_instance, + wchar_t *command_line, int show) +{ + wchar_t git_command_line[32768]; + size_t size = sizeof(git_command_line) / sizeof(wchar_t); + const wchar_t *needs_quotes = L""; + int slash = 0, i; + + STARTUPINFO startup_info = { + .dwFlags = STARTF_USESHOWWINDOW, + .wShowWindow = SW_HIDE, + }; + PROCESS_INFORMATION process_info = { 0 }; + DWORD creation_flags = CREATE_UNICODE_ENVIRONMENT | + CREATE_NEW_CONSOLE | CREATE_NO_WINDOW; + DWORD exit_code; + + /* First, determine the full path of argv[0] */ + for (i = 0; _wpgmptr[i]; i++) + if (_wpgmptr[i] == L' ') + needs_quotes = L"\""; + else if (_wpgmptr[i] == L'\\') + slash = i; + + if (slash + 11 >= sizeof(git_command_line) / sizeof(wchar_t)) + return 127; /* Too long path */ + + /* If it is in Git's exec path, add the bin/ directory to the PATH */ + extend_path(_wpgmptr, slash); + + /* Then, add the full path of `git.exe` as argv[0] */ + i = swprintf_s(git_command_line, size, L"%ls%.*ls\\git.exe%ls", + needs_quotes, slash, _wpgmptr, needs_quotes); + if (i < 0) + return 127; /* Too long path */ + + if (*command_line) { + /* Now, append the command-line arguments */ + i = swprintf_s(git_command_line + i, size - i, + L" %ls", command_line); + if (i < 0) + return 127; + } + + startup_info.cb = sizeof(STARTUPINFO); + + startup_info.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + startup_info.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + startup_info.hStdError = GetStdHandle(STD_ERROR_HANDLE); + + if (!CreateProcess(NULL, /* infer argv[0] from the command line */ + git_command_line, /* modified command line */ + NULL, /* inherit process handles? */ + NULL, /* inherit thread handles? */ + FALSE, /* handles inheritable? */ + creation_flags, + NULL, /* use this process' environment */ + NULL, /* use this process' working directory */ + &startup_info, &process_info)) + return 129; /* could not start */ + WaitForSingleObject(process_info.hProcess, INFINITE); + if (!GetExitCodeProcess(process_info.hProcess, &exit_code)) { + CloseHandle(process_info.hProcess); + return 130; /* Could not determine exit code? */ + } + + return (int)exit_code; +} diff --git a/config.mak.uname b/config.mak.uname index 198ab1e58f..3da8007cc4 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -481,6 +481,8 @@ else endif X = .exe + EXTRA_PROGRAMS += headless-git$X + compat/msvc.o: compat/msvc.c compat/mingw.c GIT-CFLAGS endif ifeq ($(uname_S),Interix) @@ -622,6 +624,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) RC = windres -O coff NATIVE_CRLF = YesPlease X = .exe + EXTRA_PROGRAMS += headless-git$X ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) htmldir = doc/git/html/ prefix = diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index c151dd7257..f4ee494248 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -642,6 +642,9 @@ if(WIN32) else() message(FATAL_ERROR "Unhandled compiler: ${CMAKE_C_COMPILER_ID}") endif() + + add_executable(headless-git ${CMAKE_SOURCE_DIR}/compat/win32/headless.c) + target_link_options(headless-git PUBLIC /NOLOGO /ENTRY:wWinMainCRTStartup /SUBSYSTEM:WINDOWS) elseif(UNIX) target_link_libraries(common-main pthread rt) endif() From f88145af9e8db31db5dfda2968b47e1d479ed667 Mon Sep 17 00:00:00 2001 From: Chris Djali Date: Thu, 27 Aug 2020 13:53:21 +0100 Subject: [PATCH 46/72] prompt: Avoid using unset variables After `set -u` in Bash, the Git prompt would not be displayed and the error `bash: GIT_PS1_COMPRESSSPARSESTATE: unbound variable` would be printed, due to `GIT_PS1_COMPRESSSPARSESTATE` being unset by default when it gets used. This change sets a default value of an empty string to use when the variable is unset to explicitly make the behaviour consistent between `set -u` and `set +u`. This fixes https://github.com/git-for-windows/git/issues/2799 Signed-off-by: Chris Djali Signed-off-by: Johannes Schindelin --- contrib/completion/git-prompt.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/completion/git-prompt.sh b/contrib/completion/git-prompt.sh index 4640a1535d..db7c0068fb 100644 --- a/contrib/completion/git-prompt.sh +++ b/contrib/completion/git-prompt.sh @@ -433,8 +433,8 @@ __git_ps1 () fi local sparse="" - if [ -z "${GIT_PS1_COMPRESSSPARSESTATE}" ] && - [ -z "${GIT_PS1_OMITSPARSESTATE}" ] && + if [ -z "${GIT_PS1_COMPRESSSPARSESTATE-}" ] && + [ -z "${GIT_PS1_OMITSPARSESTATE-}" ] && [ "$(git config --bool core.sparseCheckout)" = "true" ]; then sparse="|SPARSE" fi @@ -543,7 +543,7 @@ __git_ps1 () u="%${ZSH_VERSION+%}" fi - if [ -n "${GIT_PS1_COMPRESSSPARSESTATE}" ] && + if [ -n "${GIT_PS1_COMPRESSSPARSESTATE-}" ] && [ "$(git config --bool core.sparseCheckout)" = "true" ]; then h="?" fi From 0ded66f59c837b0404b1bd414b0404891c17f03b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 12 Aug 2020 15:06:17 +0000 Subject: [PATCH 47/72] git-gui: accommodate for intent-to-add files As of Git v2.28.0, the diff for files staged via `git add -N` marks them as new files. Git GUI was ill-prepared for that, and this patch teaches Git GUI about them. Please note that this will not even fix things with v2.28.0, as the `rp/apply-cached-with-i-t-a` patches are required on Git's side, too. This fixes https://github.com/git-for-windows/git/issues/2779 Signed-off-by: Johannes Schindelin Signed-off-by: Pratyush Yadav --- git-gui/git-gui.sh | 2 ++ git-gui/lib/diff.tcl | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh index 201524c34e..3fe9bd9f47 100755 --- a/git-gui/git-gui.sh +++ b/git-gui/git-gui.sh @@ -2087,6 +2087,7 @@ set all_icons(U$ui_index) file_merge set all_icons(T$ui_index) file_statechange set all_icons(_$ui_workdir) file_plain +set all_icons(A$ui_workdir) file_plain set all_icons(M$ui_workdir) file_mod set all_icons(D$ui_workdir) file_question set all_icons(U$ui_workdir) file_merge @@ -2113,6 +2114,7 @@ foreach i { {A_ {mc "Staged for commit"}} {AM {mc "Portions staged for commit"}} {AD {mc "Staged for commit, missing"}} + {AA {mc "Intended to be added"}} {_D {mc "Missing"}} {D_ {mc "Staged for removal"}} diff --git a/git-gui/lib/diff.tcl b/git-gui/lib/diff.tcl index 871ad488c2..36d3715f7b 100644 --- a/git-gui/lib/diff.tcl +++ b/git-gui/lib/diff.tcl @@ -582,7 +582,8 @@ proc apply_or_revert_hunk {x y revert} { if {$current_diff_side eq $ui_index} { set failed_msg [mc "Failed to unstage selected hunk."] lappend apply_cmd --reverse --cached - if {[string index $mi 0] ne {M}} { + set file_state [string index $mi 0] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -595,7 +596,8 @@ proc apply_or_revert_hunk {x y revert} { lappend apply_cmd --cached } - if {[string index $mi 1] ne {M}} { + set file_state [string index $mi 1] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -687,7 +689,8 @@ proc apply_or_revert_range_or_line {x y revert} { set failed_msg [mc "Failed to unstage selected line."] set to_context {+} lappend apply_cmd --reverse --cached - if {[string index $mi 0] ne {M}} { + set file_state [string index $mi 0] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -702,7 +705,8 @@ proc apply_or_revert_range_or_line {x y revert} { lappend apply_cmd --cached } - if {[string index $mi 1] ne {M}} { + set file_state [string index $mi 1] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } From fcf8d348779934fd34ed3cccce1fb4dec9a873b1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 3 Dec 2020 22:54:11 +0100 Subject: [PATCH 48/72] git maintenance: avoid console window in scheduled tasks on Windows We just introduced a helper to avoid showing a console window when the scheduled task runs `git.exe`. Let's actually use it. Signed-off-by: Johannes Schindelin Signed-off-by: Derrick Stolee --- builtin/gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/gc.c b/builtin/gc.c index f01f2fddfe..8d585dd8cf 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1795,7 +1795,7 @@ static int schtasks_schedule_task(const char *exec_path, enum schedule_priority "\n" "\n" "\n" - "\"%s\\git.exe\"\n" + "\"%s\\headless-git.exe\"\n" "--exec-path=\"%s\" for-each-repo --config=maintenance.repo maintenance run --schedule=%s\n" "\n" "\n" From 219a73ccb60b45616568af6698c626dcb5f3d5e1 Mon Sep 17 00:00:00 2001 From: Ian Bearman Date: Fri, 31 Jan 2020 15:37:27 -0800 Subject: [PATCH 49/72] vcxproj: support building Windows/ARM64 binaries Signed-off-by: Ian Bearman Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcxproj.pm | 23 ++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index d2584450ba..e06f300d0c 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -114,12 +114,21 @@ sub createProject { Release x64 + + Debug + ARM64 + + + Release + ARM64 + $uuid Win32Proj x86-windows - x64-windows + x64-windows + arm64-windows $cdup\\compat\\vcbuild\\vcpkg\\installed\\\$(VCPKGArch) \$(VCPKGArchDirectory)\\debug\\bin \$(VCPKGArchDirectory)\\debug\\lib @@ -140,7 +149,7 @@ sub createProject { $config_type - v140 + v142 ..\\ @@ -346,8 +355,10 @@ sub createGlueProject { print F << "EOM"; Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|ARM64 = Debug|ARM64 Debug|x64 = Debug|x64 Debug|x86 = Debug|x86 + Release|ARM64 = Release|ARM64 Release|x64 = Release|x64 Release|x86 = Release|x86 EndGlobalSection @@ -358,10 +369,14 @@ EOM foreach (@apps) { my $appname = $_; my $uuid = $build_structure{"APPS_${appname}_GUID"}; + print F "\t\t${uuid}.Debug|ARM64.ActiveCfg = Debug|ARM64\n"; + print F "\t\t${uuid}.Debug|ARM64.Build.0 = Debug|ARM64\n"; print F "\t\t${uuid}.Debug|x64.ActiveCfg = Debug|x64\n"; print F "\t\t${uuid}.Debug|x64.Build.0 = Debug|x64\n"; print F "\t\t${uuid}.Debug|x86.ActiveCfg = Debug|Win32\n"; print F "\t\t${uuid}.Debug|x86.Build.0 = Debug|Win32\n"; + print F "\t\t${uuid}.Release|ARM64.ActiveCfg = Release|ARM64\n"; + print F "\t\t${uuid}.Release|ARM64.Build.0 = Release|ARM64\n"; print F "\t\t${uuid}.Release|x64.ActiveCfg = Release|x64\n"; print F "\t\t${uuid}.Release|x64.Build.0 = Release|x64\n"; print F "\t\t${uuid}.Release|x86.ActiveCfg = Release|Win32\n"; @@ -370,10 +385,14 @@ EOM foreach (@libs) { my $libname = $_; my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + print F "\t\t${uuid}.Debug|ARM64.ActiveCfg = Debug|ARM64\n"; + print F "\t\t${uuid}.Debug|ARM64.Build.0 = Debug|ARM64\n"; print F "\t\t${uuid}.Debug|x64.ActiveCfg = Debug|x64\n"; print F "\t\t${uuid}.Debug|x64.Build.0 = Debug|x64\n"; print F "\t\t${uuid}.Debug|x86.ActiveCfg = Debug|Win32\n"; print F "\t\t${uuid}.Debug|x86.Build.0 = Debug|Win32\n"; + print F "\t\t${uuid}.Release|ARM64.ActiveCfg = Release|ARM64\n"; + print F "\t\t${uuid}.Release|ARM64.Build.0 = Release|ARM64\n"; print F "\t\t${uuid}.Release|x64.ActiveCfg = Release|x64\n"; print F "\t\t${uuid}.Release|x64.Build.0 = Release|x64\n"; print F "\t\t${uuid}.Release|x86.ActiveCfg = Release|Win32\n"; From aad625497fea8f57844432eec1214ca95159d9b0 Mon Sep 17 00:00:00 2001 From: Ian Bearman Date: Fri, 31 Jan 2020 16:00:25 -0800 Subject: [PATCH 50/72] vcbuild: install ARM64 dependencies when building ARM64 binaries Co-authored-by: Dennis Ameling Signed-off-by: Ian Bearman Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- compat/vcbuild/README | 6 +++++- compat/vcbuild/vcpkg_copy_dlls.bat | 7 ++++++- compat/vcbuild/vcpkg_install.bat | 9 +++++++-- contrib/buildsystems/Generators/Vcxproj.pm | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/compat/vcbuild/README b/compat/vcbuild/README index 51fb083dbb..432295d7e3 100644 --- a/compat/vcbuild/README +++ b/compat/vcbuild/README @@ -6,7 +6,11 @@ The Steps to Build Git with VS2015 or VS2017 from the command line. Prompt or from an SDK bash window: $ cd - $ ./compat/vcbuild/vcpkg_install.bat + $ ./compat/vcbuild/vcpkg_install.bat x64-windows + + or + + $ ./compat/vcbuild/vcpkg_install.bat arm64-windows The vcpkg tools and all of the third-party sources will be installed in this folder: diff --git a/compat/vcbuild/vcpkg_copy_dlls.bat b/compat/vcbuild/vcpkg_copy_dlls.bat index 13661c14f8..8bea0cbf83 100644 --- a/compat/vcbuild/vcpkg_copy_dlls.bat +++ b/compat/vcbuild/vcpkg_copy_dlls.bat @@ -15,7 +15,12 @@ REM ================================================================ @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD cd %cwd% - SET arch=x64-windows + SET arch=%2 + IF NOT DEFINED arch ( + echo defaulting to 'x64-windows`. Invoke %0 with 'x86-windows', 'x64-windows', or 'arm64-windows' + set arch=x64-windows + ) + SET inst=%cwd%vcpkg\installed\%arch% IF [%1]==[release] ( diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index 8330d8120f..cacef18c11 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -31,6 +31,12 @@ REM ================================================================ SETLOCAL EnableDelayedExpansion + SET arch=%1 + IF NOT DEFINED arch ( + echo defaulting to 'x64-windows`. Invoke %0 with 'x86-windows', 'x64-windows', or 'arm64-windows' + set arch=x64-windows + ) + @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD cd %cwd% @@ -55,9 +61,8 @@ REM ================================================================ echo Successfully installed %cwd%vcpkg\vcpkg.exe :install_libraries - SET arch=x64-windows - echo Installing third-party libraries... + echo Installing third-party libraries(%arch%)... FOR %%i IN (zlib expat libiconv openssl libssh2 curl) DO ( cd %cwd%vcpkg IF NOT EXIST "packages\%%i_%arch%" CALL :sub__install_one %%i diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index e06f300d0c..16acf91d94 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -193,7 +193,7 @@ EOM Initialize VCPKG del "$cdup\\compat\\vcbuild\\vcpkg" - call "$cdup\\compat\\vcbuild\\vcpkg_install.bat" + call "$cdup\\compat\\vcbuild\\vcpkg_install.bat" \$(VCPKGArch) EOM } From d9214e9e661d5abebbf137518ee71b0a7a5a56ec Mon Sep 17 00:00:00 2001 From: Ian Bearman Date: Tue, 4 Feb 2020 10:34:40 -0800 Subject: [PATCH 51/72] vcbuild: add an option to install individual 'features' In this context, a "feature" is a dependency combined with its own dependencies. Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- compat/vcbuild/vcpkg_install.bat | 35 +++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index cacef18c11..8da212487a 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -85,14 +85,47 @@ REM ================================================================ :sub__install_one echo Installing package %1... + call :%1_features + REM vcpkg may not be reliable on slow, intermittent or proxy REM connections, see e.g. REM https://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/4a8f7be5-5e15-4213-a7bb-ddf424a954e6/winhttpsendrequest-ends-with-12002-errorhttptimeout-after-21-seconds-no-matter-what-timeout?forum=windowssdk REM which explains the hidden 21 second timeout REM (last post by Dave : Microsoft - Windows Networking team) - .\vcpkg.exe install %1:%arch% + .\vcpkg.exe install %1%features%:%arch% IF ERRORLEVEL 1 ( EXIT /B 1 ) echo Finished %1 goto :EOF + +:: +:: features for each vcpkg to install +:: there should be an entry here for each package to install +:: 'set features=' means use the default otherwise +:: 'set features=[comma-delimited-feature-set]' is the syntax +:: + +:zlib_features +set features= +goto :EOF + +:expat_features +set features= +goto :EOF + +:libiconv_features +set features= +goto :EOF + +:openssl_features +set features= +goto :EOF + +:libssh2_features +set features= +goto :EOF + +:curl_features +set features=[core,openssl] +goto :EOF From 0904c9e3c75a44e8440b1671e2f6703963731b19 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Fri, 4 Dec 2020 14:11:34 +0100 Subject: [PATCH 52/72] cmake: allow building for Windows/ARM64 Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- contrib/buildsystems/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index c151dd7257..ba65ca4b4c 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -52,9 +52,9 @@ if(WIN32) set(VCPKG_DIR "${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg") if(MSVC AND NOT EXISTS ${VCPKG_DIR}) message("Initializing vcpkg and building the Git's dependencies (this will take a while...)") - execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat) + execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat ${VCPKG_ARCH}) endif() - list(APPEND CMAKE_PREFIX_PATH "${VCPKG_DIR}/installed/x64-windows") + list(APPEND CMAKE_PREFIX_PATH "${VCPKG_DIR}/installed/${VCPKG_ARCH}") # In the vcpkg edition, we need this to be able to link to libcurl set(CURL_NO_CURL_CMAKE ON) @@ -961,7 +961,7 @@ file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_GETTEXT='${NO_GETTEXT}'\n" file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "RUNTIME_PREFIX='${RUNTIME_PREFIX}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_PYTHON='${NO_PYTHON}'\n") if(WIN32) - file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "PATH=\"$PATH:$TEST_DIRECTORY/../compat/vcbuild/vcpkg/installed/x64-windows/bin\"\n") + file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "PATH=\"$PATH:$TEST_DIRECTORY/../compat/vcbuild/vcpkg/installed/${VCPKG_ARCH}/bin\"\n") endif() #Make the tests work when building out of the source tree From f5479acdf7aeaf58b5bab920aac327f75764e705 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 6 Oct 2019 18:40:55 +0100 Subject: [PATCH 53/72] vcpkg_install: detect lack of Git The vcpkg_install batch file depends on the availability of a working Git on the CMD path. This may not be present if the user has selected the 'bash only' option during Git-for-Windows install. Detect and tell the user about their lack of a working Git in the CMD window. Fixes #2348. A separate PR https://github.com/git-for-windows/build-extra/pull/258 now highlights the recommended path setting during install. Signed-off-by: Philip Oakley --- compat/vcbuild/vcpkg_install.bat | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index ebd0bad242..bcbbf536af 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -36,6 +36,13 @@ REM ================================================================ dir vcpkg\vcpkg.exe >nul 2>nul && GOTO :install_libraries + git.exe version 2>nul + IF ERRORLEVEL 1 ( + echo "***" + echo "Git not found. Please adjust your CMD path or Git install option." + echo "***" + EXIT /B 1 ) + echo Fetching vcpkg in %cwd%vcpkg git.exe clone https://github.com/Microsoft/vcpkg vcpkg IF ERRORLEVEL 1 ( EXIT /B 1 ) From 51d2f88c3db4473365b9f6d14d1579747495c89c Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 29 Nov 2020 00:12:26 +0100 Subject: [PATCH 54/72] ci(vs-build) also build Windows/ARM64 artifacts There are no Windows/ARM64 agents in GitHub Actions yet, therefore we just skip adjusting the `vs-test` job for now. Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a3fbbe6398..91a05fcd1c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -169,6 +169,9 @@ jobs: NO_PERL: 1 GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest + strategy: + matrix: + arch: [x64, arm64] steps: - uses: actions/checkout@v1 - name: download git-sdk-64-minimal @@ -200,15 +203,15 @@ jobs: - name: copy dlls to root shell: powershell run: | - & compat\vcbuild\vcpkg_copy_dlls.bat release + & compat\vcbuild\vcpkg_copy_dlls.bat release ${{ matrix.arch }}-windows if (!$?) { exit(1) } - name: generate Visual Studio solution shell: bash run: | - cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DMSGFMT_EXE=`pwd`/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/${{ matrix.arch }}-windows \ + -DMSGFMT_EXE=`pwd`/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows - name: MSBuild - run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 + run: msbuild git.sln -property:Configuration=Release -property:Platform=${{ matrix.arch }} -maxCpuCount:4 -property:PlatformToolset=v142 - name: bundle artifact tar shell: powershell env: @@ -222,7 +225,7 @@ jobs: - name: upload build artifacts uses: actions/upload-artifact@v1 with: - name: vs-artifacts + name: vs-artifacts-${{ matrix.arch }} path: artifacts vs-test: runs-on: windows-latest @@ -241,7 +244,7 @@ jobs: - name: download build artifacts uses: actions/download-artifact@v1 with: - name: vs-artifacts + name: vs-artifacts-x64 path: ${{github.workspace}} - name: extract build artifacts shell: bash From ab82245ff701a53a84025e9dc533df513fa448f6 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 6 Oct 2019 18:43:57 +0100 Subject: [PATCH 55/72] vcpkg_install: add comment regarding slow network connections The vcpkg downloads may not succeed. Warn careful readers of the time out. A simple retry will usually resolve the issue. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/vcbuild/vcpkg_install.bat | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index bcbbf536af..8330d8120f 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -80,6 +80,12 @@ REM ================================================================ :sub__install_one echo Installing package %1... + REM vcpkg may not be reliable on slow, intermittent or proxy + REM connections, see e.g. + REM https://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/4a8f7be5-5e15-4213-a7bb-ddf424a954e6/winhttpsendrequest-ends-with-12002-errorhttptimeout-after-21-seconds-no-matter-what-timeout?forum=windowssdk + REM which explains the hidden 21 second timeout + REM (last post by Dave : Microsoft - Windows Networking team) + .\vcpkg.exe install %1:%arch% IF ERRORLEVEL 1 ( EXIT /B 1 ) From 01c5ef63b304e5d0bb3b2a04886e8de7fd59620c Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 6 Dec 2020 18:39:26 +0100 Subject: [PATCH 56/72] Add schannel to curl installation Signed-off-by: Dennis Ameling --- compat/vcbuild/vcpkg_install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index 8da212487a..575c65c20b 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -127,5 +127,5 @@ set features= goto :EOF :curl_features -set features=[core,openssl] +set features=[core,openssl,schannel] goto :EOF From a0d49a459751ad11e1b216d2e312e9f2e9384be7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 10 Apr 2020 23:39:40 +0200 Subject: [PATCH 57/72] Add a GitHub workflow to generate Git for Windows' Pacman package Git for Windows uses MSYS2 as base system, and therefore the Git binaries are bundled as Pacman package. This workflow allows building the 64-bit version of this package (which is called `mingw-w64-x86_64-git`). Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 141 ++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 .github/workflows/git-artifacts.yml diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml new file mode 100644 index 0000000000..14af08ad95 --- /dev/null +++ b/.github/workflows/git-artifacts.yml @@ -0,0 +1,141 @@ +name: mingw-w64-x86_64-git + +on: + # This workflow can be triggered manually in the Actions tab, see + # https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/ + - workflow_dispatch + +env: + HOME: "${{github.workspace}}\\home" + MSYSTEM: MINGW64 + USERPROFILE: "${{github.workspace}}\\home" + +jobs: + bundle-artifacts: + runs-on: windows-latest + steps: + - name: Configure user + shell: bash + run: + USER_NAME="${{github.actor}}" && + USER_EMAIL="${{github.actor}}@users.noreply.github.com" && + mkdir "$HOME" && + git config --global user.name "$USER_NAME" && + git config --global user.email "$USER_EMAIL" && + echo "PACKAGER=$USER_NAME <$USER_EMAIL>" >>$GITHUB_ENV + - name: Download git-sdk-64-build-installers + shell: bash + run: | + # Use Git Bash to download and unpack the artifact + + ## Get artifact + urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds + id=$(curl "$urlbase?definitions=29&statusFilter=completed&resultFilter=succeeded&\$top=1" | + jq -r '.value[0].id') + download_url=$(curl "$urlbase/$id/artifacts" | + jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl') + + curl -o artifacts.zip "$download_url" + + ## Unpack artifact + unzip artifacts.zip + - name: Clone build-extra + shell: bash + run: | + d=git-sdk-64-build-installers/usr/src/build-extra && + git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d + - name: Generate bundle artifacts + shell: powershell + run: | + & .\git-sdk-64-build-installers\git-cmd.exe --command=usr\bin\bash.exe -lc @" + printf '#!/bin/sh\n\nexec /mingw64/bin/git.exe "`$@"\n' >/usr/bin/git && + mkdir -p bundle-artifacts && + + git -c init.defaultBranch=main init --bare && + git remote add -f origin https://github.com/git-for-windows/git && + git fetch https://github.com/${{github.repository}} ${{github.ref}}:${{github.ref}} && + + tag_name=\"`$(git describe --match 'v[0-9]*' FETCH_HEAD)-`$(date +%Y%m%d%H%M%S)\" && + echo \"prerelease-`${tag_name#v}\" >bundle-artifacts/ver && + echo \"`${tag_name#v}\" >bundle-artifacts/display_version && + echo \"`$tag_name\" >bundle-artifacts/next_version && + git tag -m \"Snapshot build\" \"`$tag_name\" FETCH_HEAD && + git bundle create bundle-artifacts/git.bundle origin/main..\"`$tag_name\" && + + sh -x /usr/src/build-extra/please.sh mention feature \"Snapshot of `$(git show -s --pretty='tformat:%h (%s, %ad)' --date=short FETCH_HEAD)\" && + git -C /usr/src/build-extra bundle create \"`$PWD/bundle-artifacts/build-extra.bundle\" origin/main..main + "@ + - name: 'Publish Pipeline Artifact: bundle-artifacts' + uses: actions/upload-artifact@v1 + with: + name: bundle-artifacts + path: bundle-artifacts + pkg-x86_64: + runs-on: windows-latest + needs: bundle-artifacts + steps: + - name: Configure user + shell: bash + run: + USER_NAME="${{github.actor}}" && + USER_EMAIL="${{github.actor}}@users.noreply.github.com" && + mkdir "$HOME" && + git config --global user.name "$USER_NAME" && + git config --global user.email "$USER_EMAIL" && + echo "PACKAGER=$USER_NAME <$USER_EMAIL>" >>$GITHUB_ENV + - name: Download git-sdk-64-makepkg-git + shell: bash + run: | + # Use Git Bash to download and unpack the artifact + + ## Get artifact + urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds + id=${{ needs.bundle-artifacts.outputs.latest-sdk64-extra-build-id }} + download_url="$(curl "$urlbase/$id/artifacts" | + jq -r '.value[] | select(.name == "git-sdk-64-makepkg-git").resource.downloadUrl')" + + curl -o artifacts.zip "$download_url" + + ## Unpack artifact + unzip artifacts.zip + - name: Download bundle-artifacts + uses: actions/download-artifact@v1 + with: + name: bundle-artifacts + path: bundle-artifacts + - name: Clone and update build-extra + shell: bash + run: | + d=git-sdk-64-makepkg-git/usr/src/build-extra && + git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d && + git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main + - name: Check out git/git + shell: bash + run: | + git -c init.defaultBranch=main init && + git remote add -f origin https://github.com/git-for-windows/git && + git fetch --tags bundle-artifacts/git.bundle $(cat bundle-artifacts/next_version) && + git reset --hard $(cat bundle-artifacts/next_version) + - name: Build mingw-w64-x86_64-git + shell: powershell + run: | + & git-sdk-64-makepkg-git\usr\bin\sh.exe -lc @" + set -x + # Let `cv2pdb` find the DLLs + PATH=\"`$PATH:/C/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64\" + type -p mspdb140.dll || exit 1 + sh -x /usr/src/build-extra/please.sh build-mingw-w64-git --only-64-bit --build-src-pkg -o artifacts HEAD && + cp bundle-artifacts/ver artifacts/ && + + b=`$PWD/artifacts && + version=`$(cat bundle-artifacts/next_version) && + (cd /usr/src/MINGW-packages/mingw-w64-git && + cp PKGBUILD.`$version PKGBUILD && + git commit -s -m \"mingw-w64-git: new version (`$version)\" PKGBUILD && + git bundle create \"`$b\"/MINGW-packages.bundle origin/main..main) + "@ + - name: Publish mingw-w64-x86_64-git + uses: actions/upload-artifact@v1 + with: + name: pkg-x86_64 + path: artifacts From e3e6e0bf24496d55149cd1297ab5db9119a3985a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 12 Apr 2020 00:24:49 +0200 Subject: [PATCH 58/72] git-artifacts: if GPG secrets are available, use them This expects the `GPGKEY` and `PRIVGPGKEY` secrets to be set in the respective GitHub repository. The `GPGKEY` value should be of the form --passphrase --yes --batch --no-tty --pinentry-mode loopback --digest-algo SHA256 and the `PRIVGPGKEY` should be generated via gpg --export-secret-keys | base64 | tr '\n' % Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 44 ++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 14af08ad95..7272b08221 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -6,6 +6,7 @@ on: - workflow_dispatch env: + GPG_OPTIONS: "--batch --yes --no-tty --list-options no-show-photos --verify-options no-show-photos --pinentry-mode loopback" HOME: "${{github.workspace}}\\home" MSYSTEM: MINGW64 USERPROFILE: "${{github.workspace}}\\home" @@ -44,7 +45,21 @@ jobs: run: | d=git-sdk-64-build-installers/usr/src/build-extra && git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d + - name: Prepare home directory for GPG signing + if: env.GPGKEY != '' + shell: bash + run: | + echo '${{secrets.PRIVGPGKEY}}' | tr % '\n' | gpg $GPG_OPTIONS --import && + mkdir -p home && + git config --global gpg.program "$PWD/git-sdk-64-build-installers/usr/src/build-extra/gnupg-with-gpgkey.sh" && + info="$(gpg --list-keys --with-colons "${GPGKEY%% *}" | cut -d : -f 1,10 | sed -n '/^uid/{s|uid:||p;q}')" && + git config --global user.name "${info% <*}" && + git config --global user.email "<${info#*<}" + env: + GPGKEY: ${{secrets.GPGKEY}} - name: Generate bundle artifacts + env: + GPGKEY: ${{secrets.GPGKEY}} shell: powershell run: | & .\git-sdk-64-build-installers\git-cmd.exe --command=usr\bin\bash.exe -lc @" @@ -59,12 +74,16 @@ jobs: echo \"prerelease-`${tag_name#v}\" >bundle-artifacts/ver && echo \"`${tag_name#v}\" >bundle-artifacts/display_version && echo \"`$tag_name\" >bundle-artifacts/next_version && - git tag -m \"Snapshot build\" \"`$tag_name\" FETCH_HEAD && + git tag `$(test -z \"`$GPGKEY\" || echo \" -s\") -m \"Snapshot build\" \"`$tag_name\" FETCH_HEAD && git bundle create bundle-artifacts/git.bundle origin/main..\"`$tag_name\" && sh -x /usr/src/build-extra/please.sh mention feature \"Snapshot of `$(git show -s --pretty='tformat:%h (%s, %ad)' --date=short FETCH_HEAD)\" && git -C /usr/src/build-extra bundle create \"`$PWD/bundle-artifacts/build-extra.bundle\" origin/main..main "@ + - name: Clean up temporary files + if: always() + shell: bash + run: rm -rf home - name: 'Publish Pipeline Artifact: bundle-artifacts' uses: actions/upload-artifact@v1 with: @@ -116,7 +135,19 @@ jobs: git remote add -f origin https://github.com/git-for-windows/git && git fetch --tags bundle-artifacts/git.bundle $(cat bundle-artifacts/next_version) && git reset --hard $(cat bundle-artifacts/next_version) + - name: Prepare home directory for GPG signing + if: env.GPGKEY != '' + shell: bash + run: | + echo '${{secrets.PRIVGPGKEY}}' | tr % '\n' | gpg $GPG_OPTIONS --import && + info="$(gpg --list-keys --with-colons "${GPGKEY%% *}" | cut -d : -f 1,10 | sed -n '/^uid/{s|uid:||p;q}')" && + git config --global user.name "${info% <*}" && + git config --global user.email "<${info#*<}" + env: + GPGKEY: ${{secrets.GPGKEY}} - name: Build mingw-w64-x86_64-git + env: + GPGKEY: "${{secrets.GPGKEY}}" shell: powershell run: | & git-sdk-64-makepkg-git\usr\bin\sh.exe -lc @" @@ -126,6 +157,13 @@ jobs: type -p mspdb140.dll || exit 1 sh -x /usr/src/build-extra/please.sh build-mingw-w64-git --only-64-bit --build-src-pkg -o artifacts HEAD && cp bundle-artifacts/ver artifacts/ && + if test -n \"`$GPGKEY\" + then + for tar in artifacts/*.tar* + do + /usr/src/build-extra/gnupg-with-gpgkey.sh --detach-sign --no-armor `$tar + done + fi && b=`$PWD/artifacts && version=`$(cat bundle-artifacts/next_version) && @@ -134,6 +172,10 @@ jobs: git commit -s -m \"mingw-w64-git: new version (`$version)\" PKGBUILD && git bundle create \"`$b\"/MINGW-packages.bundle origin/main..main) "@ + - name: Clean up temporary files + if: always() + shell: bash + run: rm -rf home - name: Publish mingw-w64-x86_64-git uses: actions/upload-artifact@v1 with: From 28bc2c7ace56e0262d35f52ba6252ca6a574e1a7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 12 Apr 2020 00:34:56 +0200 Subject: [PATCH 59/72] git-artifacts: also code-sign, if configured via the secrets When the secrets `CODESIGN_P12` and `CODESIGN_PASS` are set, the workflow will now code-sign the `.exe` files contained in the package. This should help with a few anti-malware programs, at least when the certificate saw some action and gained trust. Note: `CODESIGN_P12` needs to be generated via cat .p12 | base64 | tr '\n' % Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 7272b08221..7e86de31ac 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -135,6 +135,18 @@ jobs: git remote add -f origin https://github.com/git-for-windows/git && git fetch --tags bundle-artifacts/git.bundle $(cat bundle-artifacts/next_version) && git reset --hard $(cat bundle-artifacts/next_version) + - name: Prepare home directory for code-signing + env: + CODESIGN_P12: ${{secrets.CODESIGN_P12}} + CODESIGN_PASS: ${{secrets.CODESIGN_PASS}} + if: env.CODESIGN_P12 != '' && env.CODESIGN_PASS != '' + shell: bash + run: | + cd home && + mkdir -p .sig && + echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >.sig/codesign.p12 && + echo -n "$CODESIGN_PASS" >.sig/codesign.pass + git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"' - name: Prepare home directory for GPG signing if: env.GPGKEY != '' shell: bash From eb1720bb568fd24278d1a80985d89a132a982e42 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 12 Apr 2020 00:43:19 +0200 Subject: [PATCH 60/72] git-artifacts: also build the installer While at it, we might just as well build the Git for Windows installer ;-) Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 75 ++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 7e86de31ac..2b23485acc 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -1,4 +1,4 @@ -name: mingw-w64-x86_64-git +name: git-artifacts on: # This workflow can be triggered manually in the Actions tab, see @@ -193,3 +193,76 @@ jobs: with: name: pkg-x86_64 path: artifacts + installer-x86_64: + runs-on: windows-latest + needs: pkg-x86_64 + env: + MSYSTEM: MINGW64 + steps: + - name: Download pkg-x86_64 + uses: actions/download-artifact@v1 + with: + name: pkg-x86_64 + path: pkg-x86_64 + - name: Download bundle-artifacts + uses: actions/download-artifact@v1 + with: + name: bundle-artifacts + path: bundle-artifacts + - name: Download git-sdk-64-build-installers + shell: bash + run: | + # Use Git Bash to download and unpack the artifact + + ## Get artifact + urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds + id=${{ needs.pkg.outputs.latest-sdk64-extra-build-id }} + download_url="$(curl "$urlbase/$id/artifacts" | + jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl')" + + curl -o artifacts.zip "$download_url" + + ## Unpack artifact + unzip artifacts.zip + - name: Clone and update build-extra + shell: bash + run: | + d=git-sdk-64-build-installers/usr/src/build-extra && + git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d && + git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main + - name: Prepare home directory for code-signing + env: + CODESIGN_P12: ${{secrets.CODESIGN_P12}} + CODESIGN_PASS: ${{secrets.CODESIGN_PASS}} + if: (matrix.artifact.name == 'installer' || matrix.artifact.name == 'portable') && env.CODESIGN_P12 != '' && env.CODESIGN_PASS != '' + shell: bash + run: | + mkdir -p home/.sig && + echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >home/.sig/codesign.p12 && + echo -n "$CODESIGN_PASS" >home/.sig/codesign.pass && + git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"' + - name: Build 64-bit installer + shell: powershell + run: | + & .\git-sdk-64-build-installers\usr\bin\bash.exe -lc @" + set -x + /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-x86_64/ver) -o artifacts --installer --pkg=pkg-x86_64/mingw-w64-x86_64-git-[0-9]*.tar.xz --pkg=pkg-x86_64/mingw-w64-x86_64-git-doc-html-[0-9]*.tar.xz && + openssl dgst -sha256 artifacts/Git-*.exe | sed \"s/.* //\" >artifacts/sha-256.txt && + cp /usr/src/build-extra/installer/package-versions.txt artifacts/ && + + a=`$PWD/artifacts && + p=`$PWD/pkg-x86_64 && + (cd /usr/src/build-extra && + mkdir -p cached-source-packages && + cp \"`$p\"/*-pdb* cached-source-packages/ && + GIT_CONFIG_PARAMETERS=\"'windows.sdk64.path='\" ./please.sh bundle_pdbs --arch=x86_64 --directory=\"`$a\" installer/package-versions.txt) + "@ + - name: Clean up temporary files + if: always() + shell: bash + run: rm -rf home + - name: Publish installer-x86_64 + uses: actions/upload-artifact@v1 + with: + name: installer-x86_64 + path: artifacts From e24bff717aec8994ce92fbb69076ca7f47568808 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Fri, 8 Jan 2021 23:49:21 +0100 Subject: [PATCH 61/72] cmake(install): include vcpkg dlls Signed-off-by: Dennis Ameling --- contrib/buildsystems/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index c151dd7257..4c6870f976 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -58,6 +58,10 @@ if(WIN32) # In the vcpkg edition, we need this to be able to link to libcurl set(CURL_NO_CURL_CMAKE ON) + + # Copy the necessary vcpkg DLLs (like iconv) to the install dir + set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) + set(CMAKE_TOOLCHAIN_FILE ${VCPKG_DIR}/scripts/buildsystems/vcpkg.cmake CACHE STRING "Vcpkg toolchain file") endif() find_program(SH_EXE sh PATHS "C:/Program Files/Git/bin") From becf528744c132c6c531d7e87449a9dc5320fc21 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 12 Apr 2020 00:51:00 +0200 Subject: [PATCH 62/72] git-artifacts: also build portable, mingit and mingit-busybox ... because we can. Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 41 ++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 2b23485acc..3a3cf0b720 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -193,9 +193,26 @@ jobs: with: name: pkg-x86_64 path: artifacts - installer-x86_64: + artifacts: runs-on: windows-latest - needs: pkg-x86_64 + needs: pkg + strategy: + matrix: + artifact: + - name: installer + - name: portable + fileprefix: PortableGit + fileextension: exe + - name: archive + fileprefix: Git + fileextension: tar.bz2 + - name: mingit + fileprefix: MinGit + fileextension: zip + - name: mingit-busybox + fileprefix: MinGit + fileextension: zip + fail-fast: false env: MSYSTEM: MINGW64 steps: @@ -241,13 +258,23 @@ jobs: echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >home/.sig/codesign.p12 && echo -n "$CODESIGN_PASS" >home/.sig/codesign.pass && git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"' - - name: Build 64-bit installer + - name: Build 64-bit ${{matrix.artifact.name}} shell: powershell run: | & .\git-sdk-64-build-installers\usr\bin\bash.exe -lc @" set -x - /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-x86_64/ver) -o artifacts --installer --pkg=pkg-x86_64/mingw-w64-x86_64-git-[0-9]*.tar.xz --pkg=pkg-x86_64/mingw-w64-x86_64-git-doc-html-[0-9]*.tar.xz && - openssl dgst -sha256 artifacts/Git-*.exe | sed \"s/.* //\" >artifacts/sha-256.txt && + /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-x86_64/ver) -o artifacts --${{matrix.artifact.name}} --pkg=pkg-x86_64/mingw-w64-x86_64-git-[0-9]*.tar.xz --pkg=pkg-x86_64/mingw-w64-x86_64-git-doc-html-[0-9]*.tar.xz && + if test portable = '${{matrix.artifact.name}}' && test -n \"`$(git config alias.signtool)\" + then + git signtool artifacts/PortableGit-*.exe + fi && + openssl dgst -sha256 artifacts/${{matrix.artifact.fileprefix}}-*.${{matrix.artifact.fileextension}} | sed \"s/.* //\" >artifacts/sha-256.txt + "@ + - name: Copy package-versions and pdbs + if: matrix.artifact.name == 'installer' + shell: powershell + run: | + & .\git-sdk-64-build-installers\usr\bin\bash.exe -lc @" cp /usr/src/build-extra/installer/package-versions.txt artifacts/ && a=`$PWD/artifacts && @@ -261,8 +288,8 @@ jobs: if: always() shell: bash run: rm -rf home - - name: Publish installer-x86_64 + - name: Publish ${{matrix.artifact.name}}-x86_64 uses: actions/upload-artifact@v1 with: - name: installer-x86_64 + name: ${{matrix.artifact.name}}-x86_64 path: artifacts From 17eb505b3cde38d9a9f51713a69f317064d78872 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 24 Feb 2021 22:10:23 +0100 Subject: [PATCH 63/72] cmake: work around recent `vcpkg` update In f1f5dff9e7c (cmake: installation support for git, 2020-06-26), we added support for the CMake equivalent of `make install` that makes use of multiple targets within one invocation of `install(TARGETS ...). This is totally fine according to CMake's documentation at https://cmake.org/cmake/help/latest/command/install.html#command:install). And this was still fine even after ecf7ee3cd5a (cmake(install): include vcpkg dlls, 2021-01-08) where we started to use `vcpkg`'s CMake scripts to ensure that the `.dll` files of Git's dependencies are also installed. However, with a recent update of `vcpkg`, the handling of local dependencies was changed in a way that is incompatible with our CMake definition. The symptom looks like this: CMake Error at /compat/vcbuild/vcpkg/scripts/buildsystems/vcpkg.cmake:734 (get_target_property): get_target_property() called with non-existent target "git;git-shell". Call Stack (most recent call first): /compat/vcbuild/vcpkg/scripts/buildsystems/vcpkg.cmake:784 (x_vcpkg_install_local_dependencies) CMakeLists.txt:821 (install) The apparent reason for this breakage is that the `vcpkg` changes in https://github.com/microsoft/vcpkg/commit/1bb5ea10a3 no longer allows an arbitrary number of targets to be specified in `install(TARGETS ...)`. Let's work around this by feeding the target individually to the `install(TARGETS ...)` function. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/CMakeLists.txt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 4c6870f976..cf34096d9c 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -811,15 +811,19 @@ list(TRANSFORM git_shell_scripts PREPEND "${CMAKE_BINARY_DIR}/") list(TRANSFORM git_perl_scripts PREPEND "${CMAKE_BINARY_DIR}/") #install -install(TARGETS git git-shell +foreach(program ${PROGRAMS_BUILT}) +if(${program} STREQUAL git OR ${program} STREQUAL git-shell) +install(TARGETS ${program} RUNTIME DESTINATION bin) +else() +install(TARGETS ${program} + RUNTIME DESTINATION libexec/git-core) +endif() +endforeach() + install(PROGRAMS ${CMAKE_BINARY_DIR}/git-cvsserver DESTINATION bin) -list(REMOVE_ITEM PROGRAMS_BUILT git git-shell) -install(TARGETS ${PROGRAMS_BUILT} - RUNTIME DESTINATION libexec/git-core) - set(bin_links git-receive-pack git-upload-archive git-upload-pack) From 9cb7a41c7878236d38217dac76e998e24330eb60 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 12 Apr 2020 00:57:23 +0200 Subject: [PATCH 64/72] git-artifacts: also build 32-bit versions Just in case that we need to generate those real quick. Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 70 +++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 3a3cf0b720..05f5ccaaf0 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -89,9 +89,18 @@ jobs: with: name: bundle-artifacts path: bundle-artifacts - pkg-x86_64: + pkg: runs-on: windows-latest needs: bundle-artifacts + strategy: + matrix: + arch: + - name: x86_64 + bitness: 64 + bin: /amd64 + - name: i686 + bitness: 32 + bin: '' steps: - name: Configure user shell: bash @@ -157,7 +166,7 @@ jobs: git config --global user.email "<${info#*<}" env: GPGKEY: ${{secrets.GPGKEY}} - - name: Build mingw-w64-x86_64-git + - name: Build mingw-w64-${{matrix.arch.name}}-git env: GPGKEY: "${{secrets.GPGKEY}}" shell: powershell @@ -165,9 +174,9 @@ jobs: & git-sdk-64-makepkg-git\usr\bin\sh.exe -lc @" set -x # Let `cv2pdb` find the DLLs - PATH=\"`$PATH:/C/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64\" + PATH=\"`$PATH:/C/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin${{matrix.arch.bin}}\" type -p mspdb140.dll || exit 1 - sh -x /usr/src/build-extra/please.sh build-mingw-w64-git --only-64-bit --build-src-pkg -o artifacts HEAD && + sh -x /usr/src/build-extra/please.sh build-mingw-w64-git --only-${{matrix.arch.bitness}}-bit --build-src-pkg -o artifacts HEAD && cp bundle-artifacts/ver artifacts/ && if test -n \"`$GPGKEY\" then @@ -188,10 +197,10 @@ jobs: if: always() shell: bash run: rm -rf home - - name: Publish mingw-w64-x86_64-git + - name: Publish mingw-w64-${{matrix.arch.name}}-git uses: actions/upload-artifact@v1 with: - name: pkg-x86_64 + name: pkg-${{matrix.arch.name}} path: artifacts artifacts: runs-on: windows-latest @@ -212,21 +221,27 @@ jobs: - name: mingit-busybox fileprefix: MinGit fileextension: zip + arch: + - name: x86_64 + bitness: 64 + - name: i686 + bitness: 32 fail-fast: false env: - MSYSTEM: MINGW64 + MSYSTEM: MINGW${{matrix.arch.bitness}} steps: - - name: Download pkg-x86_64 + - name: Download pkg-${{matrix.arch.name}} uses: actions/download-artifact@v1 with: - name: pkg-x86_64 - path: pkg-x86_64 + name: pkg-${{matrix.arch.name}} + path: pkg-${{matrix.arch.name}} - name: Download bundle-artifacts uses: actions/download-artifact@v1 with: name: bundle-artifacts path: bundle-artifacts - name: Download git-sdk-64-build-installers + if: matrix.arch.bitness == '64' shell: bash run: | # Use Git Bash to download and unpack the artifact @@ -239,12 +254,29 @@ jobs: curl -o artifacts.zip "$download_url" + ## Unpack artifact + unzip artifacts.zip + - name: Download git-sdk-32-build-installers + if: matrix.arch.bitness == '32' + shell: bash + run: | + # Use Git Bash to download and unpack the artifact + + ## Get artifact + urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds + id=$(curl "$urlbase?definitions=30&statusFilter=completed&resultFilter=succeeded&\$top=1" | + jq -r '.value[0].id') + download_url=$(curl "$urlbase/$id/artifacts" | + jq -r '.value[] | select(.name == "git-sdk-32-build-installers").resource.downloadUrl') + + curl -o artifacts.zip "$download_url" + ## Unpack artifact unzip artifacts.zip - name: Clone and update build-extra shell: bash run: | - d=git-sdk-64-build-installers/usr/src/build-extra && + d=git-sdk-${{matrix.arch.bitness}}-build-installers/usr/src/build-extra && git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d && git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main - name: Prepare home directory for code-signing @@ -258,12 +290,12 @@ jobs: echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >home/.sig/codesign.p12 && echo -n "$CODESIGN_PASS" >home/.sig/codesign.pass && git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"' - - name: Build 64-bit ${{matrix.artifact.name}} + - name: Build ${{matrix.arch.bitness}}-bit ${{matrix.artifact.name}} shell: powershell run: | - & .\git-sdk-64-build-installers\usr\bin\bash.exe -lc @" + & .\git-sdk-${{matrix.arch.bitness}}-build-installers\usr\bin\bash.exe -lc @" set -x - /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-x86_64/ver) -o artifacts --${{matrix.artifact.name}} --pkg=pkg-x86_64/mingw-w64-x86_64-git-[0-9]*.tar.xz --pkg=pkg-x86_64/mingw-w64-x86_64-git-doc-html-[0-9]*.tar.xz && + /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-${{matrix.arch.name}}/ver) -o artifacts --${{matrix.artifact.name}} --pkg=pkg-${{matrix.arch.name}}/mingw-w64-${{matrix.arch.name}}-git-[0-9]*.tar.xz --pkg=pkg-${{matrix.arch.name}}/mingw-w64-${{matrix.arch.name}}-git-doc-html-[0-9]*.tar.xz && if test portable = '${{matrix.artifact.name}}' && test -n \"`$(git config alias.signtool)\" then git signtool artifacts/PortableGit-*.exe @@ -274,22 +306,22 @@ jobs: if: matrix.artifact.name == 'installer' shell: powershell run: | - & .\git-sdk-64-build-installers\usr\bin\bash.exe -lc @" + & .\git-sdk-${{matrix.arch.bitness}}-build-installers\usr\bin\bash.exe -lc @" cp /usr/src/build-extra/installer/package-versions.txt artifacts/ && a=`$PWD/artifacts && - p=`$PWD/pkg-x86_64 && + p=`$PWD/pkg-${{matrix.arch.name}} && (cd /usr/src/build-extra && mkdir -p cached-source-packages && cp \"`$p\"/*-pdb* cached-source-packages/ && - GIT_CONFIG_PARAMETERS=\"'windows.sdk64.path='\" ./please.sh bundle_pdbs --arch=x86_64 --directory=\"`$a\" installer/package-versions.txt) + GIT_CONFIG_PARAMETERS=\"'windows.sdk${{matrix.arch.bitness}}.path='\" ./please.sh bundle_pdbs --arch=${{matrix.arch.name}} --directory=\"`$a\" installer/package-versions.txt) "@ - name: Clean up temporary files if: always() shell: bash run: rm -rf home - - name: Publish ${{matrix.artifact.name}}-x86_64 + - name: Publish ${{matrix.artifact.name}}-${{matrix.arch.name}} uses: actions/upload-artifact@v1 with: - name: ${{matrix.artifact.name}}-x86_64 + name: ${{matrix.artifact.name}}-${{matrix.arch.name}} path: artifacts From f41c508ac5006a3744ba01add1b0a2b1aa39e5bb Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Tue, 5 Jan 2021 16:25:51 +0100 Subject: [PATCH 65/72] cmake(install): fix double .exe suffixes By mistake, the `.exe` extension is appended _twice_ when installing the dashed executables into `libexec/git-core/` on Windows (the extension is already appended when adding items to the `git_links` list in the `#Creating hardlinks` section). Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- contrib/buildsystems/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index c151dd7257..3dc6ca931b 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -828,12 +828,12 @@ install(CODE "file(CREATE_LINK ${CMAKE_INSTALL_PREFIX}/bin/git-shell${EXE_EXTENS foreach(b ${git_links}) string(REPLACE "${CMAKE_BINARY_DIR}" "" b ${b}) - install(CODE "file(CREATE_LINK ${CMAKE_INSTALL_PREFIX}/bin/git${EXE_EXTENSION} ${CMAKE_INSTALL_PREFIX}/libexec/git-core/${b}${EXE_EXTENSION})") + install(CODE "file(CREATE_LINK ${CMAKE_INSTALL_PREFIX}/bin/git${EXE_EXTENSION} ${CMAKE_INSTALL_PREFIX}/libexec/git-core/${b})") endforeach() foreach(b ${git_http_links}) string(REPLACE "${CMAKE_BINARY_DIR}" "" b ${b}) - install(CODE "file(CREATE_LINK ${CMAKE_INSTALL_PREFIX}/libexec/git-core/git-remote-http${EXE_EXTENSION} ${CMAKE_INSTALL_PREFIX}/libexec/git-core/${b}${EXE_EXTENSION})") + install(CODE "file(CREATE_LINK ${CMAKE_INSTALL_PREFIX}/libexec/git-core/git-remote-http${EXE_EXTENSION} ${CMAKE_INSTALL_PREFIX}/libexec/git-core/${b})") endforeach() install(PROGRAMS ${git_shell_scripts} ${git_perl_scripts} ${CMAKE_BINARY_DIR}/git-p4 From defe76a4eb68c7c0fb94c11be547f69bbf6f11c7 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Tue, 12 Jan 2021 12:26:16 +0100 Subject: [PATCH 66/72] GitHub Actions: Add vcpkg initialization step This commit adds a step called "initialize vcpkg" to the GitHub Actions workflow, because we need some build scripts from vcpkg that aren't present in our vcpkg artifacts from Azure Pipelines. Signed-off-by: Dennis Ameling --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a3fbbe6398..119e238621 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -164,6 +164,7 @@ jobs: vs-build: needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' + timeout-minutes: 15 env: MSYSTEM: MINGW64 NO_PERL: 1 @@ -186,6 +187,11 @@ jobs: ## Unzip and remove the artifact unzip artifacts.zip rm artifacts.zip + - name: initialize vcpkg + uses: actions/checkout@v2 + with: + repository: 'microsoft/vcpkg' + path: 'compat/vcbuild/vcpkg' - name: download vcpkg artifacts shell: powershell run: | From d2c045813e2d8c95cd1f3c3e93b86c358a96a5f8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 12 Apr 2020 00:52:22 +0200 Subject: [PATCH 67/72] git-artifacts: also build the nuget package The two NuGet artifact exists only in the 64-bit version. So let's make them in a separate, non-matrix job. Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 05f5ccaaf0..0a64f407f3 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -325,3 +325,52 @@ jobs: with: name: ${{matrix.artifact.name}}-${{matrix.arch.name}} path: artifacts + nuget: + runs-on: windows-latest + needs: pkg + steps: + - name: Download pkg-x86_64 + uses: actions/download-artifact@v1 + with: + name: pkg-x86_64 + path: pkg-x86_64 + - name: Download bundle-artifacts + uses: actions/download-artifact@v1 + with: + name: bundle-artifacts + path: bundle-artifacts + - name: Download git-sdk-64-build-installers + shell: bash + run: | + # Use Git Bash to download and unpack the artifact + + ## Get artifact + urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds + id=${{ needs.pkg.outputs.latest-sdk64-extra-build-id }} + download_url=$(curl "$urlbase/$id/artifacts" | + jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl') + + curl -o artifacts.zip "$download_url" + + ## Unpack artifact + unzip artifacts.zip + - name: Clone and update build-extra + shell: bash + run: | + d=git-sdk-64-build-installers/usr/src/build-extra && + git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d && + git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main + - uses: nuget/setup-nuget@v1 + - name: Build 64-bit NuGet packages + shell: powershell + run: | + & .\git-sdk-64-build-installers\usr\bin\bash.exe -lc @" + /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-x86_64/ver) -o artifacts --nuget --pkg=pkg-x86_64/mingw-w64-x86_64-git-[0-9]*.tar.xz --pkg=pkg-x86_64/mingw-w64-x86_64-git-doc-html-[0-9]*.tar.xz && + /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-x86_64/ver) -o artifacts --nuget-mingit && + openssl dgst -sha256 artifacts/Git*.nupkg | sed \"s/.* //\" >artifacts/sha-256.txt + "@ + - name: Publish nuget-x86_64 + uses: actions/upload-artifact@v1 + with: + name: nuget-x86_64 + path: artifacts From 6833c9e87b471dc8db7a98c3c9a6273d93c20bf8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 11 Apr 2020 23:43:55 +0200 Subject: [PATCH 68/72] git-artifacts: allow restricting which artifacts are built Users can now specify which artifacts they want to build, via the `build_only` input, which is a space-separated list of artifacts. For example, `installer portable` will build `installer-x86_64`, `installer-i686`, `portable-x86_64` and `portable-i686`, and an empty or unset value will build all artifacts. Please note that the `mingw-w64-git` packages are built always, as it would be tricky to figure out when they need to be built (for example, `build_only=portable-x86_64` technically does not need `pkg-i686` to be built, while `build_only=portable` does). Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 44 +++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 0a64f407f3..83300109a8 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -3,13 +3,17 @@ name: git-artifacts on: # This workflow can be triggered manually in the Actions tab, see # https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/ - - workflow_dispatch + workflow_dispatch: + inputs: + build_only: + description: 'Optionally restrict what artifacts to build' env: GPG_OPTIONS: "--batch --yes --no-tty --list-options no-show-photos --verify-options no-show-photos --pinentry-mode loopback" HOME: "${{github.workspace}}\\home" MSYSTEM: MINGW64 USERPROFILE: "${{github.workspace}}\\home" + BUILD_ONLY: "${{github.event.inputs.build_only}}" jobs: bundle-artifacts: @@ -230,18 +234,28 @@ jobs: env: MSYSTEM: MINGW${{matrix.arch.bitness}} steps: + - name: Determine whether this job should be skipped + shell: bash + run: | + case " $BUILD_ONLY " in + ' ') ;; # not set; build all + *" ${{matrix.artifact.name}} "*|*" ${{matrix.artifact.name}}-${{matrix.arch.name}} "*) ;; # build this artifact + *) echo "SKIP=true" >>$GITHUB_ENV;; + esac - name: Download pkg-${{matrix.arch.name}} + if: env.SKIP != 'true' uses: actions/download-artifact@v1 with: name: pkg-${{matrix.arch.name}} path: pkg-${{matrix.arch.name}} - name: Download bundle-artifacts + if: env.SKIP != 'true' uses: actions/download-artifact@v1 with: name: bundle-artifacts path: bundle-artifacts - name: Download git-sdk-64-build-installers - if: matrix.arch.bitness == '64' + if: env.SKIP != 'true' && matrix.arch.bitness == '64' shell: bash run: | # Use Git Bash to download and unpack the artifact @@ -257,7 +271,7 @@ jobs: ## Unpack artifact unzip artifacts.zip - name: Download git-sdk-32-build-installers - if: matrix.arch.bitness == '32' + if: env.SKIP != 'true' && matrix.arch.bitness == '32' shell: bash run: | # Use Git Bash to download and unpack the artifact @@ -274,6 +288,7 @@ jobs: ## Unpack artifact unzip artifacts.zip - name: Clone and update build-extra + if: env.SKIP != 'true' shell: bash run: | d=git-sdk-${{matrix.arch.bitness}}-build-installers/usr/src/build-extra && @@ -283,7 +298,7 @@ jobs: env: CODESIGN_P12: ${{secrets.CODESIGN_P12}} CODESIGN_PASS: ${{secrets.CODESIGN_PASS}} - if: (matrix.artifact.name == 'installer' || matrix.artifact.name == 'portable') && env.CODESIGN_P12 != '' && env.CODESIGN_PASS != '' + if: env.SKIP != 'true' && (matrix.artifact.name == 'installer' || matrix.artifact.name == 'portable') && env.CODESIGN_P12 != '' && env.CODESIGN_PASS != '' shell: bash run: | mkdir -p home/.sig && @@ -291,6 +306,7 @@ jobs: echo -n "$CODESIGN_PASS" >home/.sig/codesign.pass && git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"' - name: Build ${{matrix.arch.bitness}}-bit ${{matrix.artifact.name}} + if: env.SKIP != 'true' shell: powershell run: | & .\git-sdk-${{matrix.arch.bitness}}-build-installers\usr\bin\bash.exe -lc @" @@ -303,7 +319,7 @@ jobs: openssl dgst -sha256 artifacts/${{matrix.artifact.fileprefix}}-*.${{matrix.artifact.fileextension}} | sed \"s/.* //\" >artifacts/sha-256.txt "@ - name: Copy package-versions and pdbs - if: matrix.artifact.name == 'installer' + if: env.SKIP != 'true' && matrix.artifact.name == 'installer' shell: powershell run: | & .\git-sdk-${{matrix.arch.bitness}}-build-installers\usr\bin\bash.exe -lc @" @@ -317,10 +333,11 @@ jobs: GIT_CONFIG_PARAMETERS=\"'windows.sdk${{matrix.arch.bitness}}.path='\" ./please.sh bundle_pdbs --arch=${{matrix.arch.name}} --directory=\"`$a\" installer/package-versions.txt) "@ - name: Clean up temporary files - if: always() + if: always() && env.SKIP != 'true' shell: bash run: rm -rf home - name: Publish ${{matrix.artifact.name}}-${{matrix.arch.name}} + if: env.SKIP != 'true' uses: actions/upload-artifact@v1 with: name: ${{matrix.artifact.name}}-${{matrix.arch.name}} @@ -329,17 +346,28 @@ jobs: runs-on: windows-latest needs: pkg steps: + - name: Determine whether this job should be skipped + shell: bash + run: | + case " $BUILD_ONLY " in + ' ') ;; # not set; build all + *" nuget "*) ;; # build this artifact + *) echo "SKIP=true" >>$GITHUB_ENV;; + esac - name: Download pkg-x86_64 + if: env.SKIP != 'true' uses: actions/download-artifact@v1 with: name: pkg-x86_64 path: pkg-x86_64 - name: Download bundle-artifacts + if: env.SKIP != 'true' uses: actions/download-artifact@v1 with: name: bundle-artifacts path: bundle-artifacts - name: Download git-sdk-64-build-installers + if: env.SKIP != 'true' shell: bash run: | # Use Git Bash to download and unpack the artifact @@ -355,13 +383,16 @@ jobs: ## Unpack artifact unzip artifacts.zip - name: Clone and update build-extra + if: env.SKIP != 'true' shell: bash run: | d=git-sdk-64-build-installers/usr/src/build-extra && git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d && git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main - uses: nuget/setup-nuget@v1 + if: env.SKIP != 'true' - name: Build 64-bit NuGet packages + if: env.SKIP != 'true' shell: powershell run: | & .\git-sdk-64-build-installers\usr\bin\bash.exe -lc @" @@ -370,6 +401,7 @@ jobs: openssl dgst -sha256 artifacts/Git*.nupkg | sed \"s/.* //\" >artifacts/sha-256.txt "@ - name: Publish nuget-x86_64 + if: env.SKIP != 'true' uses: actions/upload-artifact@v1 with: name: nuget-x86_64 From 97a09572bb755419454346cbe089d98417785cfb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 11 Apr 2020 23:58:47 +0200 Subject: [PATCH 69/72] git-artifacts: allow specifying repo/ref via workflow_dispatch With this change, users can specify the branch and repository from which they want to build Git for Windows' artifacts, via the `ref` and `repository` inputs. This allows e.g. building `refs/heads/seen` of `git/git` (even if no `git-artifacts` workflow is configured in that repository), or `refs/pull//merge` for a given Pull Request. Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 83300109a8..ab6d56f0b5 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -7,6 +7,10 @@ on: inputs: build_only: description: 'Optionally restrict what artifacts to build' + ref: + description: 'Optionally override which branch to build' + repository: + description: 'Optionally override from where to fetch the specified ref' env: GPG_OPTIONS: "--batch --yes --no-tty --list-options no-show-photos --verify-options no-show-photos --pinentry-mode loopback" @@ -14,6 +18,8 @@ env: MSYSTEM: MINGW64 USERPROFILE: "${{github.workspace}}\\home" BUILD_ONLY: "${{github.event.inputs.build_only}}" + REPOSITORY: "${{github.event.inputs.repository}}" + REF: "${{github.event.inputs.ref}}" jobs: bundle-artifacts: @@ -70,9 +76,11 @@ jobs: printf '#!/bin/sh\n\nexec /mingw64/bin/git.exe "`$@"\n' >/usr/bin/git && mkdir -p bundle-artifacts && + { test -n \"`$REPOSITORY\" || REPOSITORY='${{github.repository}}'; } && + { test -n \"`$REF\" || REF='${{github.ref}}'; } && git -c init.defaultBranch=main init --bare && git remote add -f origin https://github.com/git-for-windows/git && - git fetch https://github.com/${{github.repository}} ${{github.ref}}:${{github.ref}} && + git fetch \"https://github.com/`$REPOSITORY\" \"`$REF:`$REF\" && tag_name=\"`$(git describe --match 'v[0-9]*' FETCH_HEAD)-`$(date +%Y%m%d%H%M%S)\" && echo \"prerelease-`${tag_name#v}\" >bundle-artifacts/ver && From d19c2664e01780481ae273f02e70b92088438bf8 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 28 Jan 2021 21:31:05 +0100 Subject: [PATCH 70/72] git-artifacts: cache the build-installers artifact It is a bit expensive to fetch just the git-sdk-64-build-installers artifact from Azure Pipelines and then to unpack it (takes some 6-7 minutes, typically). Let's cache it if possible. Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 89 +++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index ab6d56f0b5..0250040145 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -24,6 +24,8 @@ env: jobs: bundle-artifacts: runs-on: windows-latest + outputs: + latest-sdk64-extra-build-id: ${{ steps.determine-latest-sdk64-extra-build-id.outputs.id }} steps: - name: Configure user shell: bash @@ -34,15 +36,31 @@ jobs: git config --global user.name "$USER_NAME" && git config --global user.email "$USER_EMAIL" && echo "PACKAGER=$USER_NAME <$USER_EMAIL>" >>$GITHUB_ENV + - name: Determine latest git-sdk-64-extra-artifacts build ID + id: determine-latest-sdk64-extra-build-id + shell: bash + run: | + urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds + id=$(curl "$urlbase?definitions=29&statusFilter=completed&resultFilter=succeeded&\$top=1" | + jq -r '.value[0].id') + + echo "Latest ID is ${id}" + echo "::set-output name=id::$id" + - name: Cache git-sdk-64-build-installers + id: cache-sdk-build-installers + uses: actions/cache@v2 + with: + path: git-sdk-64-build-installers + key: build-installers-64-${{ steps.determine-latest-sdk64-extra-build-id.outputs.id }} - name: Download git-sdk-64-build-installers + if: steps.cache-sdk-build-installers.outputs.cache-hit != 'true' shell: bash run: | # Use Git Bash to download and unpack the artifact ## Get artifact urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=$(curl "$urlbase?definitions=29&statusFilter=completed&resultFilter=succeeded&\$top=1" | - jq -r '.value[0].id') + id=${{ steps.determine-latest-sdk64-extra-build-id.outputs.id }} download_url=$(curl "$urlbase/$id/artifacts" | jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl') @@ -54,7 +72,13 @@ jobs: shell: bash run: | d=git-sdk-64-build-installers/usr/src/build-extra && - git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d + if test ! -d $d/.git + then + git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d + else + git -C $d fetch https://github.com/git-for-windows/build-extra main && + git -C $d switch -C main FETCH_HEAD + fi - name: Prepare home directory for GPG signing if: env.GPGKEY != '' shell: bash @@ -104,6 +128,8 @@ jobs: pkg: runs-on: windows-latest needs: bundle-artifacts + outputs: + latest-sdk64-extra-build-id: ${{ needs.bundle-artifacts.outputs.latest-sdk64-extra-build-id }} strategy: matrix: arch: @@ -262,8 +288,15 @@ jobs: with: name: bundle-artifacts path: bundle-artifacts - - name: Download git-sdk-64-build-installers + - name: Cache git-sdk-64-build-installers if: env.SKIP != 'true' && matrix.arch.bitness == '64' + id: cache-sdk64-build-installers + uses: actions/cache@v2 + with: + path: git-sdk-64-build-installers + key: build-installers-64-${{ needs.pkg.outputs.latest-sdk64-extra-build-id }} + - name: Download git-sdk-64-build-installers + if: env.SKIP != 'true' && matrix.arch.bitness == '64' && steps.cache-sdk64-build-installers.outputs.cache-hit != 'true' shell: bash run: | # Use Git Bash to download and unpack the artifact @@ -278,16 +311,33 @@ jobs: ## Unpack artifact unzip artifacts.zip - - name: Download git-sdk-32-build-installers + - name: Determine latest git-sdk-32-extra-artifacts build ID if: env.SKIP != 'true' && matrix.arch.bitness == '32' + id: determine-latest-sdk32-extra-build-id + shell: bash + run: | + urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds + id=$(curl "$urlbase?definitions=30&statusFilter=completed&resultFilter=succeeded&\$top=1" | + jq -r '.value[0].id') + + echo "Latest ID is ${id}" + echo "::set-output name=id::$id" + - name: Cache git-sdk-32-build-installers + if: env.SKIP != 'true' && matrix.arch.bitness == '32' + id: cache-sdk32-build-installers + uses: actions/cache@v2 + with: + path: git-sdk-32-build-installers + key: build-installers-32-${{ steps.determine-latest-sdk32-extra-build-id.outputs.id }} + - name: Download git-sdk-32-build-installers + if: env.SKIP != 'true' && matrix.arch.bitness == '32' && steps.cache-sdk32-build-installers.outputs.cache-hit != 'true' shell: bash run: | # Use Git Bash to download and unpack the artifact ## Get artifact urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=$(curl "$urlbase?definitions=30&statusFilter=completed&resultFilter=succeeded&\$top=1" | - jq -r '.value[0].id') + id=${{ steps.determine-latest-sdk32-extra-build-id.outputs.id }} download_url=$(curl "$urlbase/$id/artifacts" | jq -r '.value[] | select(.name == "git-sdk-32-build-installers").resource.downloadUrl') @@ -300,7 +350,13 @@ jobs: shell: bash run: | d=git-sdk-${{matrix.arch.bitness}}-build-installers/usr/src/build-extra && - git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d && + if test ! -d $d/.git + then + git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d + else + git -C $d fetch https://github.com/git-for-windows/build-extra main && + git -C $d switch -C main FETCH_HEAD + fi && git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main - name: Prepare home directory for code-signing env: @@ -374,8 +430,15 @@ jobs: with: name: bundle-artifacts path: bundle-artifacts - - name: Download git-sdk-64-build-installers + - name: Cache git-sdk-64-build-installers if: env.SKIP != 'true' + id: cache-sdk-build-installers + uses: actions/cache@v2 + with: + path: git-sdk-64-build-installers + key: build-installers-64-${{ needs.pkg.outputs.latest-sdk64-extra-build-id }} + - name: Download git-sdk-64-build-installers + if: env.SKIP != 'true' && steps.cache-sdk-build-installers.outputs.cache-hit != 'true' shell: bash run: | # Use Git Bash to download and unpack the artifact @@ -395,7 +458,13 @@ jobs: shell: bash run: | d=git-sdk-64-build-installers/usr/src/build-extra && - git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d && + if test ! -d $d/.git + then + git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d + else + git -C $d fetch https://github.com/git-for-windows/build-extra main && + git -C $d switch -C main FETCH_HEAD + fi && git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main - uses: nuget/setup-nuget@v1 if: env.SKIP != 'true' From a631c918946179af37581e7a78bff681d891dde1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 3 Feb 2021 15:52:58 +0100 Subject: [PATCH 71/72] git-artifacts: use the cached build-installers instead of makepkg-git When building the Pacman packages, we technically do not need the full `build-installers` artifact (which is substantially larger than the `makepkg-git` artifact). However, the former is already cached and includes the latter's files. And it is _so_ much faster to download the cached (larger) artifact than to download the smaller `makepkg-git` artifact from Azure Pipelines. Suggested-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index 0250040145..b18ae02f29 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -149,16 +149,23 @@ jobs: git config --global user.name "$USER_NAME" && git config --global user.email "$USER_EMAIL" && echo "PACKAGER=$USER_NAME <$USER_EMAIL>" >>$GITHUB_ENV - - name: Download git-sdk-64-makepkg-git + - name: Cache git-sdk-64-build-installers + id: cache-sdk-build-installers + uses: actions/cache@v2 + with: + path: git-sdk-64-build-installers + key: build-installers-64-${{ needs.bundle-artifacts.outputs.latest-sdk64-extra-build-id }} + - name: Download git-sdk-64-build-installers + if: steps.cache-sdk-build-installers.outputs.cache-hit != 'true' shell: bash run: | # Use Git Bash to download and unpack the artifact ## Get artifact urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=${{ needs.bundle-artifacts.outputs.latest-sdk64-extra-build-id }} - download_url="$(curl "$urlbase/$id/artifacts" | - jq -r '.value[] | select(.name == "git-sdk-64-makepkg-git").resource.downloadUrl')" + id=${{ needs.pkg.outputs.latest-sdk64-extra-build-id }} + download_url=$(curl "$urlbase/$id/artifacts" | + jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl') curl -o artifacts.zip "$download_url" @@ -172,8 +179,14 @@ jobs: - name: Clone and update build-extra shell: bash run: | - d=git-sdk-64-makepkg-git/usr/src/build-extra && - git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d && + d=git-sdk-64-build-installers/usr/src/build-extra && + if test ! -d $d/.git + then + git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d + else + git -C $d fetch https://github.com/git-for-windows/build-extra main && + git -C $d switch -C main FETCH_HEAD + fi && git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main - name: Check out git/git shell: bash @@ -209,7 +222,7 @@ jobs: GPGKEY: "${{secrets.GPGKEY}}" shell: powershell run: | - & git-sdk-64-makepkg-git\usr\bin\sh.exe -lc @" + & git-sdk-64-build-installers\usr\bin\sh.exe -lc @" set -x # Let `cv2pdb` find the DLLs PATH=\"`$PATH:/C/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin${{matrix.arch.bin}}\" From 3019bd536cfa5e5db8d0c78936b1cae44baf91ff Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 4 Feb 2021 16:49:01 +0100 Subject: [PATCH 72/72] git-artifacts: add ARM64 artifacts Adds ARM64 artifacts to the git-artifacts GitHub Action workflow. Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- .github/workflows/git-artifacts.yml | 75 +++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/.github/workflows/git-artifacts.yml b/.github/workflows/git-artifacts.yml index b18ae02f29..7d15960043 100644 --- a/.github/workflows/git-artifacts.yml +++ b/.github/workflows/git-artifacts.yml @@ -253,9 +253,58 @@ jobs: with: name: pkg-${{matrix.arch.name}} path: artifacts + build-arm64: + needs: bundle-artifacts + runs-on: windows-latest + steps: + - name: Configure user + shell: bash + run: + USER_NAME="${{github.actor}}" && + USER_EMAIL="${{github.actor}}@users.noreply.github.com" && + mkdir -p "$HOME" && + git config --global user.name "$USER_NAME" && + git config --global user.email "$USER_EMAIL" + - uses: actions/checkout@v2 + - name: initialize vcpkg + uses: actions/checkout@v2 + with: + repository: 'microsoft/vcpkg' + path: 'compat/vcbuild/vcpkg' + - name: download vcpkg artifacts + shell: powershell + run: | + $urlbase = "https://dev.azure.com/git/git/_apis/build/builds" + $id = ((Invoke-WebRequest -UseBasicParsing "${urlbase}?definitions=9&statusFilter=completed&resultFilter=succeeded&`$top=1").content | ConvertFrom-JSON).value[0].id + $downloadUrl = ((Invoke-WebRequest -UseBasicParsing "${urlbase}/$id/artifacts").content | ConvertFrom-JSON).value[0].resource.downloadUrl + (New-Object Net.WebClient).DownloadFile($downloadUrl, "compat.zip") + Expand-Archive compat.zip -DestinationPath . -Force + Remove-Item compat.zip + - name: add msbuild to PATH + uses: microsoft/setup-msbuild@v1 + - name: copy dlls to root + shell: powershell + run: | + & compat\vcbuild\vcpkg_copy_dlls.bat release arm64-windows + if (!$?) { exit(1) } + - name: generate Visual Studio solution + shell: bash + run: | + cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/arm64-windows \ + -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=arm64 -DVCPKG_ARCH=arm64-windows \ + -DCMAKE_INSTALL_PREFIX="`pwd`/git-arm64" + - name: MSBuild + run: msbuild git.sln -property:Configuration=Release + - name: Link the Git executables + run: msbuild INSTALL.vcxproj -property:Configuration=Release + - name: upload build artifacts + uses: actions/upload-artifact@v1 + with: + name: arm64-artifacts + path: ./git-arm64 artifacts: runs-on: windows-latest - needs: pkg + needs: [pkg, build-arm64] strategy: matrix: artifact: @@ -275,8 +324,13 @@ jobs: arch: - name: x86_64 bitness: 64 + arm64: false - name: i686 bitness: 32 + arm64: false + - name: i686 + bitness: 32 + arm64: true fail-fast: false env: MSYSTEM: MINGW${{matrix.arch.bitness}} @@ -358,6 +412,12 @@ jobs: ## Unpack artifact unzip artifacts.zip + - name: Download arm64 artifact + if: matrix.arch.arm64 == true + uses: actions/download-artifact@v1 + with: + name: arm64-artifacts + path: ${{github.workspace}}/arm64 - name: Clone and update build-extra if: env.SKIP != 'true' shell: bash @@ -388,7 +448,8 @@ jobs: run: | & .\git-sdk-${{matrix.arch.bitness}}-build-installers\usr\bin\bash.exe -lc @" set -x - /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-${{matrix.arch.name}}/ver) -o artifacts --${{matrix.artifact.name}} --pkg=pkg-${{matrix.arch.name}}/mingw-w64-${{matrix.arch.name}}-git-[0-9]*.tar.xz --pkg=pkg-${{matrix.arch.name}}/mingw-w64-${{matrix.arch.name}}-git-doc-html-[0-9]*.tar.xz && + [[ \"${{matrix.arch.arm64}}\" = true ]] && ARM64=\"--include-arm64-artifacts=\\\"`$PWD/arm64\\\"\" || ARM64= + /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git `$ARM64 --version=`$(cat pkg-${{matrix.arch.name}}/ver) -o artifacts --${{matrix.artifact.name}} --pkg=pkg-${{matrix.arch.name}}/mingw-w64-${{matrix.arch.name}}-git-[0-9]*.tar.xz --pkg=pkg-${{matrix.arch.name}}/mingw-w64-${{matrix.arch.name}}-git-doc-html-[0-9]*.tar.xz && if test portable = '${{matrix.artifact.name}}' && test -n \"`$(git config alias.signtool)\" then git signtool artifacts/PortableGit-*.exe @@ -413,12 +474,20 @@ jobs: if: always() && env.SKIP != 'true' shell: bash run: rm -rf home + - name: Publish ${{matrix.artifact.name}}-${{matrix.arch.name}} - if: env.SKIP != 'true' + if: env.SKIP != 'true' && matrix.arch.arm64 != true uses: actions/upload-artifact@v1 with: name: ${{matrix.artifact.name}}-${{matrix.arch.name}} path: artifacts + + - name: Publish ${{matrix.artifact.name}}-arm64 + if: env.SKIP != 'true' && matrix.arch.arm64 == true + uses: actions/upload-artifact@v1 + with: + name: ${{matrix.artifact.name}}-arm64 + path: artifacts nuget: runs-on: windows-latest needs: pkg