From cc67b7549437c5f9df7ddb4120525ee93521f068 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:30:13 +0100 Subject: [PATCH 001/102] mingw: respect core.hidedotfiles = false in git-init again This is a brown paper bag. When adding the tests, we actually failed to verify that the config variable is heeded in git-init at all. And when changing the original patch that marked the .git/ directory as hidden after reading the config, it was lost on this developer that the new code would use the hide_dotfiles variable before the config was read. The fix is obvious: read the (limited, pre-init) config *before* creating the .git/ directory. This fixes https://github.com/git-for-windows/git/issues/789 Signed-off-by: Johannes Schindelin --- builtin/init-db.c | 6 ++++++ t/t0001-init.sh | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/builtin/init-db.c b/builtin/init-db.c index 41faffd28d..bcefe4a203 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -155,6 +155,9 @@ static int git_init_db_config(const char *k, const char *v, void *cb) if (!strcmp(k, "init.templatedir")) return git_config_pathname(&init_db_template_dir, k, v); + if (starts_with(k, "core.")) + return platform_core_config(k, v, cb); + return 0; } @@ -361,6 +364,9 @@ int init_db(const char *git_dir, const char *real_git_dir, } startup_info->have_repository = 1; + /* Just look for `init.templatedir` and `core.hidedotfiles` */ + git_config(git_init_db_config, NULL); + safe_create_dir(git_dir, 0); init_is_bare_repository = is_bare_repository(); diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 42a263cada..35ede1b0b0 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -453,6 +453,18 @@ test_expect_success 're-init from a linked worktree' ' ) ' +test_expect_success MINGW 'core.hidedotfiles = false' ' + git config --global core.hidedotfiles false && + rm -rf newdir && + ( + sane_unset GIT_DIR GIT_WORK_TREE GIT_CONFIG && + mkdir newdir && + cd newdir && + git init + ) && + ! is_hidden newdir/.git +' + test_expect_success MINGW 'redirect std handles' ' GIT_REDIRECT_STDOUT=output.txt git rev-parse --git-dir && test .git = "$(cat output.txt)" && From 187d6320427534a30fb6a359da5810cd809c5b78 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 7 Sep 2016 18:07:04 +0200 Subject: [PATCH 002/102] reset: support the experimental --stdin option Just like with other Git commands, this option makes it read the paths from the standard input. It comes in handy when resetting many, many paths at once and wildcards are not an option (e.g. when the paths are generated by a tool). Note: we first parse the entire list and perform the actual reset action only in a second phase. Not only does this make things simpler, it also helps performance, as do_diff_cache() traverses the index and the (sorted) pathspecs in simultaneously to avoid unnecessary lookups. This feature is marked experimental because it is still under review in the upstream Git project. Signed-off-by: Johannes Schindelin --- Documentation/git-reset.txt | 10 +++++++ builtin/reset.c | 53 ++++++++++++++++++++++++++++++++++++- t/t7108-reset-stdin.sh | 32 ++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100755 t/t7108-reset-stdin.sh diff --git a/Documentation/git-reset.txt b/Documentation/git-reset.txt index 9f69ae8b69..d0118bd028 100644 --- a/Documentation/git-reset.txt +++ b/Documentation/git-reset.txt @@ -10,6 +10,7 @@ SYNOPSIS [verse] 'git reset' [-q] [] [--] ... 'git reset' (--patch | -p) [] [--] [...] +EXPERIMENTAL: 'git reset' [-q] [--stdin [-z]] [] 'git reset' [--soft | --mixed [-N] | --hard | --merge | --keep] [-q] [] DESCRIPTION @@ -100,6 +101,15 @@ OPTIONS `reset.quiet` config option. `--quiet` and `--no-quiet` will override the default behavior. +--stdin:: + EXPERIMENTAL: Instead of taking list of paths from the + command line, read list of paths from the standard input. + Paths are separated by LF (i.e. one path per line) by + default. + +-z:: + EXPERIMENTAL: Only meaningful with `--stdin`; paths are + separated with NUL character instead of LF. EXAMPLES -------- diff --git a/builtin/reset.c b/builtin/reset.c index 58166964f8..4ba8badd0f 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -24,12 +24,15 @@ #include "cache-tree.h" #include "submodule.h" #include "submodule-config.h" +#include "strbuf.h" +#include "quote.h" #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000) static const char * const git_reset_usage[] = { N_("git reset [--mixed | --soft | --hard | --merge | --keep] [-q] []"), N_("git reset [-q] [] [--] ..."), + N_("EXPERIMENTAL: git reset [-q] [--stdin [-z]] []"), N_("git reset --patch [] [--] [...]"), NULL }; @@ -283,7 +286,9 @@ static int git_reset_config(const char *var, const char *value, void *cb) int cmd_reset(int argc, const char **argv, const char *prefix) { int reset_type = NONE, update_ref_status = 0, quiet = 0; - int patch_mode = 0, unborn; + int patch_mode = 0, nul_term_line = 0, read_from_stdin = 0, unborn; + char **stdin_paths = NULL; + int stdin_nr = 0, stdin_alloc = 0; const char *rev; struct object_id oid; struct pathspec pathspec; @@ -305,6 +310,10 @@ int cmd_reset(int argc, const char **argv, const char *prefix) OPT_BOOL('p', "patch", &patch_mode, N_("select hunks interactively")), OPT_BOOL('N', "intent-to-add", &intent_to_add, N_("record only the fact that removed paths will be added later")), + OPT_BOOL('z', NULL, &nul_term_line, + N_("EXPERIMENTAL: paths are separated with NUL character")), + OPT_BOOL(0, "stdin", &read_from_stdin, + N_("EXPERIMENTAL: read paths from ")), OPT_END() }; @@ -315,6 +324,42 @@ int cmd_reset(int argc, const char **argv, const char *prefix) PARSE_OPT_KEEP_DASHDASH); parse_args(&pathspec, argv, prefix, patch_mode, &rev); + if (read_from_stdin) { + strbuf_getline_fn getline_fn = nul_term_line ? + strbuf_getline_nul : strbuf_getline_lf; + int flags = PATHSPEC_PREFER_FULL; + struct strbuf buf = STRBUF_INIT; + struct strbuf unquoted = STRBUF_INIT; + + if (patch_mode) + die(_("--stdin is incompatible with --patch")); + + if (pathspec.nr) + die(_("--stdin is incompatible with path arguments")); + + while (getline_fn(&buf, stdin) != EOF) { + if (!nul_term_line && buf.buf[0] == '"') { + strbuf_reset(&unquoted); + if (unquote_c_style(&unquoted, buf.buf, NULL)) + die(_("line is badly quoted")); + strbuf_swap(&buf, &unquoted); + } + ALLOC_GROW(stdin_paths, stdin_nr + 1, stdin_alloc); + stdin_paths[stdin_nr++] = xstrdup(buf.buf); + strbuf_reset(&buf); + } + strbuf_release(&unquoted); + strbuf_release(&buf); + + ALLOC_GROW(stdin_paths, stdin_nr + 1, stdin_alloc); + stdin_paths[stdin_nr++] = NULL; + flags |= PATHSPEC_LITERAL_PATH; + parse_pathspec(&pathspec, 0, flags, prefix, + (const char **)stdin_paths); + + } else if (nul_term_line) + die(_("-z requires --stdin")); + unborn = !strcmp(rev, "HEAD") && get_oid("HEAD", &oid); if (unborn) { /* reset on unborn branch: treat as reset to empty tree */ @@ -415,5 +460,11 @@ int cmd_reset(int argc, const char **argv, const char *prefix) if (!pathspec.nr) remove_branch_state(); + if (stdin_paths) { + while (stdin_nr) + free(stdin_paths[--stdin_nr]); + free(stdin_paths); + } + return update_ref_status; } diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh new file mode 100755 index 0000000000..b7cbcbf869 --- /dev/null +++ b/t/t7108-reset-stdin.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description='reset --stdin' + +. ./test-lib.sh + +test_expect_success 'reset --stdin' ' + test_commit hello && + git rm hello.t && + test -z "$(git ls-files hello.t)" && + echo hello.t | git reset --stdin && + test hello.t = "$(git ls-files hello.t)" +' + +test_expect_success 'reset --stdin -z' ' + test_commit world && + git rm hello.t world.t && + test -z "$(git ls-files hello.t world.t)" && + printf world.tQworld.tQhello.tQ | q_to_nul | git reset --stdin -z && + printf "hello.t\nworld.t\n" >expect && + git ls-files >actual && + test_cmp expect actual +' + +test_expect_success '--stdin requires --mixed' ' + echo hello.t >list && + test_must_fail git reset --soft --stdin Date: Sun, 24 Jul 2011 15:54:04 +0200 Subject: [PATCH 003/102] t9350: point out that refs are not updated correctly This happens only when the corresponding commits are not exported in the current fast-export run. This can happen either when the relevant commit is already marked, or when the commit is explicitly marked as UNINTERESTING with a negative ref by another argument. This breaks fast-export basec remote helpers. Signed-off-by: Sverre Rabbelier --- t/t9350-fast-export.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t9350-fast-export.sh b/t/t9350-fast-export.sh index 6a392e87bc..6d6d5050a2 100755 --- a/t/t9350-fast-export.sh +++ b/t/t9350-fast-export.sh @@ -556,4 +556,15 @@ test_expect_success 'merge commit gets exported with --import-marks' ' ) ' +cat > expected << EOF +reset refs/heads/master +from $(git rev-parse master) + +EOF + +test_expect_failure 'refs are updated even if no commits need to be exported' ' + git fast-export master..master > actual && + test_cmp expected actual +' + test_done From 66c7dc2e9ef820cfcf383a02083ac63cef84185b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 23 Jul 2011 15:55:26 +0200 Subject: [PATCH 004/102] fast-export: do not refer to non-existing marks When calling `git fast-export a..a b` when a and b refer to the same commit, nothing would be exported, and an incorrect reset line would be printed for b ('from :0'). Signed-off-by: Johannes Schindelin Signed-off-by: Sverre Rabbelier --- builtin/fast-export.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/builtin/fast-export.c b/builtin/fast-export.c index 5790f0d554..19d2d1de37 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -874,9 +874,20 @@ static void get_tags_and_duplicates(struct rev_cmdline_info *info) } } +static void handle_reset(const char *name, struct object *object) +{ + int mark = get_object_mark(object); + + if (mark) + printf("reset %s\nfrom :%d\n\n", name, + get_object_mark(object)); + else + printf("reset %s\nfrom %s\n\n", name, + oid_to_hex(&object->oid)); +} + static void handle_tags_and_duplicates(void) { - struct commit *commit; int i; for (i = extra_refs.nr - 1; i >= 0; i--) { @@ -890,9 +901,7 @@ static void handle_tags_and_duplicates(void) if (anonymize) name = anonymize_refname(name); /* create refs pointing to already seen commits */ - commit = (struct commit *)object; - printf("reset %s\nfrom :%d\n\n", name, - get_object_mark(&commit->object)); + handle_reset(name, object); show_progress(); break; } From 699996df2111b57501273afc266fce61bc55ea47 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 28 Aug 2010 20:49:01 -0500 Subject: [PATCH 005/102] transport-helper: add trailing -- [PT: ensure we add an additional element to the argv array] --- transport-helper.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transport-helper.c b/transport-helper.c index bf225c698f..b19a56ba6d 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -460,6 +460,8 @@ static int get_exporter(struct transport *transport, for (i = 0; i < revlist_args->nr; i++) argv_array_push(&fastexport->args, revlist_args->items[i].string); + argv_array_push(&fastexport->args, "--"); + fastexport->git_cmd = 1; return start_command(fastexport); } From d67de8bb494acd42ea399e704d2b5112e86feb56 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sun, 24 Jul 2011 00:06:00 +0200 Subject: [PATCH 006/102] remote-helper: check helper status after import/export Signed-off-by: Johannes Schindelin Signed-off-by: Sverre Rabbelier --- builtin/clone.c | 4 +++- t/t5801-remote-helpers.sh | 2 +- transport-helper.c | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 15b142d646..40b6c10392 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1167,7 +1167,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } if (!is_local && !complete_refs_before_fetch) - transport_fetch_refs(transport, mapped_refs); + if (transport_fetch_refs(transport, mapped_refs)) + die(_("could not fetch refs from %s"), + transport->url); remote_head = find_ref_by_name(refs, "HEAD"); remote_head_points_at = diff --git a/t/t5801-remote-helpers.sh b/t/t5801-remote-helpers.sh index aaaa722cca..18cf138343 100755 --- a/t/t5801-remote-helpers.sh +++ b/t/t5801-remote-helpers.sh @@ -228,7 +228,7 @@ test_expect_success 'push update refs failure' ' echo "update fail" >>file && git commit -a -m "update fail" && git rev-parse --verify testgit/origin/heads/update >expect && - test_expect_code 1 env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ + test_must_fail env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ git push origin update && git rev-parse --verify testgit/origin/heads/update >actual && test_cmp expect actual diff --git a/transport-helper.c b/transport-helper.c index b19a56ba6d..24b1a8daff 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -466,6 +466,19 @@ static int get_exporter(struct transport *transport, return start_command(fastexport); } +static void check_helper_status(struct helper_data *data) +{ + int pid, status; + + pid = waitpid(data->helper->pid, &status, WNOHANG); + if (pid < 0) + die("Could not retrieve status of remote helper '%s'", + data->name); + if (pid > 0 && WIFEXITED(status)) + die("Remote helper '%s' died with %d", + data->name, WEXITSTATUS(status)); +} + static int fetch_with_import(struct transport *transport, int nr_heads, struct ref **to_fetch) { @@ -502,6 +515,7 @@ static int fetch_with_import(struct transport *transport, if (finish_command(&fastimport)) die(_("error while running fast-import")); + check_helper_status(data); /* * The fast-import stream of a remote helper that advertises @@ -989,6 +1003,7 @@ static int push_refs_with_export(struct transport *transport, if (finish_command(&exporter)) die(_("error while running fast-export")); + check_helper_status(data); if (push_update_refs_status(data, remote_refs, flags)) return 1; From bade49dc3dcbe887e6567f3d10d81c09aea230c2 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 5 Jun 2012 10:24:11 -0400 Subject: [PATCH 007/102] Fix launching of externals from Unicode paths If Git were installed in a path containing non-ASCII characters, commands such as git-am and git-submodule, which are implemented as externals, would fail to launch with the following error: > fatal: 'am' appears to be a git command, but we were not > able to execute it. Maybe git-am is broken? This was due to lookup_prog not being Unicode-aware. It was somehow missed in 2ee5a1a14ad17ff35f0ad52390a27fbbc41258f3. Note that the only problem in this function was calling GetFileAttributes instead of GetFileAttributesW. The calls to access() were fine because access() is a macro which resolves to mingw_access, which already handles Unicode correctly. But I changed lookup_prog to use _waccess directly so that we only convert the path to UTF-16 once. Signed-off-by: Adam Roben --- compat/mingw.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 34b3880b29..97cd1179ae 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1129,14 +1129,20 @@ static char *lookup_prog(const char *dir, int dirlen, const char *cmd, int isexe, int exe_only) { char path[MAX_PATH]; + wchar_t wpath[MAX_PATH]; snprintf(path, sizeof(path), "%.*s\\%s.exe", dirlen, dir, cmd); - if (!isexe && access(path, F_OK) == 0) + if (xutftowcs_path(wpath, path) < 0) + return NULL; + + if (!isexe && _waccess(wpath, F_OK) == 0) return xstrdup(path); path[strlen(path)-4] = '\0'; - if ((!exe_only || isexe) && access(path, F_OK) == 0) - if (!(GetFileAttributes(path) & FILE_ATTRIBUTE_DIRECTORY)) + if ((!exe_only || isexe) && _waccess(wpath, F_OK) == 0) { + + if (!(GetFileAttributesW(wpath) & FILE_ATTRIBUTE_DIRECTORY)) return xstrdup(path); + } return NULL; } From 92e808e7a70bfb5d1c843c3c1d211929e371a1ad Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Wed, 2 Nov 2016 13:51:20 -0400 Subject: [PATCH 008/102] cvsexportcommit: force crlf translation When using cvsnt + msys + git, it seems like the output of cvs status had \r\n in it, and caused the command to fail. This fixes that. Signed-off-by: Dustin Spicuzza --- git-cvsexportcommit.perl | 1 + 1 file changed, 1 insertion(+) diff --git a/git-cvsexportcommit.perl b/git-cvsexportcommit.perl index d13f02da95..fc00d5946a 100755 --- a/git-cvsexportcommit.perl +++ b/git-cvsexportcommit.perl @@ -431,6 +431,7 @@ END sub safe_pipe_capture { my @output; if (my $pid = open my $child, '-|') { + binmode($child, ":crlf"); @output = (<$child>); close $child or die join(' ',@_).": $! $?"; } else { From c1dd110b7155ea136666732979db65378288eae9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 12 Feb 2018 17:23:42 +0100 Subject: [PATCH 009/102] add --edit: truncate the patch file If there is already a .git/ADD_EDIT.patch file, we fail to truncate it properly, which could result in very funny errors. Let's just truncate it and not worry about it too much. Reported by J Wyman. Signed-off-by: Johannes Schindelin --- builtin/add.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/add.c b/builtin/add.c index f65c172299..53c18ea429 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -239,7 +239,7 @@ static int edit_patch(int argc, const char **argv, const char *prefix) rev.diffopt.output_format = DIFF_FORMAT_PATCH; rev.diffopt.use_color = 0; rev.diffopt.flags.ignore_dirty_submodules = 1; - out = open(file, O_CREAT | O_WRONLY, 0666); + out = open(file, O_CREAT | O_WRONLY | O_TRUNC, 0666); if (out < 0) die(_("Could not open '%s' for writing."), file); rev.diffopt.file = xfdopen(out, "w"); From 5fcd8e76ed345f9f86754152192a8467cc72000f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 9 Apr 2012 13:04:35 -0500 Subject: [PATCH 010/102] Always auto-gc after calling a fast-import transport After importing anything with fast-import, we should always let the garbage collector do its job, since the objects are written to disk inefficiently. This brings down an initial import of http://selenic.com/hg from about 230 megabytes to about 14. In the future, we may want to make this configurable on a per-remote basis, or maybe teach fast-import about it in the first place. Signed-off-by: Johannes Schindelin --- transport-helper.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/transport-helper.c b/transport-helper.c index 24b1a8daff..94868bb378 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -16,6 +16,8 @@ #include "protocol.h" static int debug; +/* TODO: put somewhere sensible, e.g. git_transport_options? */ +static int auto_gc = 1; struct helper_data { const char *name; @@ -549,6 +551,12 @@ static int fetch_with_import(struct transport *transport, } } strbuf_release(&buf); + if (auto_gc) { + const char *argv_gc_auto[] = { + "gc", "--auto", "--quiet", NULL, + }; + run_command_v_opt(argv_gc_auto, RUN_GIT_CMD); + } return 0; } From 2866787a4c9e433d9fa5a27af63f6c1643f4f970 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 5 Jun 2012 15:40:33 -0400 Subject: [PATCH 011/102] Make non-.exe externals work again 7ebac8cb94f3a06d3fbdde469414a1443ca45510 made launching of .exe externals work when installed in Unicode paths. But it broke launching of non-.exe externals, no matter where they were installed. We now correctly maintain the UTF-8 and UTF-16 paths in tandem in lookup_prog. This fixes t5526, among others. Signed-off-by: Adam Roben --- compat/mingw.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 97cd1179ae..b0883b7867 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1137,11 +1137,12 @@ static char *lookup_prog(const char *dir, int dirlen, const char *cmd, if (!isexe && _waccess(wpath, F_OK) == 0) return xstrdup(path); - path[strlen(path)-4] = '\0'; + wpath[wcslen(wpath)-4] = '\0'; if ((!exe_only || isexe) && _waccess(wpath, F_OK) == 0) { - - if (!(GetFileAttributesW(wpath) & FILE_ATTRIBUTE_DIRECTORY)) + if (!(GetFileAttributesW(wpath) & FILE_ATTRIBUTE_DIRECTORY)) { + path[strlen(path)-4] = '\0'; return xstrdup(path); + } } return NULL; } From 5cc1f4f53f0f3d62cfe049939d0b53034dd17069 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Thu, 8 May 2014 21:43:24 +0200 Subject: [PATCH 012/102] Config option to disable side-band-64k for transport Since commit 0c499ea60f the send-pack builtin uses the side-band-64k capability if advertised by the server. Unfortunately this breaks pushing over the dump git protocol if used over a network connection. The detailed reasons for this breakage are (by courtesy of Jeff Preshing, quoted from ttps://groups.google.com/d/msg/msysgit/at8D7J-h7mw/eaLujILGUWoJ): ---------------------------------------------------------------------------- MinGW wraps Windows sockets in CRT file descriptors in order to mimic the functionality of POSIX sockets. This causes msvcrt.dll to treat sockets as Installable File System (IFS) handles, calling ReadFile, WriteFile, DuplicateHandle and CloseHandle on them. This approach works well in simple cases on recent versions of Windows, but does not support all usage patterns. In particular, using this approach, any attempt to read & write concurrently on the same socket (from one or more processes) will deadlock in a scenario where the read waits for a response from the server which is only invoked after the write. This is what send_pack currently attempts to do in the use_sideband codepath. ---------------------------------------------------------------------------- The new config option "sendpack.sideband" allows to override the side-band-64k capability of the server, and thus makes the dump git protocol work. Other transportation methods like ssh and http/https still benefit from the sideband channel, therefore the default value of "sendpack.sideband" is still true. [jes: split out the documentation into Documentation/config/] Signed-off-by: Thomas Braun Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 2 ++ Documentation/config/sendpack.txt | 5 +++++ send-pack.c | 14 +++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Documentation/config/sendpack.txt diff --git a/Documentation/config.txt b/Documentation/config.txt index d87846faa6..e9b2d10e99 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -406,6 +406,8 @@ include::config/reset.txt[] include::config/sendemail.txt[] +include::config/sendpack.txt[] + include::config/sequencer.txt[] include::config/showbranch.txt[] diff --git a/Documentation/config/sendpack.txt b/Documentation/config/sendpack.txt new file mode 100644 index 0000000000..e306f657fb --- /dev/null +++ b/Documentation/config/sendpack.txt @@ -0,0 +1,5 @@ +sendpack.sideband:: + Allows to disable the side-band-64k capability for send-pack even + when it is advertised by the server. Makes it possible to work + around a limitation in the git for windows implementation together + with the dump git protocol. Defaults to true. diff --git a/send-pack.c b/send-pack.c index f692686770..9ff928e2f7 100644 --- a/send-pack.c +++ b/send-pack.c @@ -38,6 +38,16 @@ int option_parse_push_signed(const struct option *opt, die("bad %s argument: %s", opt->long_name, arg); } +static int config_use_sideband = 1; + +static int send_pack_config(const char *var, const char *value, void *unused) +{ + if (!strcmp("sendpack.sideband", var)) + config_use_sideband = git_config_bool(var, value); + + return 0; +} + static void feed_object(const struct object_id *oid, FILE *fh, int negative) { if (negative && !has_sha1_file(oid->hash)) @@ -391,6 +401,8 @@ int send_pack(struct send_pack_args *args, struct async demux; const char *push_cert_nonce = NULL; + git_config(send_pack_config, NULL); + /* Does the other end support the reporting? */ if (server_supports("report-status")) status_report = 1; @@ -398,7 +410,7 @@ int send_pack(struct send_pack_args *args, allow_deleting_refs = 1; if (server_supports("ofs-delta")) args->use_ofs_delta = 1; - if (server_supports("side-band-64k")) + if (config_use_sideband && server_supports("side-band-64k")) use_sideband = 1; if (server_supports("quiet")) quiet_supported = 1; From 528c8347f8014b185f54e17c6fdc82f89edb8dfd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 29 May 2017 13:59:31 +0200 Subject: [PATCH 013/102] setup_git_directory(): handle UNC paths correctly The first offset in a UNC path is not the host name, but the folder name after that. This fixes https://github.com/git-for-windows/git/issues/1181 Signed-off-by: Johannes Schindelin --- setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.c b/setup.c index 1be5037f12..7d495248af 100644 --- a/setup.c +++ b/setup.c @@ -916,7 +916,7 @@ static enum discovery_result setup_git_directory_gently_1(struct strbuf *dir, const char *env_ceiling_dirs = getenv(CEILING_DIRECTORIES_ENVIRONMENT); struct string_list ceiling_dirs = STRING_LIST_INIT_DUP; const char *gitdirenv; - int ceil_offset = -1, min_offset = has_dos_drive_prefix(dir->buf) ? 3 : 1; + int ceil_offset = -1, min_offset = offset_1st_component(dir->buf); dev_t current_device = 0; int one_filesystem = 1; From c838fdc17848e61860561fe749ed8316aa4b4a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0smail=20D=C3=B6nmez?= Date: Sat, 16 Jan 2016 18:59:31 +0200 Subject: [PATCH 014/102] Don't let ld strip relocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the first step for enabling ASLR (Address Space Layout Randomization) support. We want to enable ASLR for better protection against exploiting security holes in Git. The problem fixed by this commit is that `ld.exe` seems to be stripping relocations which in turn will break ASLR support. We just make sure it's not stripping the main executable entry. Signed-off-by: İsmail Dönmez Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 3ee7da0e23..0b351b3737 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -553,9 +553,11 @@ else prefix = /usr/ ifeq (MINGW32,$(MSYSTEM)) prefix = /mingw32 + BASIC_LDFLAGS += -Wl,--pic-executable,-e,_mainCRTStartup endif ifeq (MINGW64,$(MSYSTEM)) prefix = /mingw64 + BASIC_LDFLAGS += -Wl,--pic-executable,-e,mainCRTStartup else COMPAT_CFLAGS += -D_USE_32BIT_TIME_T BASIC_LDFLAGS += -Wl,--large-address-aware From 80993b7680d777713d2b16cc20011bc6a227e0c3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Oct 2017 17:32:32 +0200 Subject: [PATCH 015/102] Fix .git/ discovery at the root of UNC shares A very common assumption in Git's source code base is that offset_1st_component() returns either 0 for relative paths, or 1 for absolute paths that start with a slash. In other words, the return value is either 0 or points just after the dir separator. This assumption is not fulfilled when calling offset_1st_component() e.g. on UNC paths on Windows, e.g. "//my-server/my-share". In this case, offset_1st_component() returns the length of the entire string (which is correct, because stripping the last "component" would not result in a valid directory), yet the return value still does not point just after a dir separator. This assumption is most prominently seen in the setup_git_directory_gently_1() function, where we want to append a ".git" component and simply assume that there is already a dir separator. In the UNC example given above, this assumption is incorrect. As a consequence, Git will fail to handle a worktree at the top of a UNC share correctly. Let's fix this by adding a dir separator specifically for that case: we found that there is no first component in the path and it does not end in a dir separator? Then add it. This fixes https://github.com/git-for-windows/git/issues/1320 Signed-off-by: Johannes Schindelin --- setup.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.c b/setup.c index 7d495248af..5b2dd39a62 100644 --- a/setup.c +++ b/setup.c @@ -944,6 +944,12 @@ static enum discovery_result setup_git_directory_gently_1(struct strbuf *dir, if (ceil_offset < 0) ceil_offset = min_offset - 2; + if (min_offset && min_offset == dir->len && + !is_dir_sep(dir->buf[min_offset - 1])) { + strbuf_addch(dir, '/'); + min_offset++; + } + /* * Test in the following order (relative to the dir): * - .git (file containing "gitdir: ") From 7439cf08588e24a2a116fb4b791570a3d3c74250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0smail=20D=C3=B6nmez?= Date: Sat, 16 Jan 2016 19:09:34 +0200 Subject: [PATCH 016/102] Enable DEP and ASLR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable DEP (Data Execution Prevention) and ASLR (Address Space Layout Randomization) support. This applies to both 32bit and 64bit builds and makes it substantially harder to exploit security holes in Git by offering a much more unpredictable attack surface. ASLR interferes with GDB's ability to set breakpoints. A similar issue holds true when compiling with -O2 (in which case single-stepping is messed up because GDB cannot map the code back to the original source code properly). Therefore we simply enable ASLR only when an optimization flag is present in the CFLAGS, using it as an indicator that the developer does not want to debug in GDB anyway. Signed-off-by: İsmail Dönmez Signed-off-by: Johannes Schindelin --- config.mak.uname | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 0b351b3737..6a21ee80d7 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -551,6 +551,12 @@ else ifeq ($(shell expr "$(uname_R)" : '2\.'),2) # MSys2 prefix = /usr/ + # Enable DEP + BASIC_LDFLAGS += -Wl,--nxcompat + # Enable ASLR (unless debugging) + ifneq (,$(findstring -O,$(CFLAGS))) + BASIC_LDFLAGS += -Wl,--dynamicbase + endif ifeq (MINGW32,$(MSYSTEM)) prefix = /mingw32 BASIC_LDFLAGS += -Wl,--pic-executable,-e,_mainCRTStartup From be8808f286b55f53bb3dc1e977e01c0b58c00d3a Mon Sep 17 00:00:00 2001 From: yaras Date: Tue, 1 Mar 2016 16:12:23 +0100 Subject: [PATCH 017/102] Do not mask the username when reading credentials When user is asked for credentials there is no need to mask username, so PROMPT_ASKPASS flag on calling credential_ask_one for login is unnecessary. credential_ask_one internally uses git_prompt which in case of given flag PROMPT_ASKPASS uses masked input method instead of git_terminal_prompt, which does not mask user input. This fixes #675 Signed-off-by: yaras --- credential.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/credential.c b/credential.c index 62be651b03..e9108a9e8a 100644 --- a/credential.c +++ b/credential.c @@ -136,7 +136,9 @@ static void credential_getpass(struct credential *c) { if (!c->username) c->username = credential_ask_one("Username", c, - PROMPT_ASKPASS|PROMPT_ECHO); + (getenv("GIT_ASKPASS") ? + PROMPT_ASKPASS : 0) | + PROMPT_ECHO); if (!c->password) c->password = credential_ask_one("Password", c, PROMPT_ASKPASS); From 1f10e97a310917e46512960b92203808f8937f30 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 20 Dec 2016 17:18:05 +0100 Subject: [PATCH 018/102] poll: lazy-load GetTickCount64() This fixes the compilation, actually, as we still did not make the jump to post-Windows XP completely: we still compile with _WIN32_WINNT set to 0x0502 (which corresponds to Windows Server 2003 and is technically greater than Windows XP's 0x0501). However, GetTickCount64() is only available starting with Windows Vista/Windows Server 2008. Let's just lazy-load the function, which should also help Git for Windows contributors who want to reinstate Windows XP support. Signed-off-by: Johannes Schindelin --- compat/poll/poll.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/compat/poll/poll.c b/compat/poll/poll.c index 4459408c7d..8e6b8860c5 100644 --- a/compat/poll/poll.c +++ b/compat/poll/poll.c @@ -269,6 +269,20 @@ win32_compute_revents_socket (SOCKET h, int sought, long lNetworkEvents) return happened; } +#include +#include "compat/win32/lazyload.h" + +static ULONGLONG CompatGetTickCount64(void) +{ + DECLARE_PROC_ADDR(kernel32.dll, ULONGLONG, GetTickCount64, void); + + if (!INIT_PROC_ADDR(GetTickCount64)) + return (ULONGLONG)GetTickCount(); + + return GetTickCount64(); +} +#define GetTickCount64 CompatGetTickCount64 + #else /* !MinGW */ /* Convert select(2) returned fd_sets into poll(2) revents values. */ From e624b122a28e2ac68dba90913a5181978557d39a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 11 Jan 2017 21:42:45 +0100 Subject: [PATCH 019/102] t5580: verify that alternates can be UNC paths On Windows, UNC paths are a very convenient way to share data, and alternates are all about sharing data. We fixed a bug where alternates specifying UNC paths were not handled properly, and it is high time that we add a regression test to ensure that this bug is not reintroduced. Signed-off-by: Johannes Schindelin --- t/t5580-clone-push-unc.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-clone-push-unc.sh index ba548df4a9..265cd50526 100755 --- a/t/t5580-clone-push-unc.sh +++ b/t/t5580-clone-push-unc.sh @@ -57,4 +57,16 @@ test_expect_success MINGW 'remote nick cannot contain backslashes' ' test_i18ngrep ! "unable to access" err ' +test_expect_success 'unc alternates' ' + tree="$(git rev-parse HEAD:)" && + mkdir test-unc-alternate && + ( + cd test-unc-alternate && + git init && + test_must_fail git show $tree && + echo "$UNCPATH/.git/objects" >.git/objects/info/alternates && + git show $tree + ) +' + test_done From eb44548a504e607aa4adf7bc51e2ea573ad93632 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 2 Mar 2018 22:40:19 +0100 Subject: [PATCH 020/102] setup_git_directory(): handle UNC root paths correctly When working in the root directory of a file share (this is only possible in Git Bash and Powershell, but not in CMD), the current directory is reported without a trailing slash. This is different from Unix and standard Windows directories: both / and C:\ are reported with a trailing slash as current directories. If a Git worktree is located there, Git is not quite prepared for that: while it does manage to find the .git directory/file, it returns as length of the top-level directory's path *one more* than the length of the current directory, and setup_git_directory_gently() would then return an undefined string as prefix. In practice, this undefined string usually points to NUL bytes, and does not cause much harm. Under rare circumstances that are really involved to reproduce (and not reliably so), the reported prefix could be a suffix string of Git's exec path, though. A careful analysis determined that this bug is unlikely to be exploitable, therefore we mark this as a regular bug fix. Signed-off-by: Johannes Schindelin --- setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.c b/setup.c index 5b2dd39a62..624da83685 100644 --- a/setup.c +++ b/setup.c @@ -784,7 +784,7 @@ static const char *setup_discovered_git_dir(const char *gitdir, set_git_dir(gitdir); inside_git_dir = 0; inside_work_tree = 1; - if (offset == cwd->len) + if (offset >= cwd->len) return NULL; /* Make "offset" point past the '/' (already the case for root dirs) */ From 145c57175ded0f4193dcf2fa68619b5236b62cf8 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 15 Dec 2016 11:34:39 -0500 Subject: [PATCH 021/102] diffcore-rename: speed up register_rename_src Teach register_rename_src() to see if new file pair can simply be appended to the rename_src[] array before performing the binary search to find the proper insertion point. This is a performance optimization. This routine is called during run_diff_files in status and the caller is iterating over the sorted index, so we should expect to be able to append in the normal case. The existing insert logic is preserved so we don't have to assume that, but simply take advantage of it if possible. Signed-off-by: Jeff Hostetler --- diffcore-rename.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/diffcore-rename.c b/diffcore-rename.c index 07bd34b631..5bfc5f6c22 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -82,6 +82,18 @@ static struct diff_rename_src *register_rename_src(struct diff_filepair *p) first = 0; last = rename_src_nr; + + if (last > 0) { + struct diff_rename_src *src = &(rename_src[last-1]); + int cmp = strcmp(one->path, src->p->one->path); + if (!cmp) + return src; + if (cmp > 0) { + first = last; + goto append_it; + } + } + while (last > first) { int next = (last + first) >> 1; struct diff_rename_src *src = &(rename_src[next]); @@ -95,6 +107,7 @@ static struct diff_rename_src *register_rename_src(struct diff_filepair *p) first = next+1; } +append_it: /* insert to make it at "first" */ ALLOC_GROW(rename_src, rename_src_nr + 1, rename_src_alloc); rename_src_nr++; From c667118031cac510d3ac7d484bfdd6dffefd2691 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 6 Apr 2017 00:56:44 +0200 Subject: [PATCH 022/102] Move Windows-specific config settings into compat/mingw.c Signed-off-by: Johannes Schindelin --- setup.c | 6 +++--- t/t3700-add.sh | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/setup.c b/setup.c index 1be5037f12..291bfb2128 100644 --- a/setup.c +++ b/setup.c @@ -39,7 +39,7 @@ static int abspath_part_inside_repo(char *path) off = offset_1st_component(path); /* check if work tree is already the prefix */ - if (wtlen <= len && !strncmp(path, work_tree, wtlen)) { + if (wtlen <= len && !fspathncmp(path, work_tree, wtlen)) { if (path[wtlen] == '/') { memmove(path, path + wtlen + 1, len - wtlen); return 0; @@ -59,7 +59,7 @@ static int abspath_part_inside_repo(char *path) path++; if (*path == '/') { *path = '\0'; - if (strcmp(real_path(path0), work_tree) == 0) { + if (fspathcmp(real_path(path0), work_tree) == 0) { memmove(path0, path + 1, len - (path - path0)); return 0; } @@ -68,7 +68,7 @@ static int abspath_part_inside_repo(char *path) } /* check whole path */ - if (strcmp(real_path(path0), work_tree) == 0) { + if (fspathcmp(real_path(path0), work_tree) == 0) { *path0 = '\0'; return 0; } diff --git a/t/t3700-add.sh b/t/t3700-add.sh index 37729ba258..8ee4fc70ad 100755 --- a/t/t3700-add.sh +++ b/t/t3700-add.sh @@ -402,4 +402,11 @@ test_expect_success 'all statuses changed in folder if . is given' ' test $(git ls-files --stage | grep ^100755 | wc -l) -eq 0 ' +test_expect_success MINGW 'path is case-insensitive' ' + path="$(pwd)/BLUB" && + touch "$path" && + downcased="$(echo "$path" | tr A-Z a-z)" && + git add "$downcased" +' + test_done From b2e138468860142ae645b74816be4cb1f30bde22 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Jul 2017 22:17:20 +0200 Subject: [PATCH 023/102] tests: add t/helper/ to the PATH with --with-dashes We really need to be able to find the test helpers... Really. This change was forgotten when we moved the test helpers into t/helper/ Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 0f1faa24b2..50563daef0 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -997,7 +997,7 @@ else # normal case, use ../bin-wrappers only unless $with_dashes: GIT_EXEC_PATH=$GIT_BUILD_DIR if test -n "$with_dashes" then - PATH="$GIT_BUILD_DIR:$PATH" + PATH="$GIT_BUILD_DIR:$GIT_BUILD_DIR/t/helper:$PATH" fi fi GIT_TEMPLATE_DIR="$GIT_BUILD_DIR"/templates/blt From 1e326feb9bf3cfc835529f7558c79d81acbad3e9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 Apr 2017 12:09:08 +0200 Subject: [PATCH 024/102] mingw: demonstrate a problem with certain absolute paths On Windows, there are several categories of absolute paths. One such category starts with a backslash and is implicitly relative to the drive associated with the current working directory. Example: c: git clone https://github.com/git-for-windows/git \G4W should clone into C:\G4W. There is currently a problem with that, in that mingw_mktemp() does not expect the _wmktemp() function to prefix the absolute path with the drive prefix, and as a consequence, the resulting path does not fit into the originally-passed string buffer. The symptom is a "Result too large" error. Reported by Juan Carlos Arevalo Baeza. Signed-off-by: Johannes Schindelin --- t/t5580-clone-push-unc.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-clone-push-unc.sh index ba548df4a9..c2b0082296 100755 --- a/t/t5580-clone-push-unc.sh +++ b/t/t5580-clone-push-unc.sh @@ -17,14 +17,11 @@ fi UNCPATH="$(winpwd)" case "$UNCPATH" in [A-Z]:*) + WITHOUTDRIVE="${UNCPATH#?:}" # Use administrative share e.g. \\localhost\C$\git-sdk-64\usr\src\git # (we use forward slashes here because MSYS2 and Git accept them, and # they are easier on the eyes) - UNCPATH="//localhost/${UNCPATH%%:*}\$/${UNCPATH#?:}" - test -d "$UNCPATH" || { - skip_all='could not access administrative share; skipping' - test_done - } + UNCPATH="//localhost/${UNCPATH%%:*}\$$WITHOUTDRIVE" ;; *) skip_all='skipping UNC path tests, cannot determine current path as UNC' @@ -32,6 +29,18 @@ case "$UNCPATH" in ;; esac +test_expect_failure 'clone into absolute path lacking a drive prefix' ' + USINGBACKSLASHES="$(echo "$WITHOUTDRIVE"/without-drive-prefix | + tr / \\\\)" && + git clone . "$USINGBACKSLASHES" && + test -f without-drive-prefix/.git/HEAD +' + +test -d "$UNCPATH" || { + skip_all='could not access administrative share; skipping' + test_done +} + test_expect_success setup ' test_commit initial ' From dd16ecdaf65d4e4bb5e01d5d5b66263e98972dd3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 19 Jun 2017 17:11:16 +0200 Subject: [PATCH 025/102] mingw (t5580): document bug when cloning from backslashed UNC paths Due to a quirk in Git's method to spawn git-upload-pack, there is a problem when passing paths with backslashes in them: Git will force the command-line through the shell, which has different quoting semantics in Git for Windows (being an MSYS2 program) than regular Win32 executables such as git.exe itself. The symptom is that the first of the two backslashes in UNC paths of the form \\myserver\folder\repository.git is *stripped off*. Document this bug by introducing a test case. Signed-off-by: Johannes Schindelin --- t/t5580-clone-push-unc.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-clone-push-unc.sh index ba548df4a9..c3703765f4 100755 --- a/t/t5580-clone-push-unc.sh +++ b/t/t5580-clone-push-unc.sh @@ -40,6 +40,11 @@ test_expect_success clone ' git clone "file://$UNCPATH" clone ' +test_expect_failure 'clone with backslashed path' ' + BACKSLASHED="$(echo "$UNCPATH" | tr / \\\\)" && + git clone "$BACKSLASHED" backslashed +' + test_expect_success push ' ( cd clone && From 9685a45a7e1cca46210d352e9ddcdad70bddb697 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Oct 2018 14:19:06 +0200 Subject: [PATCH 026/102] t0061: fix with --with-dashes and RUNTIME_PREFIX When building Git with RUNTIME_PREFIX and starting a test helper from t/helper/, it fails to detect the system prefix correctly. This is the reason that the warning RUNTIME_PREFIX requested, but prefix computation failed. [...] to be printed. In t0061, we did not expect that to happen, and it actually did not happen in the normal case, because bin-wrappers/test-tool specifically sets GIT_TEXTDOMAINDIR (and as a consequence, nothing in test-tool wants to know about the runtime prefix). However, with --with-dashes, bin-wrappers/test-tool is no longer called, but t/helper/test-tool is called directly. So let's just ignore the RUNTIME_PREFIX warning. Signed-off-by: Johannes Schindelin --- t/t0061-run-command.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index 96bf6d6a7d..c13ca678ad 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -158,7 +158,8 @@ test_trace () { expect="$1" shift GIT_TRACE=1 test-tool run-command "$@" run-command true 2>&1 >/dev/null | \ - sed -e 's/.* run_command: //' -e '/trace: .*/d' >actual && + sed -e 's/.* run_command: //' -e '/trace: .*/d' \ + -e '/RUNTIME_PREFIX requested/d' >actual && echo "$expect true" >expect && test_cmp expect actual } From 4d74e8c1e24a59574b21d5a48510f718dfaf229c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 Apr 2017 12:38:30 +0200 Subject: [PATCH 027/102] mingw: allow absolute paths without drive prefix When specifying an absolute path without a drive prefix, we convert that path internally. Let's make sure that we handle that case properly, too ;-) This fixes the command git clone https://github.com/git-for-windows/git \G4W Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 +++++++++- t/t5580-clone-push-unc.sh | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 34b3880b29..4d009901d8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -928,11 +928,19 @@ unsigned int sleep (unsigned int seconds) char *mingw_mktemp(char *template) { wchar_t wtemplate[MAX_PATH]; + int offset = 0; + if (xutftowcs_path(wtemplate, template) < 0) return NULL; + + if (is_dir_sep(template[0]) && !is_dir_sep(template[1]) && + iswalpha(wtemplate[0]) && wtemplate[1] == L':') { + /* We have an absolute path missing the drive prefix */ + offset = 2; + } if (!_wmktemp(wtemplate)) return NULL; - if (xwcstoutf(template, wtemplate, strlen(template) + 1) < 0) + if (xwcstoutf(template, wtemplate + offset, strlen(template) + 1) < 0) return NULL; return template; } diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-clone-push-unc.sh index c2b0082296..17c38c33a5 100755 --- a/t/t5580-clone-push-unc.sh +++ b/t/t5580-clone-push-unc.sh @@ -29,7 +29,7 @@ case "$UNCPATH" in ;; esac -test_expect_failure 'clone into absolute path lacking a drive prefix' ' +test_expect_success 'clone into absolute path lacking a drive prefix' ' USINGBACKSLASHES="$(echo "$WITHOUTDRIVE"/without-drive-prefix | tr / \\\\)" && git clone . "$USINGBACKSLASHES" && From 24d0bffd341df419d061201cc1464197f1a118b8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 8 Jun 2016 08:32:21 +0200 Subject: [PATCH 028/102] gc/repack: release packs when needed On Windows, files cannot be removed nor renamed if there are still handles held by a process. To remedy that, we introduced the close_all_packs() function. Earlier, we made sure that the packs are released just before `git gc` is spawned, in case that gc wants to remove no-longer needed packs. But this developer forgot that gc itself also needs to let go of packs, e.g. when consolidating all packs via the --aggressive option. Likewise, `git repack -d` wants to delete obsolete packs and therefore needs to close all pack handles, too. Signed-off-by: Johannes Schindelin --- builtin/gc.c | 4 +++- builtin/repack.c | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/builtin/gc.c b/builtin/gc.c index 871a56f1c5..df90fd7f51 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -659,8 +659,10 @@ int cmd_gc(int argc, const char **argv, const char *prefix) report_garbage = report_pack_garbage; reprepare_packed_git(the_repository); - if (pack_garbage.nr > 0) + if (pack_garbage.nr > 0) { + close_all_packs(the_repository->objects); clean_pack_garbage(); + } if (gc_write_commit_graph) write_commit_graph_reachable(get_object_directory(), 0, diff --git a/builtin/repack.c b/builtin/repack.c index 45583683ee..f9319defe4 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -419,6 +419,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (!names.nr && !po_args.quiet) printf("Nothing new to pack.\n"); + close_all_packs(the_repository->objects); + /* * Ok we have prepared all new packfiles. * First see if there are packs of the same name and if so From ae6a15aa9300f86d8ae3b349d49d4d0c0a92d549 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 19 Jun 2017 16:35:17 +0200 Subject: [PATCH 029/102] mingw: special-case arguments to `sh` The MSYS2 runtime does its best to emulate the command-line wildcard expansion and de-quoting which would be performed by the calling Unix shell on Unix systems. Those Unix shell quoting rules differ from the quoting rules applying to Windows' cmd and Powershell, making it a little awkward to quote command-line parameters properly when spawning other processes. In particular, git.exe passes arguments to subprocesses that are *not* intended to be interpreted as wildcards, and if they contain backslashes, those are not to be interpreted as escape characters, e.g. when passing Windows paths. Note: this is only a problem when calling MSYS2 executables, not when calling MINGW executables such as git.exe. However, we do call MSYS2 executables frequently, most notably when setting the use_shell flag in the child_process structure. There is no elegant way to determine whether the .exe file to be executed is an MSYS2 program or a MINGW one. But since the use case of passing a command line through the shell is so prevalent, we need to work around this issue at least when executing sh.exe. Let's introduce an ugly, hard-coded test whether argv[0] is "sh", and whether it refers to the MSYS2 Bash, to determine whether we need to quote the arguments differently than usual. That still does not fix the issue completely, but at least it is something. Incidentally, this also fixes the problem where `git clone \\server\repo` failed due to incorrect handling of the backslashes when handing the path to the git-upload-pack process. We need to take care to quote not only whitespace, but also curly brackets. As aliases frequently go through the MSYS2 Bash, and as aliases frequently get parameters such as HEAD@{yesterday}, let's make sure that this does not regress by adding a test case for that. Helped-by: Kim Gybels Signed-off-by: Johannes Schindelin --- compat/mingw.c | 63 ++++++++++++++++++++++++++++++++++++++- t/t0061-run-command.sh | 10 +++++++ t/t5580-clone-push-unc.sh | 2 +- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 34b3880b29..8e530bc51f 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1031,7 +1031,7 @@ char *mingw_getcwd(char *pointer, int len) * See "Parsing C++ Command-Line Arguments" at Microsoft's Docs: * https://docs.microsoft.com/en-us/cpp/cpp/parsing-cpp-command-line-arguments */ -static const char *quote_arg(const char *arg) +static const char *quote_arg_msvc(const char *arg) { /* count chars to quote */ int len = 0, n = 0; @@ -1086,6 +1086,37 @@ static const char *quote_arg(const char *arg) return q; } +#include "quote.h" + +static const char *quote_arg_msys2(const char *arg) +{ + struct strbuf buf = STRBUF_INIT; + const char *p2 = arg, *p; + + for (p = arg; *p; p++) { + int ws = isspace(*p); + if (!ws && *p != '\\' && *p != '"' && *p != '{') + continue; + if (!buf.len) + strbuf_addch(&buf, '"'); + if (p != p2) + strbuf_add(&buf, p2, p - p2); + if (!ws && *p != '{') + strbuf_addch(&buf, '\\'); + p2 = p; + } + + if (p == arg) + strbuf_addch(&buf, '"'); + else if (!buf.len) + return arg; + else + strbuf_add(&buf, p2, p - p2), + + strbuf_addch(&buf, '"'); + return strbuf_detach(&buf, 0); +} + static const char *parse_interpreter(const char *cmd) { static char buf[100]; @@ -1317,6 +1348,34 @@ struct pinfo_t { static struct pinfo_t *pinfo = NULL; CRITICAL_SECTION pinfo_cs; +static int is_msys2_sh(const char *cmd) +{ + if (cmd && !strcmp(cmd, "sh")) { + static int ret = -1; + char *p; + + if (ret >= 0) + return ret; + + p = path_lookup(cmd, 0); + if (!p) + ret = 0; + else { + size_t len = strlen(p); + ret = len > 15 && + is_dir_sep(p[len - 15]) && + !strncasecmp(p + len - 14, "usr", 3) && + is_dir_sep(p[len - 11]) && + !strncasecmp(p + len - 10, "bin", 3) && + is_dir_sep(p[len - 7]) && + !strcasecmp(p + len - 6, "sh.exe"); + free(p); + } + return ret; + } + return 0; +} + static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaenv, const char *dir, int prepend_cmd, int fhin, int fhout, int fherr) @@ -1328,6 +1387,8 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen unsigned flags = CREATE_UNICODE_ENVIRONMENT; BOOL ret; HANDLE cons; + const char *(*quote_arg)(const char *arg) = + is_msys2_sh(*argv) ? quote_arg_msys2 : quote_arg_msvc; do_unset_environment_variables(); diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index 96bf6d6a7d..db5f0222d7 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -191,4 +191,14 @@ test_expect_success 'GIT_TRACE with environment variables' ' ) ' +test_expect_success MINGW 'verify curlies are quoted properly' ' + : force the rev-parse through the MSYS2 Bash && + git -c alias.r="!git rev-parse" r -- a{b}c >actual && + cat >expect <<-\EOF && + -- + a{b}c + EOF + test_cmp expect actual +' + test_done diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-clone-push-unc.sh index c3703765f4..217adf3a63 100755 --- a/t/t5580-clone-push-unc.sh +++ b/t/t5580-clone-push-unc.sh @@ -40,7 +40,7 @@ test_expect_success clone ' git clone "file://$UNCPATH" clone ' -test_expect_failure 'clone with backslashed path' ' +test_expect_success 'clone with backslashed path' ' BACKSLASHED="$(echo "$UNCPATH" | tr / \\\\)" && git clone "$BACKSLASHED" backslashed ' From 91d7e87371784c13d73e247f332ba6e0abe9933e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 13 Jul 2017 14:28:42 +0200 Subject: [PATCH 030/102] t5580: test cloning without file://, test fetching via UNC paths It gets a bit silly to add the commands to the name of the test script, so let's just rename it while we're testing more UNC stuff. Signed-off-by: Johannes Schindelin --- t/{t5580-clone-push-unc.sh => t5580-unc-paths.sh} | 12 ++++++++++++ 1 file changed, 12 insertions(+) rename t/{t5580-clone-push-unc.sh => t5580-unc-paths.sh} (88%) diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-unc-paths.sh similarity index 88% rename from t/t5580-clone-push-unc.sh rename to t/t5580-unc-paths.sh index 217adf3a63..254fefccde 100755 --- a/t/t5580-clone-push-unc.sh +++ b/t/t5580-unc-paths.sh @@ -40,11 +40,23 @@ test_expect_success clone ' git clone "file://$UNCPATH" clone ' +test_expect_success 'clone without file://' ' + git clone "$UNCPATH" clone-without-file +' + test_expect_success 'clone with backslashed path' ' BACKSLASHED="$(echo "$UNCPATH" | tr / \\\\)" && git clone "$BACKSLASHED" backslashed ' +test_expect_success fetch ' + git init to-fetch && + ( + cd to-fetch && + git fetch "$UNCPATH" master + ) +' + test_expect_success push ' ( cd clone && From 9dc07d8190858ae2fc1516df6c12fae374ecabf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20B=C3=B6gershausen?= Date: Tue, 15 Aug 2017 08:55:38 +0200 Subject: [PATCH 031/102] mingw: support UNC in git clone file://server/share/repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the parser to accept file://server/share/repo in the way that Windows users expect it to be parsed who are used to referring to file shares by UNC paths of the form \\server\share\folder. [jes: tightened check to avoid handling file://C:/some/path as a UNC path.] This closes https://github.com/git-for-windows/git/issues/1264. Signed-off-by: Torsten Bögershausen Signed-off-by: Johannes Schindelin --- connect.c | 4 ++++ t/t5500-fetch-pack.sh | 13 +++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/connect.c b/connect.c index 24281b6082..d72772a36d 100644 --- a/connect.c +++ b/connect.c @@ -918,6 +918,10 @@ static enum protocol parse_connect_url(const char *url_orig, char **ret_host, if (protocol == PROTO_LOCAL) path = end; + else if (protocol == PROTO_FILE && *host != '/' && + !has_dos_drive_prefix(host) && + offset_1st_component(host - 2) > 1) + path = host - 2; /* include the leading "//" */ else if (protocol == PROTO_FILE && has_dos_drive_prefix(end)) path = end; /* "file://$(pwd)" may be "file://C:/projects/repo" */ else diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index 086f2c40f6..f021db44b1 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -698,13 +698,22 @@ do # file with scheme for p in file do - test_expect_success "fetch-pack --diag-url $p://$h/$r" ' + test_expect_success !MINGW "fetch-pack --diag-url $p://$h/$r" ' check_prot_path $p://$h/$r $p "/$r" ' + test_expect_success MINGW "fetch-pack --diag-url $p://$h/$r" ' + check_prot_path $p://$h/$r $p "//$h/$r" + ' + test_expect_success MINGW "fetch-pack --diag-url $p:///$r" ' + check_prot_path $p:///$r $p "/$r" + ' # No "/~" -> "~" conversion for file - test_expect_success "fetch-pack --diag-url $p://$h/~$r" ' + test_expect_success !MINGW "fetch-pack --diag-url $p://$h/~$r" ' check_prot_path $p://$h/~$r $p "/~$r" ' + test_expect_success MINGW "fetch-pack --diag-url $p://$h/~$r" ' + check_prot_path $p://$h/~$r $p "//$h/~$r" + ' done # file without scheme for h in nohost nohost:12 [::1] [::1]:23 [ [:aa From 17ff2d1502facb3c5fbc365444c21bb9c7f5aee2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 24 Oct 2018 18:34:09 +0200 Subject: [PATCH 032/102] tests: optionally skip bin-wrappers/ This speeds up the tests by a bit on Windows, where running Unix shell scripts (and spawning processes) is not exactly a cheap operation. Signed-off-by: Johannes Schindelin --- t/README | 9 +++++++++ t/test-lib.sh | 21 ++++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/t/README b/t/README index 28711cc508..3f645b083a 100644 --- a/t/README +++ b/t/README @@ -170,6 +170,15 @@ appropriately before running "make". implied by other options like --valgrind and GIT_TEST_INSTALLED. +--no-bin-wrappers:: + By default, the test suite uses the wrappers in + `../bin-wrappers/` to execute `git` and friends. With this option, + `../git` and friends are run directly. This is not recommended + in general, as the wrappers contain safeguards to ensure that no + files from an installed Git are used, but can speed up test runs + especially on platforms where running shell scripts is expensive + (most notably, Windows). + --root=:: Create "trash" directories used to store all temporary data during testing under , instead of the t/ directory. diff --git a/t/test-lib.sh b/t/test-lib.sh index 50563daef0..14dff5cf1f 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -294,6 +294,8 @@ do test -z "$HARNESS_ACTIVE" && quiet=t; shift ;; --with-dashes) with_dashes=t; shift ;; + --no-bin-wrappers) + no_bin_wrappers=t; shift ;; --no-color) color=; shift ;; --va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind) @@ -984,16 +986,21 @@ then PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH} else # normal case, use ../bin-wrappers only unless $with_dashes: - git_bin_dir="$GIT_BUILD_DIR/bin-wrappers" - if ! test -x "$git_bin_dir/git" + if test -n "$no_bin_wrappers" then - if test -z "$with_dashes" - then - say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH" - fi with_dashes=t + else + git_bin_dir="$GIT_BUILD_DIR/bin-wrappers" + if ! test -x "$git_bin_dir/git" + then + if test -z "$with_dashes" + then + say "$git_bin_dir/git is not executable; using GIT_EXEC_PATH" + fi + with_dashes=t + fi + PATH="$git_bin_dir:$PATH" fi - PATH="$git_bin_dir:$PATH" GIT_EXEC_PATH=$GIT_BUILD_DIR if test -n "$with_dashes" then From 7bf19defbde01e6afb8c983c152389dca296a01a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 30 Aug 2018 21:16:47 +0200 Subject: [PATCH 033/102] ci: rename the library of common functions The name is hard-coded to reflect that we use Travis CI for continuous testing. In the next commits, we will extend this to be able use Azure DevOps, too. So let's adjust the name to make it more generic. Signed-off-by: Johannes Schindelin --- ci/install-dependencies.sh | 2 +- ci/{lib-travisci.sh => lib.sh} | 0 ci/print-test-failures.sh | 2 +- ci/run-build-and-tests.sh | 2 +- ci/run-linux32-docker.sh | 2 +- ci/run-static-analysis.sh | 2 +- ci/run-windows-build.sh | 2 +- ci/test-documentation.sh | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename ci/{lib-travisci.sh => lib.sh} (100%) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 06c3546e1e..fe65144152 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -3,7 +3,7 @@ # Install dependencies required to build and test Git on Linux and macOS # -. ${0%/*}/lib-travisci.sh +. ${0%/*}/lib.sh P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION diff --git a/ci/lib-travisci.sh b/ci/lib.sh similarity index 100% rename from ci/lib-travisci.sh rename to ci/lib.sh diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index d55460a212..7aef39a2fd 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -3,7 +3,7 @@ # Print output of failing tests # -. ${0%/*}/lib-travisci.sh +. ${0%/*}/lib.sh # Tracing executed commands would produce too much noise in the loop below. set +x diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index cda170d5c2..db342bb6a8 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -3,7 +3,7 @@ # Build and test Git # -. ${0%/*}/lib-travisci.sh +. ${0%/*}/lib.sh ln -s "$cache_dir/.prove" t/.prove diff --git a/ci/run-linux32-docker.sh b/ci/run-linux32-docker.sh index 21637903ce..751acfcf8a 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-linux32-docker.sh @@ -3,7 +3,7 @@ # Download and run Docker image to build and test 32-bit Git # -. ${0%/*}/lib-travisci.sh +. ${0%/*}/lib.sh docker pull daald/ubuntu32:xenial diff --git a/ci/run-static-analysis.sh b/ci/run-static-analysis.sh index 5688f261d0..dc189c7456 100755 --- a/ci/run-static-analysis.sh +++ b/ci/run-static-analysis.sh @@ -3,7 +3,7 @@ # Perform various static code analysis checks # -. ${0%/*}/lib-travisci.sh +. ${0%/*}/lib.sh make --jobs=2 coccicheck diff --git a/ci/run-windows-build.sh b/ci/run-windows-build.sh index d99a180e52..a73a4eca0a 100755 --- a/ci/run-windows-build.sh +++ b/ci/run-windows-build.sh @@ -6,7 +6,7 @@ # supported) and a commit hash. # -. ${0%/*}/lib-travisci.sh +. ${0%/*}/lib.sh test $# -ne 2 && echo "Unexpected number of parameters" && exit 1 test -z "$GFW_CI_TOKEN" && echo "GFW_CI_TOKEN not defined" && exit diff --git a/ci/test-documentation.sh b/ci/test-documentation.sh index a20de9ca12..d3cdbac73f 100755 --- a/ci/test-documentation.sh +++ b/ci/test-documentation.sh @@ -3,7 +3,7 @@ # Perform sanity checks on documentation and build it. # -. ${0%/*}/lib-travisci.sh +. ${0%/*}/lib.sh gem install asciidoctor From 074988290354b992c32a9a18ea19efa22e04d22a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 30 Aug 2018 21:57:28 +0200 Subject: [PATCH 034/102] ci/lib.sh: encapsulate Travis-specific things The upcoming patches will allow building git.git via Azure Pipelines (i.e. Azure DevOps' Continuous Integration), where variable names and URLs look a bit different than in Travis CI. Signed-off-by: Johannes Schindelin --- ci/install-dependencies.sh | 3 ++- ci/lib.sh | 44 +++++++++++++++++++++++++++----------- ci/print-test-failures.sh | 2 +- ci/test-documentation.sh | 1 + 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index fe65144152..bcdcc71592 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -37,7 +37,8 @@ osx-clang|osx-gcc) brew update --quiet # Uncomment this if you want to run perf tests: # brew install gnu-time - brew install git-lfs gettext + test -z "$BREW_INSTALL_PACKAGES" || + brew install $BREW_INSTALL_PACKAGES brew link --force gettext brew install caskroom/cask/perforce ;; diff --git a/ci/lib.sh b/ci/lib.sh index 69dff4d1ec..a0182ddc6b 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -1,5 +1,26 @@ # Library of functions shared by all CI scripts +if test true = "$TRAVIS" +then + # We are running within Travis CI + CI_BRANCH="$TRAVIS_BRANCH" + CI_COMMIT="$TRAVIS_COMMIT" + CI_JOB_ID="$TRAVIS_JOB_ID" + CI_JOB_NUMBER="$TRAVIS_JOB_NUMBER" + CI_OS_NAME="$TRAVIS_OS_NAME" + CI_REPO_SLUG="$TRAVIS_REPO_SLUG" + + cache_dir="$HOME/travis-cache" + + url_for_job_id () { + echo "https://travis-ci.org/$CI_REPO_SLUG/jobs/$1" + } + + BREW_INSTALL_PACKAGES="git-lfs gettext" + export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" + export GIT_TEST_OPTS="--verbose-log -x --immediate" +fi + skip_branch_tip_with_tag () { # Sometimes, a branch is pushed at the same time the tag that points # at the same commit as the tip of the branch is pushed, and building @@ -13,10 +34,10 @@ skip_branch_tip_with_tag () { # we can skip the build because we won't be skipping a build # of a tag. - if TAG=$(git describe --exact-match "$TRAVIS_BRANCH" 2>/dev/null) && - test "$TAG" != "$TRAVIS_BRANCH" + if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) && + test "$TAG" != "$CI_BRANCH" then - echo "$(tput setaf 2)Tip of $TRAVIS_BRANCH is exactly at $TAG$(tput sgr0)" + echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)" exit 0 fi } @@ -25,7 +46,7 @@ skip_branch_tip_with_tag () { # job if we encounter the same tree again and can provide a useful info # message. save_good_tree () { - echo "$(git rev-parse $TRAVIS_COMMIT^{tree}) $TRAVIS_COMMIT $TRAVIS_JOB_NUMBER $TRAVIS_JOB_ID" >>"$good_trees_file" + echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file" # limit the file size tail -1000 "$good_trees_file" >"$good_trees_file".tmp mv "$good_trees_file".tmp "$good_trees_file" @@ -35,7 +56,7 @@ save_good_tree () { # successfully before (e.g. because the branch got rebased, changing only # the commit messages). skip_good_tree () { - if ! good_tree_info="$(grep "^$(git rev-parse $TRAVIS_COMMIT^{tree}) " "$good_trees_file")" + if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")" then # Haven't seen this tree yet, or no cached good trees file yet. # Continue the build job. @@ -45,18 +66,18 @@ skip_good_tree () { echo "$good_tree_info" | { read tree prev_good_commit prev_good_job_number prev_good_job_id - if test "$TRAVIS_JOB_ID" = "$prev_good_job_id" + if test "$CI_JOB_ID" = "$prev_good_job_id" then cat <<-EOF - $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) This commit has already been built and tested successfully by this build job. To force a re-build delete the branch's cache and then hit 'Restart job'. EOF else cat <<-EOF - $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. - The log of that build job is available at https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$prev_good_job_id + The log of that build job is available at $(url_for_job_id $prev_good_job_id) To force a re-build delete the branch's cache and then hit 'Restart job'. EOF fi @@ -81,7 +102,6 @@ check_unignored_build_artifacts () # and installing dependencies. set -ex -cache_dir="$HOME/travis-cache" good_trees_file="$cache_dir/good-trees" mkdir -p "$cache_dir" @@ -91,13 +111,11 @@ skip_good_tree if test -z "$jobname" then - jobname="$TRAVIS_OS_NAME-$CC" + jobname="$CI_OS_NAME-$CC" fi export DEVELOPER=1 export DEFAULT_TEST_TARGET=prove -export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" -export GIT_TEST_OPTS="--verbose-log -x --immediate" export GIT_TEST_CLONE_2GB=YesPlease if [ "$jobname" = linux-gcc ]; then export CC=gcc-8 diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 7aef39a2fd..d2045b63a6 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -69,7 +69,7 @@ do fi done -if [ $combined_trash_size -gt 0 ] +if [ -n "$TRAVIS_JOB_ID" -a $combined_trash_size -gt 0 ] then echo "------------------------------------------------------------------------" echo "Trash directories embedded in this log can be extracted by running:" diff --git a/ci/test-documentation.sh b/ci/test-documentation.sh index d3cdbac73f..7d0beb2832 100755 --- a/ci/test-documentation.sh +++ b/ci/test-documentation.sh @@ -5,6 +5,7 @@ . ${0%/*}/lib.sh +test -n "$ALREADY_HAVE_ASCIIDOCTOR" || gem install asciidoctor make check-builtins From 521e8d566e13efb74b7f3d01b3609313e889a085 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 12 Dec 2018 12:38:53 +0100 Subject: [PATCH 035/102] ci: inherit --jobs via MAKEFLAGS in run-build-and-tests Let's not decide in the generic ci/ script how many jobs to run in parallel; it is easy enough to hand that information down via the `MAKEFLAGS`. Signed-off-by: Johannes Schindelin --- ci/run-build-and-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index db342bb6a8..80d72d120f 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -7,7 +7,7 @@ ln -s "$cache_dir/.prove" t/.prove -make --jobs=2 +make make --quiet test if test "$jobname" = "linux-gcc" then From 39ce1fdd3a547df6968ed38b1ddcee5c634ecc99 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 12 Dec 2018 13:48:07 +0100 Subject: [PATCH 036/102] ci: use a junction on Windows instead of a symlink Symbolic links are still not quite as easy to use on Windows as on Linux (for example, on versions older than Windows 10, only administrators can create symlinks, and on Windows 10 you still need to be in developer mode for regular users to have permission), but NTFS junctions can give us a way out. Signed-off-by: Johannes Schindelin --- ci/run-build-and-tests.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 80d72d120f..74d838ea01 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -5,7 +5,10 @@ . ${0%/*}/lib.sh -ln -s "$cache_dir/.prove" t/.prove +case "$CI_OS_NAME" in +windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";; +*) ln -s "$cache_dir/.prove" t/.prove;; +esac make make --quiet test From d790b5e9bf1fe85d43583e86dcd86e154da65b72 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 12 Dec 2018 22:09:24 +0100 Subject: [PATCH 037/102] ci: try to work around issues with the test cleanup It seems that on some build agents, there might be some transient file locking issues, so when we cannot delete the trash directory, let's be gentle and try again five seconds later, and only error out if it still fails the second time. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 14dff5cf1f..251f4d6a96 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -880,7 +880,11 @@ test_done () { error "Tests passed but trash directory already removed before test cleanup; aborting" cd "$TRASH_DIRECTORY/.." && - rm -fr "$TRASH_DIRECTORY" || + rm -fr "$TRASH_DIRECTORY" || { + # try again in a bit + sleep 5; + rm -fr "$TRASH_DIRECTORY" + } || error "Tests passed but test cleanup failed; aborting" fi test_at_end_hook_ From 64757af28f27c0e7080ac5cfa9e6b05598539ea7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Aug 2018 11:29:26 +0200 Subject: [PATCH 038/102] test-date: add a subcommand to measure times in shell scripts In the next commit, we want to teach Git's test suite to optionally output test results in JUnit-style .xml files. These files contain information about the time spent. So we need a way to measure time. While we could use `date +%s` for that, this will give us only seconds, i.e. very coarse-grained timings. GNU `date` supports `date +%s.%N` (i.e. nanosecond-precision output), but there is no equivalent in BSD `date` (read: on macOS, we would not be able to obtain precise timings). So let's introduce `test-tool date getnanos`, with an optional start time, that outputs preciser values. Granted, it is a bit pointless to try measuring times accurately in shell scripts, certainly to nanosecond precision. But it is better than second-granularity. Signed-off-by: Johannes Schindelin --- t/helper/test-date.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/t/helper/test-date.c b/t/helper/test-date.c index a0837371ab..792a805374 100644 --- a/t/helper/test-date.c +++ b/t/helper/test-date.c @@ -7,6 +7,7 @@ static const char *usage_msg = "\n" " test-tool date parse [date]...\n" " test-tool date approxidate [date]...\n" " test-tool date timestamp [date]...\n" +" test-tool date getnanos [start-nanos]\n" " test-tool date is64bit\n" " test-tool date time_t-is64bit\n"; @@ -82,6 +83,15 @@ static void parse_approx_timestamp(const char **argv, struct timeval *now) } } +static void getnanos(const char **argv, struct timeval *now) +{ + double seconds = getnanotime() / 1.0e9; + + if (*argv) + seconds -= strtod(*argv, NULL); + printf("%lf\n", seconds); +} + int cmd__date(int argc, const char **argv) { struct timeval now; @@ -108,6 +118,8 @@ int cmd__date(int argc, const char **argv) parse_approxidate(argv+1, &now); else if (!strcmp(*argv, "timestamp")) parse_approx_timestamp(argv+1, &now); + else if (!strcmp(*argv, "getnanos")) + getnanos(argv+1, &now); else if (!strcmp(*argv, "is64bit")) return sizeof(timestamp_t) == 8 ? 0 : 1; else if (!strcmp(*argv, "time_t-is64bit")) From c7d97e443824e68dd81ac58a295fe4b0fd9fc7f9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 29 Aug 2018 23:19:27 +0200 Subject: [PATCH 039/102] tests: optionally write results as JUnit-style .xml This will come in handy when publishing the results of Git's test suite during an automated Azure DevOps run. Signed-off-by: Johannes Schindelin --- t/.gitignore | 1 + t/test-lib.sh | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/t/.gitignore b/t/.gitignore index 348715f0e4..91cf5772fe 100644 --- a/t/.gitignore +++ b/t/.gitignore @@ -2,3 +2,4 @@ /test-results /.prove /chainlinttmp +/out/ diff --git a/t/test-lib.sh b/t/test-lib.sh index 251f4d6a96..394a08bfb1 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -341,6 +341,9 @@ do -V|--verbose-log) verbose_log=t shift ;; + --write-junit-xml) + write_junit_xml=t + shift ;; *) echo "error: unknown test option '$1'" >&2; exit 1 ;; esac @@ -488,11 +491,24 @@ trap 'exit $?' INT # the test_expect_* functions instead. test_ok_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$*" + fi test_success=$(($test_success + 1)) say_color "" "ok $test_count - $@" } test_failure_ () { + if test -n "$write_junit_xml" + then + junit_insert="" + junit_insert="$junit_insert $(xml_attr_encode \ + "$(printf '%s\n' "$@" | sed 1d)")" + junit_insert="$junit_insert" + write_junit_xml_testcase "$1" " $junit_insert" + fi test_failure=$(($test_failure + 1)) say_color error "not ok $test_count - $1" shift @@ -501,11 +517,19 @@ test_failure_ () { } test_known_broken_ok_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$* (breakage fixed)" + fi test_fixed=$(($test_fixed+1)) say_color error "ok $test_count - $@ # TODO known breakage vanished" } test_known_broken_failure_ () { + if test -n "$write_junit_xml" + then + write_junit_xml_testcase "$* (known breakage)" + fi test_broken=$(($test_broken+1)) say_color warn "not ok $test_count - $@ # TODO known breakage" } @@ -763,6 +787,10 @@ test_start_ () { test_count=$(($test_count+1)) maybe_setup_verbose maybe_setup_valgrind + if test -n "$write_junit_xml" + then + junit_start=$(test-tool date getnanos) + fi } test_finish_ () { @@ -800,6 +828,13 @@ test_skip () { case "$to_skip" in t) + if test -n "$write_junit_xml" + then + message="$(xml_attr_encode "$skipped_reason")" + write_junit_xml_testcase "$1" \ + " " + fi + say_color skip >&3 "skipping test: $@" say_color skip "ok $test_count # skip $1 ($skipped_reason)" : true @@ -815,9 +850,62 @@ test_at_end_hook_ () { : } +write_junit_xml () { + case "$1" in + --truncate) + >"$junit_xml_path" + junit_have_testcase= + shift + ;; + esac + printf '%s\n' "$@" >>"$junit_xml_path" +} + +xml_attr_encode () { + # We do not translate CR to because BSD sed does not handle + # \r in the regex. In practice, the output should not even have any + # carriage returns. + printf '%s\n' "$@" | + sed -e 's/&/\&/g' -e "s/'/\'/g" -e 's/"/\"/g' \ + -e 's//\>/g' \ + -e "s/$(printf \\x1c)/\\�/g" \ + -e "s/$(printf \\x1d)/\\�/g" \ + -e "s/$(printf \\x1e)/\\�/g" \ + -e "s/$(printf \\x1f)/\\�/g" \ + -e 's/ /\ /g' -e 's/$/\ /' -e '$s/ $//' | + tr -d '\012\015' +} + +write_junit_xml_testcase () { + junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\"" + shift + junit_attrs="$junit_attrs classname=\"$this_test\"" + junit_attrs="$junit_attrs time=\"$(test-tool \ + date getnanos $junit_start)\"" + write_junit_xml "$(printf '%s\n' \ + " " "$@" " ")" + junit_have_testcase=t +} + test_done () { GIT_EXIT_OK=t + if test -n "$write_junit_xml" && test -n "$junit_xml_path" + then + test -n "$junit_have_testcase" || { + junit_start=$(test-tool date getnanos) + write_junit_xml_testcase "all tests skipped" + } + + # adjust the overall time + junit_time=$(test-tool date getnanos $junit_suite_start) + sed "s/]*/& time=\"$junit_time\"/" \ + <"$junit_xml_path" >"$junit_xml_path.new" + mv "$junit_xml_path.new" "$junit_xml_path" + + write_junit_xml " " "" + fi + if test -z "$HARNESS_ACTIVE" then test_results_dir="$TEST_OUTPUT_DIRECTORY/test-results" @@ -1062,6 +1150,7 @@ then else mkdir -p "$TRASH_DIRECTORY" fi + # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 @@ -1075,6 +1164,19 @@ then test_done fi +if test -n "$write_junit_xml" +then + junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out" + mkdir -p "$junit_xml_dir" + junit_xml_base=${0##*/} + junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml" + junit_attrs="name=\"${junit_xml_base%.sh}\"" + junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \ + date +%Y-%m-%dT%H:%M:%S)\"" + write_junit_xml --truncate "" " " + junit_suite_start=$(test-tool date getnanos) +fi + # Provide an implementation of the 'yes' utility yes () { if test $# = 0 From 56b34b29a53754fa1907bb41b9e5b0addf6d7ed2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 30 Aug 2018 21:59:46 +0200 Subject: [PATCH 040/102] ci/lib.sh: add support for Azure Pipelines This patch introduces a conditional arm that defines some environment variables and a function that displays the URL given the job id (to identify previous runs for known-good trees). Signed-off-by: Johannes Schindelin --- ci/lib.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ci/lib.sh b/ci/lib.sh index a0182ddc6b..d592dff22a 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -19,6 +19,29 @@ then BREW_INSTALL_PACKAGES="git-lfs gettext" export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" export GIT_TEST_OPTS="--verbose-log -x --immediate" +elif test -n "$SYSTEM_TASKDEFINITIONSURI" +then + # We are running in Azure Pipelines + CI_BRANCH="$BUILD_SOURCEBRANCH" + CI_COMMIT="$BUILD_SOURCEVERSION" + CI_JOB_ID="$BUILD_BUILDID" + CI_JOB_NUMBER="$BUILD_BUILDNUMBER" + CI_OS_NAME="$(echo "$AGENT_OS" | tr A-Z a-z)" + test darwin != "$CI_OS_NAME" || CI_OS_NAME=osx + CI_REPO_SLUG="$(expr "$BUILD_REPOSITORY_URI" : '.*/\([^/]*/[^/]*\)$')" + CC="${CC:-gcc}" + + # use a subdirectory of the cache dir (because the file share is shared + # among *all* phases) + cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" + + url_for_job_id () { + echo "$SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$1" + } + + BREW_INSTALL_PACKAGES= + export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" + export GIT_TEST_OPTS="--quiet --write-junit-xml" fi skip_branch_tip_with_tag () { From 79e1525068bbf29e455b2d01c5ccfec79aa63c7f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 30 Aug 2018 18:15:42 +0200 Subject: [PATCH 041/102] Add a build definition for Azure DevOps This commit adds an azure-pipelines.yml file which is Azure DevOps' equivalent to Travis CI's .travis.yml. To make things a bit easier to understand, we refrain from using the `matrix` feature here because (while it is powerful) it can be a bit confusing to users who are not familiar with CI setups. Therefore, we use a separate phase even for similar configurations (such as GCC vs Clang on Linux, GCC vs Clang on macOS). Also, we make use of the shiny new feature we just introduced where the test suite can output JUnit-style .xml files. This information is made available in a nice UI that allows the viewer to filter by phase and/or test number, and to see trends such as: number of (failing) tests, time spent running the test suite, etc. Signed-off-by: Johannes Schindelin --- azure-pipelines.yml | 342 ++++++++++++++++++++++++++++++++++++++++++ ci/lib.sh | 2 +- ci/mount-fileshare.sh | 26 ++++ 3 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 azure-pipelines.yml create mode 100755 ci/mount-fileshare.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000..757c43c5f4 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,342 @@ +resources: +- repo: self + fetchDepth: 1 + +phases: +- phase: linux_clang + displayName: linux-clang + condition: succeeded() + queue: + name: Hosted Ubuntu 1604 + steps: + - bash: | + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + + sudo apt-get update && + sudo rm /var/lib/apt/lists/lock && + sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev apache2-bin && + + export CC=clang || exit 1 + + ci/install-dependencies.sh + ci/run-build-and-tests.sh || { + ci/print-test-failures.sh + exit 1 + } + + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 + displayName: 'ci/run-build-and-tests.sh' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: PublishTestResults@2 + displayName: 'Publish Test Results **/TEST-*.xml' + inputs: + mergeTestResults: true + testRunTitle: 'linux-clang' + platform: Linux + publishRunAttachments: false + condition: succeededOrFailed() + +- phase: linux_gcc + displayName: linux-gcc + condition: succeeded() + queue: + name: Hosted Ubuntu 1604 + steps: + - bash: | + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + + sudo apt-get update && + sudo rm /var/lib/apt/lists/lock && + sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev apache2-bin || exit 1 + + ci/install-dependencies.sh + ci/run-build-and-tests.sh || { + ci/print-test-failures.sh + exit 1 + } + + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 + displayName: 'ci/run-build-and-tests.sh' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: PublishTestResults@2 + displayName: 'Publish Test Results **/TEST-*.xml' + inputs: + mergeTestResults: true + testRunTitle: 'linux-gcc' + platform: Linux + publishRunAttachments: false + condition: succeededOrFailed() + +- phase: osx_clang + displayName: osx-clang + condition: succeeded() + queue: + name: Hosted macOS + steps: + - bash: | + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + + export CC=clang + + ci/install-dependencies.sh + ci/run-build-and-tests.sh || { + ci/print-test-failures.sh + exit 1 + } + + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || umount "$HOME/test-cache" || exit 1 + displayName: 'ci/run-build-and-tests.sh' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: PublishTestResults@2 + displayName: 'Publish Test Results **/TEST-*.xml' + inputs: + mergeTestResults: true + testRunTitle: 'osx-clang' + platform: macOS + publishRunAttachments: false + condition: succeededOrFailed() + +- phase: osx_gcc + displayName: osx-gcc + condition: succeeded() + queue: + name: Hosted macOS + steps: + - bash: | + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + + ci/install-dependencies.sh + ci/run-build-and-tests.sh || { + ci/print-test-failures.sh + exit 1 + } + + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || umount "$HOME/test-cache" || exit 1 + displayName: 'ci/run-build-and-tests.sh' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: PublishTestResults@2 + displayName: 'Publish Test Results **/TEST-*.xml' + inputs: + mergeTestResults: true + testRunTitle: 'osx-gcc' + platform: macOS + publishRunAttachments: false + condition: succeededOrFailed() + +- phase: gettext_poison + displayName: GETTEXT_POISON + condition: succeeded() + queue: + name: Hosted Ubuntu 1604 + steps: + - bash: | + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + + sudo apt-get update && + sudo rm /var/lib/apt/lists/lock && + sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev && + + export jobname=GETTEXT_POISON || exit 1 + + ci/run-build-and-tests.sh || { + ci/print-test-failures.sh + exit 1 + } + + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 + displayName: 'ci/run-build-and-tests.sh' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: PublishTestResults@2 + displayName: 'Publish Test Results **/TEST-*.xml' + inputs: + mergeTestResults: true + testRunTitle: 'gettext-poison' + platform: Linux + publishRunAttachments: false + condition: succeededOrFailed() + +- phase: windows + displayName: Windows + queue: + name: Hosted + timeoutInMinutes: 240 + steps: + - powershell: | + if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { + net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no + cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ + } + displayName: 'Mount test-cache' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - powershell: | + # Helper to check the error level of the latest command (exit with error when appropriate) + function c() { if (!$?) { exit(1) } } + + # Add build agent's MinGit to PATH + $env:PATH = $env:AGENT_HOMEDIRECTORY +"\externals\\git\cmd;" +$env:PATH + + # Helper to initialize (or update) a Git worktree + function init ($path, $url, $set_origin) { + if (Test-Path $path) { + cd $path; c + if (Test-Path .git) { + git init; c + } else { + git status + } + } else { + git init $path; c + cd $path; c + } + git config core.autocrlf false; c + git config core.untrackedCache true; c + if (($set_origin -ne 0) -and !(git config remote.origin.url)) { + git remote add origin $url; c + } + git fetch --depth=1 $url master; c + git reset --hard FETCH_HEAD; c + git clean -df; c + } + + # Initialize Git for Windows' SDK + $sdk_path = "$(Build.SourcesDirectory)\git-sdk-64" + init "$sdk_path" "https://dev.azure.com/git-for-windows/git-sdk-64/_git/git-sdk-64" 0 + init usr\src\build-extra https://github.com/git-for-windows/build-extra 1 + + # Let Git ignore the SDK and the test-cache + "/git-sdk-64/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" + + # Help MSYS2 runtime startup by initializing /etc/passwd + & "$sdk_path\git-cmd" --command=usr\\bin\\bash.exe -lc "mkpasswd -c >>/etc/passwd"; c + displayName: 'Initialize the Git for Windows SDK' + - powershell: | + # Helper to check the error level of the latest command (exit with error when appropriate) + function c() { if (!$?) { exit(1) } } + + cd "$(Build.SourcesDirectory)"; c + + git-sdk-64\git-cmd --command=usr\\bin\\bash.exe -lc @" + export MAKEFLAGS=-j10 + export DEVELOPER=1 + export NO_PERL=1 + export NO_SVN_TESTS=1 + export GIT_TEST_SKIP_REBASE_P=1 + + ci/run-build-and-tests.sh || { + ci/print-test-failures.sh + exit 1 + } + "@ + c + displayName: 'Build & Test' + env: + HOME: $(Build.SourcesDirectory) + MSYSTEM: MINGW64 + - powershell: | + if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { + cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" + } + displayName: 'Unmount test-cache' + condition: true + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: PublishTestResults@2 + displayName: 'Publish Test Results **/TEST-*.xml' + inputs: + mergeTestResults: true + testRunTitle: 'windows' + platform: Windows + publishRunAttachments: false + condition: succeededOrFailed() + +- phase: linux32 + displayName: Linux32 + condition: succeeded() + queue: + name: Hosted Ubuntu 1604 + steps: + - bash: | + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + + sudo apt-get update && + sudo rm /var/lib/apt/lists/lock && + sudo apt-get -y install \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common && + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && + sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" && + sudo apt-get update && + sudo apt-get -y install docker-ce && + + sudo AGENT_OS="$AGENT_OS" BUILD_BUILDNUMBER="$BUILD_BUILDNUMBER" BUILD_REPOSITORY_URI="$BUILD_REPOSITORY_URI" BUILD_SOURCEBRANCH="$BUILD_SOURCEBRANCH" BUILD_SOURCEVERSION="$BUILD_SOURCEVERSION" SYSTEM_PHASENAME="$SYSTEM_PHASENAME" SYSTEM_TASKDEFINITIONSURI="$SYSTEM_TASKDEFINITIONSURI" SYSTEM_TEAMPROJECT="$SYSTEM_TEAMPROJECT" CC=$CC MAKEFLAGS=-j3 bash -lxc ci/run-linux32-docker.sh || exit 1 + + sudo chmod a+r t/out/TEST-*.xml + + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 + displayName: 'ci/run-linux32-docker.sh' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + - task: PublishTestResults@2 + displayName: 'Publish Test Results **/TEST-*.xml' + inputs: + mergeTestResults: true + testRunTitle: 'linux32' + platform: Linux + publishRunAttachments: false + condition: succeededOrFailed() + +- phase: static_analysis + displayName: StaticAnalysis + condition: succeeded() + queue: + name: Hosted Ubuntu 1604 + steps: + - bash: | + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + + sudo apt-get update && + sudo rm /var/lib/apt/lists/lock && + sudo apt-get install -y coccinelle && + + export jobname=StaticAnalysis && + + ci/run-static-analysis.sh || exit 1 + + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 + displayName: 'ci/run-static-analysis.sh' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) + +- phase: documentation + displayName: Documentation + condition: succeeded() + queue: + name: Hosted Ubuntu 1604 + steps: + - bash: | + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 + + sudo apt-get update && + sudo rm /var/lib/apt/lists/lock && + sudo apt-get install -y asciidoc xmlto asciidoctor && + + export ALREADY_HAVE_ASCIIDOCTOR=yes. && + export jobname=Documentation && + + ci/test-documentation.sh || exit 1 + + test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 + displayName: 'ci/test-documentation.sh' + env: + GITFILESHAREPWD: $(gitfileshare.pwd) diff --git a/ci/lib.sh b/ci/lib.sh index d592dff22a..31f4b559fd 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -41,7 +41,7 @@ then BREW_INSTALL_PACKAGES= export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" - export GIT_TEST_OPTS="--quiet --write-junit-xml" + export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" fi skip_branch_tip_with_tag () { diff --git a/ci/mount-fileshare.sh b/ci/mount-fileshare.sh new file mode 100755 index 0000000000..5fb5f74b70 --- /dev/null +++ b/ci/mount-fileshare.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +die () { + echo "$*" >&2 + exit 1 +} + +test $# = 4 || +die "Usage: $0 Date: Wed, 12 Dec 2018 12:42:01 +0100 Subject: [PATCH 042/102] ci: speed up Windows phase As Unix shell scripting comes at a hefty price on Windows, we have to see where we can save some time to run the test suite. Let's skip the chain linting and the bin-wrappers/ redirection on Windows; this seems to shave of anywhere between 10-30% from the overall runtime. Signed-off-by: Johannes Schindelin --- ci/lib.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/lib.sh b/ci/lib.sh index 31f4b559fd..1207caafaa 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -42,6 +42,8 @@ then BREW_INSTALL_PACKAGES= export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" + test windows_nt != "$CI_OS_NAME" || + GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" fi skip_branch_tip_with_tag () { From 095e9ec07ea66ff0deedc804c4234555ecbad74c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Aug 2018 15:40:14 +0200 Subject: [PATCH 046/102] tests: include detailed trace logs with --write-junit-xml upon failure The JUnit XML format lends itself to be presented in a powerful UI, where you can drill down to the information you are interested in very quickly. For test failures, this usually means that you want to see the detailed trace of the failing tests. With Travis CI, we passed the `--verbose-log` option to get those traces. However, that seems excessive, as we do not need/use the logs in almost all of those cases: only when a test fails do we have a way to include the trace. So let's do something different when using Azure DevOps: let's run all the tests with `--quiet` first, and only if a failure is encountered, try to trace the commands as they are executed. Of course, we cannot turn on `--verbose-log` after the fact. So let's just re-run the test with all the same options, adding `--verbose-log`. And then munging the output file into the JUnit XML on the fly. Note: there is an off chance that re-running the test in verbose mode "fixes" the failures (and this does happen from time to time!). That is a possibility we should be able to live with. Ideally, we would label this as "Passed upon rerun", and Azure Pipelines even know about that outcome, but it is not available when using the JUnit XML format for now: https://github.com/Microsoft/azure-pipelines-agent/blob/master/src/Agent.Worker/TestResults/JunitResultReader.cs Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 394a08bfb1..5101720dbd 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -95,6 +95,13 @@ done,*) test "$(cat "$BASE.exit")" = 0 exit ;; +*' --write-junit-xml '*) + # record how to call this script *with* --verbose-log, in case + # we encounter a breakage + junit_rerun_options_sq="$(printf '%s\n' "$0" --verbose-log -x "$@" | + sed -e "s/'/'\\\\''/g" -e "s/^/'/" -e "s/\$/'/" | + tr '\012' ' ')" + ;; esac # For repeatability, reset the environment to known value. @@ -505,8 +512,18 @@ test_failure_ () { junit_insert="" junit_insert="$junit_insert $(xml_attr_encode \ - "$(printf '%s\n' "$@" | sed 1d)")" + "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + cut -c "$GIT_TEST_TEE_OFFSET-" <"$GIT_TEST_TEE_OUTPUT_FILE" + else + printf '%s\n' "$@" | sed 1d + fi)")" junit_insert="$junit_insert" + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + junit_insert="$junit_insert$(xml_attr_encode \ + "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")" + fi write_junit_xml_testcase "$1" " $junit_insert" fi test_failure=$(($test_failure + 1)) @@ -885,6 +902,10 @@ write_junit_xml_testcase () { write_junit_xml "$(printf '%s\n' \ " " "$@" " ")" junit_have_testcase=t + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + GIT_TEST_TEE_OFFSET=$(perl -e 'print -s $ARGV[0]' "$GIT_TEST_TEE_OUTPUT_FILE") + fi } test_done () { @@ -1175,6 +1196,11 @@ then date +%Y-%m-%dT%H:%M:%S)\"" write_junit_xml --truncate "" " " junit_suite_start=$(test-tool date getnanos) + if test -n "$GIT_TEST_TEE_OUTPUT_FILE" + then + GIT_TEST_TEE_OFFSET=0 + GIT_TEST_TEE_ERR_OFFSET=0 + fi fi # Provide an implementation of the 'yes' utility From 2f334815c662cb1891e15384a7c8c65030a0202e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Aug 2018 23:53:22 +0200 Subject: [PATCH 047/102] README: add a build badge (status of the Azure Pipelines build) Just like so many other OSS projects, we now also have a build badge. Signed-off-by: Johannes Schindelin --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f920a42fad..bf4780c22d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https:/dev.azure.com/git/git/_apis/build/status/test-git.git)](https://dev.azure.com/git/git/_build/latest?definitionId=2) + Git - fast, scalable, distributed revision control system ========================================================= From 0c0882077371c2b9a8fda1321ae3d459c2ec6283 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 4 Sep 2018 13:29:26 +0200 Subject: [PATCH 048/102] travis: fix skipping tagged releases When building a PR, TRAVIS_BRANCH refers to the *target branch*. Therefore, if a PR targets `master`, and `master` happened to be tagged, we skipped the build by mistake. Fix this by using TRAVIS_PULL_REQUEST_BRANCH (i.e. the *source branch*) when available, falling back to TRAVIS_BRANCH (i.e. for CI builds, also known as "push builds"). Signed-off-by: Johannes Schindelin --- ci/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/lib.sh b/ci/lib.sh index 1207caafaa..21d430f157 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -3,7 +3,7 @@ if test true = "$TRAVIS" then # We are running within Travis CI - CI_BRANCH="$TRAVIS_BRANCH" + CI_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" CI_COMMIT="$TRAVIS_COMMIT" CI_JOB_ID="$TRAVIS_JOB_ID" CI_JOB_NUMBER="$TRAVIS_JOB_NUMBER" From 6005f91aab08a7e77cd8f6c964f98e75c665e518 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 3 Nov 2016 09:40:12 -0700 Subject: [PATCH 049/102] tests: explicitly use `test-tool.exe` on Windows In 8abfdf44c882 (tests: explicitly use `git.exe` on Windows, 2018-11-14), we made sure to use the `.exe` file extension when using an absolute path to `git.exe`, to avoid getting confused with a file or directory in the same place that lacks said file extension. For the same reason, we need to handle test-tool.exe the same way. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 0f1faa24b2..caf648a87d 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1021,7 +1021,7 @@ test -d "$GIT_BUILD_DIR"/templates/blt || { error "You haven't built things yet, have you?" } -if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool +if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool$X then echo >&2 'You need to build test-tool:' echo >&2 'Run "make t/helper/test-tool" in the source (toplevel) directory' From 084a5b7dc8108a5d6b5a251891de35f8ff35e6d8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 11 Dec 2015 06:59:13 +0100 Subject: [PATCH 050/102] mingw: handle absolute paths in expand_user_path() On Windows, an absolute POSIX path needs to be turned into a Windows one. Signed-off-by: Johannes Schindelin --- path.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/path.c b/path.c index dc3294c71e..9b15a8483c 100644 --- a/path.c +++ b/path.c @@ -11,6 +11,7 @@ #include "path.h" #include "packfile.h" #include "object-store.h" +#include "exec-cmd.h" static int get_st_mode_bits(const char *path, int *mode) { @@ -711,6 +712,10 @@ char *expand_user_path(const char *path, int real_home) if (path == NULL) goto return_null; +#ifdef __MINGW32__ + if (path[0] == '/') + return system_path(path + 1); +#endif if (path[0] == '~') { const char *first_slash = strchrnul(path, '/'); const char *username = path + 1; From 3faec067ca30c8f6ac7c4bf8a518eff932259e09 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 Nov 2017 18:02:51 +0100 Subject: [PATCH 051/102] Mark .bat files as requiring CR/LF endings Signed-off-by: Johannes Schindelin --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 9fa72ad450..b08a1416d8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,6 +5,7 @@ *.pl eof=lf diff=perl *.pm eol=lf diff=perl *.py eol=lf diff=python +*.bat eol=crlf /Documentation/**/*.txt eol=lf /command-list.txt eol=lf /GIT-VERSION-GEN eol=lf From 5483b5b0c5f988d4eda3b948adbf800496f72b85 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 25 Oct 2018 11:11:48 +0200 Subject: [PATCH 052/102] t0001 (mingw): do not expect specific order of stdout/stderr When redirecting stdout/stderr to the same file, we cannot guarantee that stdout will come first. In fact, in this test case, it seems that an MSVC build always prints stderr first. In any case, this test case does not want to verify the *order* but the *presence* of both outputs, so let's relax the test a little. Signed-off-by: Johannes Schindelin --- t/t0001-init.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 35ede1b0b0..0ba28c72a0 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -474,7 +474,8 @@ test_expect_success MINGW 'redirect std handles' ' GIT_REDIRECT_STDERR="2>&1" \ git rev-parse --git-dir --verify refs/invalid && printf ".git\nfatal: Needed a single revision\n" >expect && - test_cmp expect output.txt + sort output.sorted && + test_cmp expect output.sorted ' test_done From 8c3ac5adeb0a47ce42ed17975e2b3987c889e700 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 16:01:35 -0400 Subject: [PATCH 053/102] cache-tree.c: avoid reusing the DEBUG constant In MSVC, the DEBUG constant is set automatically whenever compiling with debug information. This is clearly not what was intended in cache-tree.c, so let's use a less ambiguous constant there. Signed-off-by: Jeff Hostetler --- cache-tree.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cache-tree.c b/cache-tree.c index 9d454d24bc..7361a42ded 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -6,8 +6,8 @@ #include "object-store.h" #include "replace-object.h" -#ifndef DEBUG -#define DEBUG 0 +#ifndef DEBUG_CACHE_TREE +#define DEBUG_CACHE_TREE 0 #endif struct cache_tree *cache_tree(void) @@ -111,7 +111,7 @@ static int do_invalidate_path(struct cache_tree *it, const char *path) int namelen; struct cache_tree_sub *down; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree invalidate <%s>\n", path); #endif @@ -398,7 +398,7 @@ static int update_one(struct cache_tree *it, strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); strbuf_add(&buffer, oid->hash, the_hash_algo->rawsz); -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one %o %.*s\n", mode, entlen, path + baselen); #endif @@ -421,7 +421,7 @@ static int update_one(struct cache_tree *it, strbuf_release(&buffer); it->entry_count = to_invalidate ? -1 : i - *skip_count; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one (%d ent, %d subtree) %s\n", it->entry_count, it->subtree_nr, oid_to_hex(&it->oid)); @@ -462,7 +462,7 @@ static void write_one(struct strbuf *buffer, struct cache_tree *it, strbuf_add(buffer, path, pathlen); strbuf_addf(buffer, "%c%d %d\n", 0, it->entry_count, it->subtree_nr); -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%.*s> (%d ent, %d subtree) %s\n", pathlen, path, it->entry_count, it->subtree_nr, @@ -536,7 +536,7 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p) size -= rawsz; } -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%s> (%d ent, %d subtree) %s\n", *buffer, it->entry_count, subtree_nr, From 70998cee8c33eb05ba36f791f067353d8f1aed5d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Oct 2016 06:25:56 -0700 Subject: [PATCH 054/102] obstack: fix compiler warning MS Visual C suggests that the construct condition ? (int) i : (ptrdiff_t) d is incorrect. Let's fix this by casting to ptrdiff_t also for the positive arm of the conditional. Signed-off-by: Johannes Schindelin --- compat/obstack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/obstack.h b/compat/obstack.h index 6bc24b7644..f0807eaa3b 100644 --- a/compat/obstack.h +++ b/compat/obstack.h @@ -492,7 +492,7 @@ __extension__ \ ( (h)->temp.tempint = (char *) (obj) - (char *) (h)->chunk, \ ((((h)->temp.tempint > 0 \ && (h)->temp.tempint < (h)->chunk_limit - (char *) (h)->chunk)) \ - ? (int) ((h)->next_free = (h)->object_base \ + ? (ptrdiff_t) ((h)->next_free = (h)->object_base \ = (h)->temp.tempint + (char *) (h)->chunk) \ : (((obstack_free) ((h), (h)->temp.tempint + (char *) (h)->chunk), 0), 0))) From 1d4c7cf5d9c5786051d5cadc99dd703c5eea30f2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 30 Oct 2018 21:39:05 +0100 Subject: [PATCH 055/102] mingw: replace mingw_startup() hack Git for Windows has special code to retrieve the command-line parameters (and even the environment) in UTF-16 encoding, so that they can be converted to UTF-8. This is necessary because Git for Windows wants to use UTF-8 encoded strings throughout its code, and the main() function does not get the parameters in that encoding. To do that, we used the __wgetmainargs() function, which is not even a Win32 API function, but provided by the MINGW "runtime" instead. Obviously, this method would not work with any other compiler than GCC, and in preparation for compiling with Visual C++, we would like to avoid that. Lucky us, there is a much more elegant way: we simply implement wmain() and link with -municode. The command-line parameters are passed to wmain() encoded in UTF-16, as desired, and this method also works with Visual C++ after adjusting the MSVC linker flags to force it to use wmain(). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 53 +++++++++++++++++++++++++++++++----------------- compat/mingw.h | 22 ++++++++++---------- config.mak.uname | 3 ++- 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 9efef23180..02ff65bd35 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2492,18 +2492,13 @@ static void setup_windows_environment(void) setenv("TERM", "cygwin", 1); } +#if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). */ int _CRT_glob = 0; - -typedef struct { - int newmode; -} _startupinfo; - -extern int __wgetmainargs(int *argc, wchar_t ***argv, wchar_t ***env, int glob, - _startupinfo *si); +#endif static NORETURN void die_startup(void) { @@ -2581,20 +2576,23 @@ static void maybe_redirect_std_handles(void) GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); } -void mingw_startup(void) +/* + * We implement wmain() and compile with -municode, which would + * normally ignore main(), but we call the latter from the former + * so that we can handle non-ASCII command-line parameters + * appropriately. + * + * To be more compatible with the core git code, we convert + * argv into UTF8 and pass them directly to main(). + */ +int wmain(int argc, const wchar_t **wargv) { - int i, maxlen, argc; - char *buffer; - wchar_t **wenv, **wargv; - _startupinfo si; + int i, maxlen, exit_status; + char *buffer, **save; + const char **argv; maybe_redirect_std_handles(); - /* get wide char arguments and environment */ - si.newmode = 0; - if (__wgetmainargs(&argc, &wargv, &wenv, _CRT_glob, &si) < 0) - die_startup(); - /* determine size of argv and environ conversion buffer */ maxlen = wcslen(wargv[0]); for (i = 1; i < argc; i++) @@ -2604,9 +2602,16 @@ void mingw_startup(void) maxlen = 3 * maxlen + 1; buffer = malloc_startup(maxlen); - /* convert command line arguments and environment to UTF-8 */ + /* + * Create a UTF-8 version of w_argv. Also create a "save" copy + * to remember all the string pointers because parse_options() + * will remove claimed items from the argv that we pass down. + */ + ALLOC_ARRAY(argv, argc + 1); + ALLOC_ARRAY(save, argc + 1); for (i = 0; i < argc; i++) - __argv[i] = wcstoutfdup_startup(buffer, wargv[i], maxlen); + argv[i] = save[i] = wcstoutfdup_startup(buffer, wargv[i], maxlen); + argv[i] = save[i] = NULL; free(buffer); /* fix Windows specific environment settings */ @@ -2625,6 +2630,16 @@ void mingw_startup(void) /* initialize Unicode console */ winansi_init(); + + /* invoke the real main() using our utf8 version of argv. */ + exit_status = main(argc, argv); + + for (i = 0; i < argc; i++) + free(save[i]); + free(save); + free(argv); + + return exit_status; } int uname(struct utsname *buf) diff --git a/compat/mingw.h b/compat/mingw.h index 8c24ddaa3e..c30ae2d651 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -591,18 +591,18 @@ int xwcstoutf(char *utf, const wchar_t *wcs, size_t utflen); extern CRITICAL_SECTION pinfo_cs; /* - * A replacement of main() that adds win32 specific initialization. + * Git, like most portable C applications, implements a main() function. On + * Windows, this main() function would receive parameters encoded in the + * current locale, but Git for Windows would prefer UTF-8 encoded parameters. + * + * To make that happen, we still declare main() here, and then declare and + * implement wmain() (which is the Unicode variant of main()) and compile with + * -municode. This wmain() function reencodes the parameters from UTF-16 to + * UTF-8 format, sets up a couple of other things as required on Windows, and + * then hands off to the main() function. */ - -void mingw_startup(void); -#define main(c,v) dummy_decl_mingw_main(void); \ -static int mingw_main(c,v); \ -int main(int argc, const char **argv) \ -{ \ - mingw_startup(); \ - return mingw_main(__argc, (void *)__argv); \ -} \ -static int mingw_main(c,v) +int wmain(int argc, const wchar_t **w_argv); +int main(int argc, const char **argv); /* * Used by Pthread API implementation for Windows diff --git a/config.mak.uname b/config.mak.uname index 6a21ee80d7..d5b793c9a4 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -392,7 +392,7 @@ ifeq ($(uname_S),Windows) compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" - BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE + BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -ENTRY:wmainCRTStartup -SUBSYSTEM:CONSOLE EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj PTHREAD_LIBS = lib = @@ -524,6 +524,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) ETAGS_TARGET = ETAGS NO_POSIX_GOODIES = UnfortunatelyYes DEFAULT_HELP_FORMAT = html + BASIC_LDFLAGS += -municode COMPAT_CFLAGS += -DNOGDI -Icompat -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ From ef72965e1dc2902437f3471e66e768bac5f9b833 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 06:31:47 -0700 Subject: [PATCH 056/102] msvc: fix dependencies of compat/msvc.c The file compat/msvc.c includes compat/mingw.c, which means that we have to recompile compat/msvc.o if compat/mingw.c changes. Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index d5b793c9a4..fd727a0bac 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -405,6 +405,8 @@ else BASIC_CFLAGS += -Zi -MDd endif X = .exe + +compat/msvc.o: compat/msvc.c compat/mingw.c GIT-CFLAGS endif ifeq ($(uname_S),Interix) NO_INITGROUPS = YesPlease From 6f09260bfffab0782b35c69ebf50e9abcab252e7 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:07:20 +0100 Subject: [PATCH 057/102] msvc: include sigset_t definition On MSVC (VS2008) sigset_t is not defined. Signed-off-by: Philip Oakley --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index 29a8ce8204..04b4750b87 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -18,6 +18,8 @@ #undef ERROR +typedef int sigset_t; + #include "compat/mingw.h" #endif From 462acd37e47fab320366002f715f12ef7b754f82 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:52:05 +0100 Subject: [PATCH 058/102] msvc: define O_ACCMODE This constant is not defined in MSVC's headers. In UCRT's fcntl.h, _O_RDONLY, _O_WRONLY and _O_RDWR are defined as 0, 1 and 2, respectively. Yes, that means that UCRT breaks with the tradition that O_RDWR == O_RDONLY | O_WRONLY. It is a perfectly legal way to define those constants, though, therefore we need to take care of defining O_ACCMODE accordingly. This is particularly important in order to keep our "open() can set errno to EISDIR" emulation working: it tests that (flags & O_ACCMODE) is not identical to O_RDONLY before going on to test specifically whether the file for which open() reported EACCES is, in fact, a directory. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index 04b4750b87..d336d80670 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -19,6 +19,8 @@ #undef ERROR typedef int sigset_t; +/* open for reading, writing, or both (not in fcntl.h) */ +#define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) #include "compat/mingw.h" From c07761ad9877d3e761c377267226542047aa70e2 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 059/102] msvc: mark a variable as non-const VS2015 complains when using a const pointer in memcpy()/free(). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 02ff65bd35..997462bae4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1553,7 +1553,10 @@ static int try_shell_exec(const char *cmd, char *const *argv) prog = path_lookup(interpr, 1); if (prog) { int argc = 0; - const char **argv2; +#ifndef _MSC_VER + const +#endif + char **argv2; while (argv[argc]) argc++; ALLOC_ARRAY(argv2, argc + 1); argv2[0] = (char *)cmd; /* full path to the script file */ From 26498191178ce3a5569252cbe2499e06077fd4ae Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 060/102] msvc: do not re-declare the timespec struct VS2015's headers already declare that struct. Signed-off-by: Jeff Hostetler --- compat/mingw.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index c30ae2d651..90a9e84a60 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -361,11 +361,13 @@ static inline int getrlimit(int resource, struct rlimit *rlp) #ifndef __MINGW64_VERSION_MAJOR #define off_t off64_t #define lseek _lseeki64 +#ifndef _MSC_VER struct timespec { time_t tv_sec; long tv_nsec; }; #endif +#endif struct mingw_stat { _dev_t st_dev; From 3dd34f77672df69ef0af3c452c5634219638c277 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 061/102] msvc: define ftello() It is just called differently in MSVC's headers. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index d336d80670..d7525cf61d 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -18,6 +18,8 @@ #undef ERROR +#define ftello _ftelli64 + typedef int sigset_t; /* open for reading, writing, or both (not in fcntl.h) */ #define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) From f82e359e47922aa61b900cbc40a473c21f6d52eb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 10 Jan 2017 22:53:36 +0100 Subject: [PATCH 062/102] msvc: fix detect_msys_tty() The ntstatus.h header is only available in MINGW. Signed-off-by: Jeff Hostetler --- compat/winansi.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index f4f08237f9..11cd9b82cc 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -544,7 +544,20 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle) #ifdef DETECT_MSYS_TTY #include + +#if defined(_MSC_VER) + +typedef struct _OBJECT_NAME_INFORMATION +{ + UNICODE_STRING Name; + WCHAR NameBuffer[0]; +} OBJECT_NAME_INFORMATION, *POBJECT_NAME_INFORMATION; + +#define ObjectNameInformation 1 + +#else #include +#endif static void detect_msys_tty(int fd) { From e001b365ee88da1512fd414c71a068d01a9368fd Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Wed, 4 May 2016 16:18:07 +0100 Subject: [PATCH 063/102] msvc: add pragmas for common warnings MSVC can be overzealous about some warnings. Disable them. Signed-off-by: Philip Oakley --- compat/msvc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index d7525cf61d..1d7a8c6145 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -6,6 +6,10 @@ #include #include +#pragma warning(disable: 4018) /* signed/unsigned comparison */ +#pragma warning(disable: 4244) /* type conversion, possible loss of data */ +#pragma warning(disable: 4090) /* 'function' : different 'const' qualifiers (ALLOC_GROW etc.)*/ + /* porting function */ #define inline __inline #define __inline__ __inline From 3f2076907982bedd075609307b53557972709825 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 064/102] msvc: do not pretend to support all signals This special-cases various signals that are not supported on Windows, such as SIGPIPE. These cause the UCRT to throw asserts (at least in debug mode). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 997462bae4..c73a8b9177 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2283,8 +2283,34 @@ int mingw_raise(int sig) sigint_fn(SIGINT); return 0; +#if defined(_MSC_VER) + /* + * in the CRT defines 8 signals as being + * supported on the platform. Anything else causes + * an "Invalid signal or error" (which in DEBUG builds + * causes the Abort/Retry/Ignore dialog). We by-pass + * the CRT for things we already know will fail. + */ + /*case SIGINT:*/ + case SIGILL: + case SIGFPE: + case SIGSEGV: + case SIGTERM: + case SIGBREAK: + case SIGABRT: + case SIGABRT_COMPAT: + return raise(sig); + default: + errno = EINVAL; + return -1; + +#else + default: return raise(sig); + +#endif + } } From 8403f979586a9feca2e02cc726873b31deab120a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 11:32:01 -0400 Subject: [PATCH 065/102] msvc: support building Git using MS Visual C++ With this patch, Git can be built using the Microsoft toolchain, via: make MSVC=1 [DEBUG=1] Third party libraries are built from source using the open source "vcpkg" tool set. See https://github.com/Microsoft/vcpkg On a first build, the vcpkg tools and the third party libraries are automatically downloaded and built. DLLs for the third party libraries are copied to the top-level (and t/helper) directory to facilitate debugging. See compat/vcbuild/README. A series of .bat files are invoked by the Makefile to find the location of the installed version of Visual Studio and the associated compiler tools (essentially replicating the environment setup performed by a "Developer Command Prompt"). This should find the most recent VS2015 or VS2017 installation. Output from these scripts are used by the Makefile to define compiler and linker pathnames and -I and -L arguments. The build produces .pdb files for both debug and release builds. Note: This commit was squashed from an organic series of commits developed between 2016 and 2018 in Git for Windows' `master` branch. This combined commit eliminates the obsolete commits related to fetching NuGet packages for third party libraries. It is difficult to use NuGet packages for C/C++ sources because they may be built by earlier versions of the MSVC compiler and have CRT version and linking issues. Additionally, the C/C++ NuGet packages that were using tended to not be updated concurrently with the sources. And in the case of cURL and OpenSSL, this could expose us to security issues. Helped-by: Yue Lin Ho Helped-by: Philip Oakley Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- Makefile | 42 ++++++- compat/mingw.c | 12 ++ compat/vcbuild/.gitignore | 3 + compat/vcbuild/README | 51 +++++++++ compat/vcbuild/find_vs_env.bat | 169 +++++++++++++++++++++++++++++ compat/vcbuild/scripts/clink.pl | 26 ++++- compat/vcbuild/vcpkg_copy_dlls.bat | 39 +++++++ compat/vcbuild/vcpkg_install.bat | 81 ++++++++++++++ config.mak.uname | 76 +++++++++++-- git-compat-util.h | 9 ++ 10 files changed, 492 insertions(+), 16 deletions(-) create mode 100644 compat/vcbuild/.gitignore create mode 100644 compat/vcbuild/find_vs_env.bat create mode 100644 compat/vcbuild/vcpkg_copy_dlls.bat create mode 100644 compat/vcbuild/vcpkg_install.bat diff --git a/Makefile b/Makefile index 1a44c811aa..51934084a9 100644 --- a/Makefile +++ b/Makefile @@ -1203,7 +1203,7 @@ endif ifdef SANE_TOOL_PATH SANE_TOOL_PATH_SQ = $(subst ','\'',$(SANE_TOOL_PATH)) -BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix $(SANE_TOOL_PATH_SQ)|' +BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix "$(SANE_TOOL_PATH_SQ)"|' PATH := $(SANE_TOOL_PATH):${PATH} else BROKEN_PATH_FIX = '/^\# @@BROKEN_PATH_FIX@@$$/d' @@ -2788,6 +2788,33 @@ install: all $(INSTALL) $(ALL_PROGRAMS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)' +ifdef MSVC + # We DO NOT install the individual foo.o.pdb files because they + # have already been rolled up into the exe's pdb file. + # We DO NOT have pdb files for the builtin commands (like git-status.exe) + # because it is just a copy/hardlink of git.exe, rather than a unique binary. + $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-upload-pack.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-fetch.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-imap-send.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' +ifndef DEBUG + $(INSTALL) $(vcpkg_rel_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) $(vcpkg_rel_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' +else + $(INSTALL) $(vcpkg_dbg_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) $(vcpkg_dbg_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' +endif +endif $(MAKE) -C templates DESTDIR='$(DESTDIR_SQ)' install $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' $(INSTALL) -m 644 mergetools/* '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' @@ -2989,6 +3016,19 @@ endif $(RM) GIT-VERSION-FILE GIT-CFLAGS GIT-LDFLAGS GIT-BUILD-OPTIONS $(RM) GIT-USER-AGENT GIT-PREFIX $(RM) GIT-SCRIPT-DEFINES GIT-PERL-DEFINES GIT-PERL-HEADER GIT-PYTHON-VARS +ifdef MSVC + $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS)) + $(RM) $(patsubst %.exe,%.pdb,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.iobj,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.pdb,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.iobj,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.pdb,$(TEST_PROGRAMS)) + $(RM) $(patsubst %.exe,%.iobj,$(TEST_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(TEST_PROGRAMS)) + $(RM) compat/vcbuild/MSVC-DEFS-GEN +endif .PHONY: all install profile-clean cocciclean clean strip .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell diff --git a/compat/mingw.c b/compat/mingw.c index c73a8b9177..243a9ccc3e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2605,6 +2605,12 @@ static void maybe_redirect_std_handles(void) GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); } +#ifdef _MSC_VER +#ifdef _DEBUG +#include +#endif +#endif + /* * We implement wmain() and compile with -municode, which would * normally ignore main(), but we call the latter from the former @@ -2620,6 +2626,12 @@ int wmain(int argc, const wchar_t **wargv) char *buffer, **save; const char **argv; +#ifdef _MSC_VER +#ifdef USE_MSVC_CRTDBG + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); +#endif +#endif + maybe_redirect_std_handles(); /* determine size of argv and environ conversion buffer */ diff --git a/compat/vcbuild/.gitignore b/compat/vcbuild/.gitignore new file mode 100644 index 0000000000..8f8b794ef3 --- /dev/null +++ b/compat/vcbuild/.gitignore @@ -0,0 +1,3 @@ +/vcpkg/ +/MSVC-DEFS-GEN +/VCPKG-DEFS diff --git a/compat/vcbuild/README b/compat/vcbuild/README index 60fd873fe8..81da36a93b 100644 --- a/compat/vcbuild/README +++ b/compat/vcbuild/README @@ -1,3 +1,54 @@ +The Steps to Build Git with VS2015 or VS2017 from the command line. + +1. Install the "vcpkg" open source package manager and build essential + third-party libraries. The steps for this have been captured in a + set of convenience scripts. These can be run from a stock Command + Prompt or from an SDK bash window: + + $ cd + $ ./compat/vcbuild/vcpkg_install.bat + + The vcpkg tools and all of the third-party sources will be installed + in this folder: + /compat/vcbuild/vcpkg/ + + A file will be created with a set of Makefile macros pointing to a + unified "include", "lib", and "bin" directory (release and debug) for + all of the required packages. This file will be included by the main + Makefile: + /compat/vcbuild/MSVC-DEFS-GEN + +2. OPTIONALLY copy the third-party *.dll and *.pdb files into the repo + root to make it easier to run and debug git.exe without having to + manipulate your PATH. This is especially true for debug sessions in + Visual Studio. + + Use ONE of the following forms which should match how you want to + compile git.exe. + + $ ./compat/vcbuild/vcpkg_copy_packages.bat debug + $ ./compat/vcbuild/vcpkg_copy_packages.bat release + +3. Build git using MSVC from an SDK bash window using one of the + following commands: + + $ make MSVC=1 + $ make MSVC=1 DEBUG=1 + +================================================================ + +Alternatively, run `make MSVC=1 vcxproj` and then load the generated +git.sln in Visual Studio. The initial build will install the vcpkg +system and build the dependencies automatically. This will take a while. + +Note that this will automatically add and commit the generated +.sln and .vcxproj files to the repo. You may want to drop this +commit before submitting a Pull Request.... + +Or maybe we should put the .sln/.vcxproj files in the .gitignore file +and not do this. I'm not sure. + +================================================================ The Steps of Build Git with VS2008 1. You need the build environment, which contains the Git dependencies diff --git a/compat/vcbuild/find_vs_env.bat b/compat/vcbuild/find_vs_env.bat new file mode 100644 index 0000000000..1232f200f7 --- /dev/null +++ b/compat/vcbuild/find_vs_env.bat @@ -0,0 +1,169 @@ +@ECHO OFF +REM ================================================================ +REM You can use either GCC (the default) or MSVC to build git +REM using the GIT-SDK command line tools. +REM $ make +REM $ make MSVC=1 +REM +REM GIT-SDK BASH windows inherit environment variables with all of +REM the bin/lib/include paths for GCC. It DOES NOT inherit values +REM for the corresponding MSVC tools. +REM +REM During normal (non-git) Windows development, you launch one +REM of the provided "developer command prompts" to set environment +REM variables for the MSVC tools. +REM +REM Therefore, to allow MSVC command line builds of git from BASH +REM and MAKE, we must blend these two different worlds. This script +REM attempts to do that. +REM ================================================================ +REM This BAT file starts in a plain (non-developer) command prompt, +REM searches for the "best" commmand prompt setup script, installs +REM it into the current CMD process, and exports the various MSVC +REM environment variables for use by MAKE. +REM +REM The output of this script should be written to a make "include +REM file" and referenced by the top-level Makefile. +REM +REM See "config.mak.uname" (look for compat/vcbuild/MSVC-DEFS-GEN). +REM ================================================================ +REM The provided command prompts are custom to each VS release and +REM filled with lots of internal knowledge (such as Registry settings); +REM even their names vary by release, so it is not appropriate for us +REM to look inside them. Rather, just run them in a subordinate +REM process and extract the settings we need. +REM ================================================================ +REM +REM Current (VS2017 and beyond) +REM ------------------- +REM Visual Studio 2017 introduced a new installation layout and +REM support for side-by-side installation of multiple versions of +REM VS2017. Furthermore, these can all coexist with installations +REM of previous versions of VS (which have a completely different +REM layout on disk). +REM +REM VS2017 Update 2 introduced a "vswhere.exe" command: +REM https://github.com/Microsoft/vswhere +REM https://blogs.msdn.microsoft.com/heaths/2017/02/25/vswhere-available/ +REM https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/ +REM +REM VS2015 +REM ------ +REM Visual Studio 2015 uses the traditional VcVarsAll. +REM +REM Earlier Versions +REM ---------------- +REM TODO +REM +REM ================================================================ +REM Note: Throughout this script we use "dir && " rather +REM than "if exist " because of script problems with pathnames +REM containing spaces. +REM ================================================================ + +REM Sanitize PATH to prevent git-sdk paths from confusing "wmic.exe" +REM (called internally in some of the system BAT files). +SET PATH=%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem; + +REM ================================================================ + +:current + SET vs_where=C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe + dir "%vs_where%" >nul 2>nul && GOTO have_vs_where + GOTO not_2017 + +:have_vs_where + REM Try to use VsWhere to get the location of VsDevCmd. + + REM Keep VsDevCmd from cd'ing away. + SET VSCMD_START_DIR=. + + REM Get the root of the VS product installation. + FOR /F "usebackq tokens=*" %%i IN (`"%vs_where%" -latest -requires Microsoft.VisualStudio.Workload.NativeDesktop -property installationPath`) DO @SET vs_ip=%%i + + SET vs_devcmd=%vs_ip%\Common7\Tools\VsDevCmd.bat + dir "%vs_devcmd%" >nul 2>nul && GOTO have_vs_devcmd + GOTO not_2017 + +:have_vs_devcmd + REM Use VsDevCmd to setup the environment of this process. + REM Setup CL for building 64-bit apps using 64-bit tools. + @call "%vs_devcmd%" -no_logo -arch=x64 -host_arch=x64 + + SET tgt=%VSCMD_ARG_TGT_ARCH% + + SET mn=%VCToolsInstallDir% + SET msvc_includes=-I"%mn%INCLUDE" + SET msvc_libs=-L"%mn%lib\%tgt%" + SET msvc_bin_dir=%mn%bin\Host%VSCMD_ARG_HOST_ARCH%\%tgt% + + SET sdk_dir=%WindowsSdkDir% + SET sdk_ver=%WindowsSDKVersion% + SET si=%sdk_dir%Include\%sdk_ver% + SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" + SET sl=%sdk_dir%lib\%sdk_ver% + SET sdk_libs=-L"%sl%ucrt\%tgt%" -L"%sl%um\%tgt%" + + SET vs_ver=%VisualStudioVersion% + + GOTO print_vars + +REM ================================================================ + +:not_2017 + REM See if VS2015 is installed. + + SET vs_2015_bat=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat + dir "%vs_2015_bat%" >nul 2>nul && GOTO have_vs_2015 + GOTO not_2015 + +:have_vs_2015 + REM Use VcVarsAll like the "x64 Native" command prompt. + REM Setup CL for building 64-bit apps using 64-bit tools. + @call "%vs_2015_bat%" amd64 + + REM Note that in VS2015 they use "x64" in some contexts and "amd64" in others. + SET mn=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\ + SET msvc_includes=-I"%mn%INCLUDE" + SET msvc_libs=-L"%mn%lib\amd64" + SET msvc_bin_dir=%mn%bin\amd64 + + SET sdk_dir=%WindowsSdkDir% + SET sdk_ver=%WindowsSDKVersion% + SET si=%sdk_dir%Include\%sdk_ver% + SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" -I"%si%winrt" + SET sl=%sdk_dir%lib\%sdk_ver% + SET sdk_libs=-L"%sl%ucrt\x64" -L"%sl%um\x64" + + SET vs_ver=%VisualStudioVersion% + + GOTO print_vars + +REM ================================================================ + +:not_2015 + REM TODO.... + echo TODO support older versions of VS. >&2 + EXIT /B 1 + +REM ================================================================ + +:print_vars + REM Dump the essential vars to stdout to allow the main + REM Makefile to include it. See config.mak.uname. + REM Include DOS-style and BASH-style path for bin dir. + + echo msvc_bin_dir=%msvc_bin_dir% + SET X1=%msvc_bin_dir:C:=/C% + SET X2=%X1:\=/% + echo msvc_bin_dir_msys=%X2% + + echo msvc_includes=%msvc_includes% + echo msvc_libs=%msvc_libs% + + echo sdk_includes=%sdk_includes% + echo sdk_libs=%sdk_libs% + + echo vs_ver=%vs_ver% + + EXIT /B 0 diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a87d0da512..3d6fa21c1e 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -12,32 +12,49 @@ use strict; my @args = (); my @cflags = (); +my @lflags = (); my $is_linking = 0; +my $is_debug = 0; while (@ARGV) { my $arg = shift @ARGV; - if ("$arg" =~ /^-[DIMGO]/) { + if ("$arg" eq "-DDEBUG") { + # Some vcpkg-based libraries have different names for release + # and debug versions. This hack assumes that -DDEBUG comes + # before any "-l*" flags. + $is_debug = 1; + } + if ("$arg" =~ /^-[DIMGOZ]/) { push(@cflags, $arg); } elsif ("$arg" eq "-o") { my $file_out = shift @ARGV; if ("$file_out" =~ /exe$/) { $is_linking = 1; + # Create foo.exe and foo.pdb push(@args, "-OUT:$file_out"); } else { + # Create foo.o and foo.o.pdb push(@args, "-Fo$file_out"); + push(@args, "-Fd$file_out.pdb"); } } elsif ("$arg" eq "-lz") { + if ($is_debug) { + push(@args, "zlibd.lib"); + } else{ push(@args, "zlib.lib"); + } } elsif ("$arg" eq "-liconv") { - push(@args, "iconv.lib"); + push(@args, "libiconv.lib"); } elsif ("$arg" eq "-lcrypto") { push(@args, "libeay32.lib"); } elsif ("$arg" eq "-lssl") { push(@args, "ssleay32.lib"); } elsif ("$arg" eq "-lcurl") { push(@args, "libcurl.lib"); + } elsif ("$arg" eq "-lexpat") { + push(@args, "expat.lib"); } elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") { $arg =~ s/^-L/-LIBPATH:/; - push(@args, $arg); + push(@lflags, $arg); } elsif ("$arg" =~ /^-R/) { # eat } else { @@ -45,10 +62,11 @@ while (@ARGV) { } } if ($is_linking) { + push(@args, @lflags); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); push(@args, @cflags); } -#printf("**** @args\n"); +printf(STDERR "**** @args\n\n\n") if (!defined($ENV{'QUIET_GEN'})); exit (system(@args) != 0); diff --git a/compat/vcbuild/vcpkg_copy_dlls.bat b/compat/vcbuild/vcpkg_copy_dlls.bat new file mode 100644 index 0000000000..13661c14f8 --- /dev/null +++ b/compat/vcbuild/vcpkg_copy_dlls.bat @@ -0,0 +1,39 @@ +@ECHO OFF +REM ================================================================ +REM This script is an optional step. It copies the *.dll and *.pdb +REM files (created by vcpkg_install.bat) into the top-level directory +REM of the repo so that you can type "./git.exe" and find them without +REM having to fixup your PATH. +REM +REM NOTE: Because the names of some DLL files change between DEBUG and +REM NOTE: RELEASE builds when built using "vcpkg.exe", you will need +REM NOTE: to copy up the corresponding version. +REM ================================================================ + + SETLOCAL EnableDelayedExpansion + + @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD + cd %cwd% + + SET arch=x64-windows + SET inst=%cwd%vcpkg\installed\%arch% + + IF [%1]==[release] ( + echo Copying RELEASE mode DLLs to repo root... + ) ELSE IF [%1]==[debug] ( + SET inst=%inst%\debug + echo Copying DEBUG mode DLLs to repo root... + ) ELSE ( + echo ERROR: Invalid argument. + echo Usage: %~0 release + echo Usage: %~0 debug + EXIT /B 1 + ) + + xcopy /e/s/v/y %inst%\bin\*.dll ..\..\ + xcopy /e/s/v/y %inst%\bin\*.pdb ..\..\ + + xcopy /e/s/v/y %inst%\bin\*.dll ..\..\t\helper\ + xcopy /e/s/v/y %inst%\bin\*.pdb ..\..\t\helper\ + + EXIT /B 0 diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat new file mode 100644 index 0000000000..3d086c39c3 --- /dev/null +++ b/compat/vcbuild/vcpkg_install.bat @@ -0,0 +1,81 @@ +@ECHO OFF +REM ================================================================ +REM This script installs the "vcpkg" source package manager and uses +REM it to build the third-party libraries that git requires when it +REM is built using MSVC. +REM +REM [1] Install VCPKG. +REM [a] Create /compat/vcbuild/vcpkg/ +REM [b] Download "vcpkg". +REM [c] Compile using the currently installed version of VS. +REM [d] Create /compat/vcbuild/vcpkg/vcpkg.exe +REM +REM [2] Install third-party libraries. +REM [a] Download each (which may also install CMAKE). +REM [b] Compile in RELEASE mode and install in: +REM vcpkg/installed//{bin,lib} +REM [c] Compile in DEBUG mode and install in: +REM vcpkg/installed//debug/{bin,lib} +REM [d] Install headers in: +REM vcpkg/installed//include +REM +REM [3] Create a set of MAKE definitions for the top-level +REM Makefile to allow "make MSVC=1" to find the above +REM third-party libraries. +REM [a] Write vcpkg/VCPGK-DEFS +REM +REM https://blogs.msdn.microsoft.com/vcblog/2016/09/19/vcpkg-a-tool-to-acquire-and-build-c-open-source-libraries-on-windows/ +REM https://github.com/Microsoft/vcpkg +REM https://vcpkg.readthedocs.io/en/latest/ +REM ================================================================ + + SETLOCAL EnableDelayedExpansion + + @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD + cd %cwd% + + dir vcpkg\vcpkg.exe >nul 2>nul && GOTO :install_libraries + + echo Fetching vcpkg in %cwd%vcpkg + git.exe clone https://github.com/Microsoft/vcpkg vcpkg + IF ERRORLEVEL 1 ( EXIT /B 1 ) + + cd vcpkg + echo Building vcpkg + powershell -exec bypass scripts\bootstrap.ps1 + IF ERRORLEVEL 1 ( EXIT /B 1 ) + + echo Successfully installed %cwd%vcpkg\vcpkg.exe + +:install_libraries + SET arch=x64-windows + + echo Installing third-party libraries... + FOR %%i IN (zlib expat libiconv openssl libssh2 curl) DO ( + cd %cwd%vcpkg + SET p="packages\%%i_%arch%" + IF NOT EXIST "%p%" CALL :sub__install_one %%i + IF ERRORLEVEL 1 ( EXIT /B 1 ) + ) + +:install_defines + cd %cwd% + SET inst=%cwd%vcpkg\installed\%arch% + + echo vcpkg_inc=-I"%inst%\include">VCPKG-DEFS + echo vcpkg_rel_lib=-L"%inst%\lib">>VCPKG-DEFS + echo vcpkg_rel_bin="%inst%\bin">>VCPKG-DEFS + echo vcpkg_dbg_lib=-L"%inst%\debug\lib">>VCPKG-DEFS + echo vcpkg_dbg_bin="%inst%\debug\bin">>VCPKG-DEFS + + EXIT /B 0 + + +:sub__install_one + echo Installing package %1... + + .\vcpkg.exe install %1:%arch% + IF ERRORLEVEL 1 ( EXIT /B 1 ) + + echo Finished %1 + goto :EOF diff --git a/config.mak.uname b/config.mak.uname index fd727a0bac..e0a0ae59cc 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -1,5 +1,9 @@ # Platform specific Makefile tweaks based on uname detection +# Define NO_SAFESEH if you need MSVC/Visual Studio to ignore the lack of +# Microsoft's Safe Exception Handling in libraries (such as zlib). +# Typically required for VS2013+/32-bit compilation on Vista+ versions. + uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') uname_O := $(shell sh -c 'uname -o 2>/dev/null || echo not') @@ -11,6 +15,19 @@ ifdef MSVC # avoid the MingW and Cygwin configuration sections uname_S := Windows uname_O := Windows + + # Generate and include makefile variables that point to the + # currently installed set of MSVC command line tools. +compat/vcbuild/MSVC-DEFS-GEN: compat/vcbuild/find_vs_env.bat + @"$<" | tr '\\' / >"$@" +include compat/vcbuild/MSVC-DEFS-GEN + + # See if vcpkg and the vcpkg-build versions of the third-party + # libraries that we use are installed. We include the result + # to get $(vcpkg_*) variables defined for the Makefile. +compat/vcbuild/VCPKG-DEFS: compat/vcbuild/vcpkg_install.bat + @"$<" +include compat/vcbuild/VCPKG-DEFS endif # We choose to avoid "if .. else if .. else .. endif endif" @@ -349,6 +366,19 @@ endif ifeq ($(uname_S),Windows) GIT_VERSION := $(GIT_VERSION).MSVC pathsep = ; + # Assume that this is built in Git for Windows' SDK + ifeq (MINGW32,$(MSYSTEM)) + prefix = /mingw32 + else + prefix = /mingw64 + endif + # Prepend MSVC 64-bit tool-chain to PATH. + # + # A regular Git Bash *does not* have cl.exe in its $PATH. As there is a + # link.exe next to, and required by, cl.exe, we have to prepend this + # onto the existing $PATH. + # + SANE_TOOL_PATH ?= $(msvc_bin_dir_msys) HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease @@ -361,11 +391,14 @@ ifeq ($(uname_S),Windows) NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease NO_MEMMEM = YesPlease - # NEEDS_LIBICONV = YesPlease - NO_ICONV = YesPlease + NEEDS_LIBICONV = YesPlease NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease - SNPRINTF_RETURNS_BOGUS = YesPlease + NO_INTTYPES_H = YesPlease + # VS2015 with UCRT claims that snprintf and friends are C99 compliant, + # so we don't need this: + # + # SNPRINTF_RETURNS_BOGUS = YesPlease NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo @@ -378,7 +411,6 @@ ifeq ($(uname_S),Windows) NO_REGEX = YesPlease NO_GETTEXT = YesPlease NO_PYTHON = YesPlease - BLK_SHA1 = YesPlease ETAGS_TARGET = ETAGS NO_POSIX_GOODIES = UnfortunatelyYes NATIVE_CRLF = YesPlease @@ -387,22 +419,44 @@ ifeq ($(uname_S),Windows) CC = compat/vcbuild/scripts/clink.pl AR = compat/vcbuild/scripts/lib.pl CFLAGS = - BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE + BASIC_CFLAGS = -nologo -I. -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o - COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" + COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -ENTRY:wmainCRTStartup -SUBSYSTEM:CONSOLE - EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj + # invalidcontinue.obj allows Git's source code to close the same file + # handle twice, or to access the osfhandle of an already-closed stdout + # See https://msdn.microsoft.com/en-us/library/ms235330.aspx + EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib PTHREAD_LIBS = lib = - BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 + BASIC_CFLAGS += $(vcpkg_inc) $(sdk_includes) $(msvc_includes) ifndef DEBUG - BASIC_CFLAGS += -GL -Os -MD - BASIC_LDFLAGS += -LTCG + BASIC_CFLAGS += $(vcpkg_rel_lib) +else + BASIC_CFLAGS += $(vcpkg_dbg_lib) +endif + BASIC_CFLAGS += $(sdk_libs) $(msvc_libs) + + # Optionally enable memory leak reporting. + # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG + BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 + # Always give "-Zi" to the compiler and "-debug" to linker (even in + # release mode) to force a PDB to be generated (like RelWithDebInfo). + BASIC_CFLAGS += -Zi + BASIC_LDFLAGS += -debug + +ifdef NO_SAFESEH + LDFLAGS += -SAFESEH:NO +endif + +ifndef DEBUG + BASIC_CFLAGS += -GL -Gy -O2 -Oy- -MD -DNDEBUG + BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd + BASIC_CFLAGS += -MDd -DDEBUG -D_DEBUG endif X = .exe diff --git a/git-compat-util.h b/git-compat-util.h index 09b0102cae..43e401b886 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1,6 +1,15 @@ #ifndef GIT_COMPAT_UTIL_H #define GIT_COMPAT_UTIL_H +#ifdef USE_MSVC_CRTDBG +/* + * For these to work they must appear very early in each + * file -- before most of the standard header files. + */ +#include +#include +#endif + #define _FILE_OFFSET_BITS 64 From 350cca1c2c1dcc6db8286feb6cfbbd7ffb98adcf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Nov 2016 22:26:20 +0100 Subject: [PATCH 066/102] msvc: avoid debug assertion windows in Debug Mode For regular debugging, it is pretty helpful when a debug assertion in a running application triggers a window that offers to start the debugger. However, when running the test suite, it is not so helpful, in particular when the debug assertions are then suppressed anyway because we disable the invalid parameter checking (via invalidcontinue.obj, see the comment in config.mak.uname about that object for more information). So let's simply disable that window in Debug Mode (it is already disabled in Release Mode). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 243a9ccc3e..5487c8bfca 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2627,6 +2627,10 @@ int wmain(int argc, const wchar_t **wargv) const char **argv; #ifdef _MSC_VER +#ifdef _DEBUG + _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG); +#endif + #ifdef USE_MSVC_CRTDBG _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); #endif From 10c4925e01f3a85503a211536a2c62c6371424d2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Feb 2018 23:50:03 +0100 Subject: [PATCH 067/102] Vcproj.pm: auto-generate GUIDs We ran out GUIDs. Again. But there is no need to: we can generate them semi-randomly from the target file name of the project. Note: the Vcproj generator is probably only interesting for historical reasons; nevertheless, the upcoming Vcxproj generator (to support modern Visual Studio versions) is based on the Vcproj generator and it is better to fix this here first. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcproj.pm | 66 ++++------------------- 1 file changed, 9 insertions(+), 57 deletions(-) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index cfa74adcc2..c79b706bc8 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -3,6 +3,7 @@ require Exporter; use strict; use vars qw($VERSION); +use Digest::SHA qw(sha256_hex); our $VERSION = '1.00'; our(@ISA, @EXPORT, @EXPORT_OK, @AVAILABLE); @@ -12,59 +13,12 @@ BEGIN { push @EXPORT_OK, qw(generate); } -my $guid_index = 0; -my @GUIDS = ( - "{E07B9989-2BF7-4F21-8918-BE22BA467AC3}", - "{278FFB51-0296-4A44-A81A-22B87B7C3592}", - "{7346A2C4-F0FD-444F-9EBE-1AF23B2B5650}", - "{67F421AC-EB34-4D49-820B-3196807B423F}", - "{385DCFE1-CC8C-4211-A451-80FCFC31CA51}", - "{97CC46C5-D2CC-4D26-B634-E75792B79916}", - "{C7CE21FE-6EF8-4012-A5C7-A22BCEDFBA11}", - "{51575134-3FDF-42D1-BABD-3FB12669C6C9}", - "{0AE195E4-9823-4B87-8E6F-20C5614AF2FF}", - "{4B918255-67CA-43BB-A46C-26704B666E6B}", - "{18CCFEEF-C8EE-4CC1-A265-26F95C9F4649}", - "{5D5D90FA-01B7-4973-AFE5-CA88C53AC197}", - "{1F054320-036D-49E1-B384-FB5DF0BC8AC0}", - "{7CED65EE-F2D9-4171-825B-C7D561FE5786}", - "{8D341679-0F07-4664-9A56-3BA0DE88B9BC}", - "{C189FEDC-2957-4BD7-9FA4-7622241EA145}", - "{66844203-1B9F-4C53-9274-164FFF95B847}", - "{E4FEA145-DECC-440D-AEEA-598CF381FD43}", - "{73300A8E-C8AC-41B0-B555-4F596B681BA7}", - "{873FDEB1-D01D-40BF-A1BF-8BBC58EC0F51}", - "{7922C8BE-76C5-4AC6-8BF7-885C0F93B782}", - "{E245D370-308B-4A49-BFC1-1E527827975F}", - "{F6FA957B-66FC-4ED7-B260-E59BBE4FE813}", - "{E6055070-0198-431A-BC49-8DB6CEE770AE}", - "{54159234-C3EB-43DA-906B-CE5DA5C74654}", - "{594CFC35-0B60-46F6-B8EF-9983ACC1187D}", - "{D93FCAB7-1F01-48D2-B832-F761B83231A5}", - "{DBA5E6AC-E7BE-42D3-8703-4E787141526E}", - "{6171953F-DD26-44C7-A3BE-CC45F86FC11F}", - "{9E19DDBE-F5E4-4A26-A2FE-0616E04879B8}", - "{AE81A615-99E3-4885-9CE0-D9CAA193E867}", - "{FBF4067E-1855-4F6C-8BCD-4D62E801A04D}", - "{17007948-6593-4AEB-8106-F7884B4F2C19}", - "{199D4C8D-8639-4DA6-82EF-08668C35DEE0}", - "{E085E50E-C140-4CF3-BE4B-094B14F0DDD6}", - "{00785268-A9CC-4E40-AC29-BAC0019159CE}", - "{4C06F56A-DCDB-46A6-B67C-02339935CF12}", - "{3A62D3FD-519E-4EC9-8171-D2C1BFEA022F}", - "{3A62D3FD-519E-4EC9-8171-D2C1BFEA022F}", - "{9392EB58-D7BA-410B-B1F0-B2FAA6BC89A7}", - "{2ACAB2D5-E0CE-4027-BCA0-D78B2D7A6C66}", - "{86E216C3-43CE-481A-BCB2-BE5E62850635}", - "{FB631291-7923-4B91-9A57-7B18FDBB7A42}", - "{0A176EC9-E934-45B8-B87F-16C7F4C80039}", - "{DF55CA80-46E8-4C53-B65B-4990A23DD444}", - "{3A0F9895-55D2-4710-BE5E-AD7498B5BF44}", - "{294BDC5A-F448-48B6-8110-DD0A81820F8C}", - "{4B9F66E9-FAC9-47AB-B1EF-C16756FBFD06}", - "{72EA49C6-2806-48BD-B81B-D4905102E19C}", - "{5728EB7E-8929-486C-8CD5-3238D060E768}" -); +sub generate_guid ($) { + my $hex = sha256_hex($_[0]); + $hex =~ s/^(.{8})(.{4})(.{4})(.{4})(.{12}).*/{$1-$2-$3-$4-$5}/; + $hex =~ tr/a-z/A-Z/; + return $hex; +} sub generate { my ($git_dir, $out_dir, $rel_dir, %build_structure) = @_; @@ -92,9 +46,8 @@ sub createLibProject { $target =~ s/\//_/g; $target =~ s/\.a//; - my $uuid = $GUIDS[$guid_index]; + my $uuid = generate_guid($libname); $$build_structure{"LIBS_${target}_GUID"} = $uuid; - $guid_index += 1; my @srcs = sort(map("$rel_dir\\$_", @{$$build_structure{"LIBS_${libname}_SOURCES"}})); my @sources; @@ -311,9 +264,8 @@ sub createAppProject { $target =~ s/\//_/g; $target =~ s/\.exe//; - my $uuid = $GUIDS[$guid_index]; + my $uuid = generate_guid($appname); $$build_structure{"APPS_${target}_GUID"} = $uuid; - $guid_index += 1; my @srcs = sort(map("$rel_dir\\$_", @{$$build_structure{"APPS_${appname}_SOURCES"}})); my @sources; From de2f616f76cc6a8fedf2104f8946fb50bbfdc3d7 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 17:43:29 +0100 Subject: [PATCH 068/102] Vcproj.pm: list git.exe first to be startup project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Visual Studio takes the first listed application/library as the default startup project [1]. Detect the 'git' project and place it the head of the apps list, rather than the tail. Export the apps list before libs list for both the projects and global structures of the .sln file. [1] http://stackoverflow.com/questions/1238553/ vs2008-where-is-the-startup-project-setting-stored-for-a-solution "In the solution file, there are a list of pseudo-XML "Project" entries. It turns out that whatever is the first one ends up as the Startup Project, unless it’s overridden in the suo file. Argh. I just rearranged the order in the file and it’s good." "just moving the pseudo-xml isn't enough. You also have to move the group of entries in the "GlobalSection(ProjectConfigurationPlatforms) = postSolution" group that has the GUID of the project you moved to the top. So there are two places to move lines." Signed-off-by: Philip Oakley --- contrib/buildsystems/Generators/Vcproj.pm | 33 +++++++++++++---------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index c79b706bc8..d862cae503 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -513,20 +513,18 @@ sub createGlueProject { foreach (@apps) { $_ =~ s/\//_/g; $_ =~ s/\.exe//; - push(@tmp, $_); + if ($_ eq "git" ) { + unshift(@tmp, $_); + } else { + push(@tmp, $_); + } } @apps = @tmp; open F, ">git.sln" || die "Could not open git.sln for writing!\n"; binmode F, ":crlf"; print F "$SLN_HEAD"; - foreach (@libs) { - my $libname = $_; - my $uuid = $build_structure{"LIBS_${libname}_GUID"}; - print F "$SLN_PRE"; - print F "\"${libname}\", \"${libname}\\${libname}.vcproj\", \"${uuid}\""; - print F "$SLN_POST"; - } + my $uuid_libgit = $build_structure{"LIBS_libgit_GUID"}; my $uuid_xdiff_lib = $build_structure{"LIBS_xdiff_lib_GUID"}; foreach (@apps) { @@ -540,6 +538,13 @@ sub createGlueProject { print F " EndProjectSection"; print F "$SLN_POST"; } + foreach (@libs) { + my $libname = $_; + my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + print F "$SLN_PRE"; + print F "\"${libname}\", \"${libname}\\${libname}.vcproj\", \"${uuid}\""; + print F "$SLN_POST"; + } print F << "EOM"; Global @@ -551,17 +556,17 @@ EOM print F << "EOM"; GlobalSection(ProjectConfigurationPlatforms) = postSolution EOM - foreach (@libs) { - my $libname = $_; - my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + foreach (@apps) { + my $appname = $_; + my $uuid = $build_structure{"APPS_${appname}_GUID"}; print F "\t\t${uuid}.Debug|Win32.ActiveCfg = Debug|Win32\n"; print F "\t\t${uuid}.Debug|Win32.Build.0 = Debug|Win32\n"; print F "\t\t${uuid}.Release|Win32.ActiveCfg = Release|Win32\n"; print F "\t\t${uuid}.Release|Win32.Build.0 = Release|Win32\n"; } - foreach (@apps) { - my $appname = $_; - my $uuid = $build_structure{"APPS_${appname}_GUID"}; + foreach (@libs) { + my $libname = $_; + my $uuid = $build_structure{"LIBS_${libname}_GUID"}; print F "\t\t${uuid}.Debug|Win32.ActiveCfg = Debug|Win32\n"; print F "\t\t${uuid}.Debug|Win32.Build.0 = Debug|Win32\n"; print F "\t\t${uuid}.Release|Win32.ActiveCfg = Release|Win32\n"; From 5cb0e35d2b01c4b509883023616874fb93966bce Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Oct 2016 03:02:36 -0700 Subject: [PATCH 069/102] Vcproj.pm: do not configure VCWebServiceProxyGeneratorTool It is not necessary, and Visual Studio 2015 no longer supports it, anyway. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcproj.pm | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index d862cae503..b17800184c 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -115,9 +115,6 @@ sub createLibProject { - @@ -181,9 +178,6 @@ sub createLibProject { - @@ -339,9 +333,6 @@ sub createAppProject { - @@ -410,9 +401,6 @@ sub createAppProject { - From 5ebafe7da10dc5e00b60aae986bbc95c8c9d6a33 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Oct 2016 03:11:22 -0700 Subject: [PATCH 070/102] Vcproj.pm: urlencode '<' and '>' when generating VC projects Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcproj.pm | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index b17800184c..737647e76a 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -59,6 +59,8 @@ sub createLibProject { my $includes= join(";", sort(map(""$rel_dir\\$_"", @{$$build_structure{"LIBS_${libname}_INCLUDES"}}))); my $cflags = join(" ", sort(@{$$build_structure{"LIBS_${libname}_CFLAGS"}})); $cflags =~ s/\"/"/g; + $cflags =~ s//>/g; my $cflags_debug = $cflags; $cflags_debug =~ s/-MT/-MTd/; @@ -80,6 +82,8 @@ sub createLibProject { $defines =~ s/-D//g; $defines =~ s/\"/\\"/g; + $defines =~ s//>/g; $defines =~ s/\'//g; $includes =~ s/-I//g; mkdir "$target" || die "Could not create the directory $target for lib project!\n"; @@ -271,6 +275,8 @@ sub createAppProject { my $includes= join(";", sort(map(""$rel_dir\\$_"", @{$$build_structure{"APPS_${appname}_INCLUDES"}}))); my $cflags = join(" ", sort(@{$$build_structure{"APPS_${appname}_CFLAGS"}})); $cflags =~ s/\"/"/g; + $cflags =~ s//>/g; my $cflags_debug = $cflags; $cflags_debug =~ s/-MT/-MTd/; @@ -297,6 +303,8 @@ sub createAppProject { $defines =~ s/-D//g; $defines =~ s/\"/\\"/g; + $defines =~ s//>/g; $defines =~ s/\'//g; $defines =~ s/\\\\/\\/g; $includes =~ s/-I//g; From 2de8257cde948815385540c542c899e45c1dd5f7 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Mon, 20 Jul 2015 16:44:59 +0100 Subject: [PATCH 071/102] contrib/buildsystems: ignore invalidcontinue.obj Since 4b623d8 (MSVC: link in invalidcontinue.obj for better POSIX compatibility, 2014-03-29), invalidcontinue.obj is linked in the MSVC build, but it was not parsed correctly by the buildsystem. Ignore it, as it is known to Visual Studio and will be handled elsewhere. Also only substitute filenames ending with .o when generating the source .c filename, otherwise we would start to expect .cbj files to generate .obj files (which are not generated by our build)... In the future there may be source files that produce .obj files so keep the two issues (.obj files with & without source files) separate. Signed-off-by: Philip Oakley Signed-off-by: Duncan Smart Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 23da787dc5..53e65d4db7 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -282,7 +282,7 @@ sub handleLibLine # exit(1); foreach (@objfiles) { my $sourcefile = $_; - $sourcefile =~ s/\.o/.c/; + $sourcefile =~ s/\.o$/.c/; push(@sources, $sourcefile); push(@cflags, @{$compile_options{"${sourcefile}_CFLAGS"}}); push(@defines, @{$compile_options{"${sourcefile}_DEFINES"}}); @@ -326,8 +326,12 @@ sub handleLinkLine } elsif ($part =~ /\.(a|lib)$/) { $part =~ s/\.a$/.lib/; push(@libs, $part); - } elsif ($part =~ /\.(o|obj)$/) { + } elsif ($part eq 'invalidcontinue.obj') { + # ignore - known to MSVC + } elsif ($part =~ /\.o$/) { push(@objfiles, $part); + } elsif ($part =~ /\.obj$/) { + # do nothing, 'make' should not be producing .obj, only .o files } else { die "Unhandled lib option @ line $lineno: $part"; } @@ -336,7 +340,7 @@ sub handleLinkLine # exit(1); foreach (@objfiles) { my $sourcefile = $_; - $sourcefile =~ s/\.o/.c/; + $sourcefile =~ s/\.o$/.c/; push(@sources, $sourcefile); push(@cflags, @{$compile_options{"${sourcefile}_CFLAGS"}}); push(@defines, @{$compile_options{"${sourcefile}_DEFINES"}}); From fa91e2a5c6b50bdeea96dfe7b6ae76b369e5e3a7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 26 Oct 2016 04:59:06 -0700 Subject: [PATCH 072/102] contrib/buildsystems: ignore irrelevant files in Generators/ The Generators/ directory can contain spurious files such as editors' backup files. Even worse, there could be .swp files which are not even valid Perl scripts. Let's just ignore anything but .pm files in said directory. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/buildsystems/Generators.pm b/contrib/buildsystems/Generators.pm index 408ef714b8..aa4cbaa2ad 100644 --- a/contrib/buildsystems/Generators.pm +++ b/contrib/buildsystems/Generators.pm @@ -17,7 +17,7 @@ BEGIN { $me = dirname($me); if (opendir(D,"$me/Generators")) { foreach my $gen (readdir(D)) { - next if ($gen =~ /^\.\.?$/); + next unless ($gen =~ /\.pm$/); require "${me}/Generators/$gen"; $gen =~ s,\.pm,,; push(@AVAILABLE, $gen); From 999575f1f30643f4b6ff3547700a03bfd0106aff Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 17:41:13 +0100 Subject: [PATCH 073/102] contrib/buildsystems: fix misleading error message The error message talked about a "lib option", but it clearly referred to a link option. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 53e65d4db7..11f0e16dda 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -333,7 +333,7 @@ sub handleLinkLine } elsif ($part =~ /\.obj$/) { # do nothing, 'make' should not be producing .obj, only .o files } else { - die "Unhandled lib option @ line $lineno: $part"; + die "Unhandled link option @ line $lineno: $part"; } } # print "AppOut: '$appout'\nLFlags: @lflags\nLibs : @libs\nOfiles: @objfiles\n"; From 7808d4b4be443ccc5b119d5a302c68a16dfa789a Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 16:08:21 +0100 Subject: [PATCH 074/102] contrib/buildsystems: handle quoted spaces in filenames The engine.pl script expects file names not to contain spaces. However, paths with spaces are quite prevalent on Windows. Use shellwords() rather than split() to parse them correctly. Helped-by: Junio C Hamano Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 11f0e16dda..ad6a82c30c 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -12,6 +12,7 @@ use File::Basename; use File::Spec; use Cwd; use Generators; +use Text::ParseWords; my (%build_structure, %compile_options, @makedry); my $out_dir = getcwd(); @@ -231,7 +232,7 @@ sub removeDuplicates sub handleCompileLine { my ($line, $lineno) = @_; - my @parts = split(' ', $line); + my @parts = shellwords($line); my $sourcefile; shift(@parts); # ignore cmd while (my $part = shift @parts) { @@ -265,7 +266,7 @@ sub handleLibLine my (@objfiles, @lflags, $libout, $part); # kill cmd and rm 'prefix' $line =~ s/^rm -f .* && .* rcs //; - my @parts = split(' ', $line); + my @parts = shellwords($line); while ($part = shift @parts) { if ($part =~ /^-/) { push(@lflags, $part); @@ -306,7 +307,7 @@ sub handleLinkLine { my ($line, $lineno) = @_; my (@objfiles, @lflags, @libs, $appout, $part); - my @parts = split(' ', $line); + my @parts = shellwords($line); shift(@parts); # ignore cmd while ($part = shift @parts) { if ($part =~ /^-IGNORE/) { From 3b3079b39c6bf2bd398f8e93e5166d91bdce55a3 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Mon, 9 Feb 2015 14:34:29 +0000 Subject: [PATCH 075/102] contrib/buildsystems: ignore gettext stuff Git's build contains steps to handle internationalization. This caused hiccups in the parser used to generate QMake/Visual Studio project files. As those steps are irrelevant in this context, let's just ignore them. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index ad6a82c30c..9db3d43a1e 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -141,6 +141,12 @@ sub parseMakeOutput next; } + if ($text =~ /^(mkdir|msgfmt) /) { + # options to the Portable Object translations + # the line "mkdir ... && msgfmt ..." contains no linker options + next; + } + if($text =~ / -c /) { # compilation handleCompileLine($text, $line); From 8d34315c9f309fd7dc60981f71f9c76dffa1161d Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 16:45:32 +0100 Subject: [PATCH 076/102] contrib/buildsystems: redirect errors of the dry run into a log file Rather than swallowing the errors, it is better to have them in a file. To make it obvious what this is about, use the file name 'msvc-build-makedryerrors.txt'. Further, if the output is empty, simply delete that file. As we target Git for Windows' SDK (which, unlike its predecessor msysGit, offers Perl versions newer than 5.8), we can use the quite readable syntax `if -f -z $ErrsFile` (available in Perl >=5.10). Note that the file will contain the new values of the GIT_VERSION and GITGUI_VERSION if they were generated by the make file. They are omitted if the release is tagged and indentically defined in their respective GIT_VERSION_GEN file DEF_VER variables. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 9db3d43a1e..de5c0b6b25 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -73,7 +73,12 @@ Running GNU Make to figure out build structure... EOM # Pipe a make --dry-run into a variable, if not already loaded from file -@makedry = `cd $git_dir && make -n MSVC=1 V=1 2>/dev/null` if !@makedry; +# Capture the make dry stderr to file for review (will be empty for a release build). + +my $ErrsFile = "msvc-build-makedryerrors.txt"; +@makedry = `make -C $git_dir -n MSVC=1 V=1 2>$ErrsFile` if !@makedry; +# test for an empty Errors file and remove it +unlink $ErrsFile if -f -z $ErrsFile; # Parse the make output into usable info parseMakeOutput(); From 02a8269a0a7a7ef88905c75e5145e79521528903 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 16 Jul 2015 23:40:13 +0100 Subject: [PATCH 077/102] contrib/buildsystems: optionally capture the dry-run in a file Add an option for capturing the output of the make dry-run used in determining the msvc-build structure for easy debugging. You can use the output of `--make-out ` in subsequent runs via the `--in ` option. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index de5c0b6b25..732239d817 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -32,6 +32,7 @@ generate usage: -g --gen Specify the buildsystem generator (default: $gen) Available: $genlist -o --out Specify output directory generation (default: .) + --make-out Write the output of GNU Make into a file -i --in Specify input file, instead of running GNU Make -h,-? --help This help EOM @@ -39,6 +40,7 @@ EOM } # Parse command-line options +my $make_out; while (@ARGV) { my $arg = shift @ARGV; if ("$arg" eq "-h" || "$arg" eq "--help" || "$arg" eq "-?") { @@ -46,6 +48,8 @@ while (@ARGV) { exit(0); } elsif("$arg" eq "--out" || "$arg" eq "-o") { $out_dir = shift @ARGV; + } elsif("$arg" eq "--make-out") { + $make_out = shift @ARGV; } elsif("$arg" eq "--gen" || "$arg" eq "-g") { $gen = shift @ARGV; } elsif("$arg" eq "--in" || "$arg" eq "-i") { @@ -80,6 +84,12 @@ my $ErrsFile = "msvc-build-makedryerrors.txt"; # test for an empty Errors file and remove it unlink $ErrsFile if -f -z $ErrsFile; +if (defined $make_out) { + open OUT, ">" . $make_out; + print OUT @makedry; + close OUT; +} + # Parse the make output into usable info parseMakeOutput(); From c53442557c9d171d1e1047946ee8084eab7e6080 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:00:01 +0100 Subject: [PATCH 078/102] msvc: ignore .dll and incremental compile output Ignore .dll files copied into the top-level directory. Ignore MSVC incremental compiler output files. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 0d77ea5894..81be1c907b 100644 --- a/.gitignore +++ b/.gitignore @@ -227,5 +227,10 @@ *.user *.idb *.pdb +*.ilk +*.iobj +*.ipdb +*.dll +.vs/ /Debug/ /Release/ From 6425f609ff146daabccd43298aa8426816770855 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Wed, 4 May 2016 15:53:48 +0100 Subject: [PATCH 079/102] contrib/buildsystems: handle the curl library option Upon seeing the '-lcurl' option, point to the libcurl.lib. While there, fix the elsif indentation. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 732239d817..fddf2dc151 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -339,10 +339,12 @@ sub handleLinkLine $appout = shift @parts; } elsif ("$part" eq "-lz") { push(@libs, "zlib.lib"); - } elsif ("$part" eq "-lcrypto") { + } elsif ("$part" eq "-lcrypto") { push(@libs, "libeay32.lib"); } elsif ("$part" eq "-lssl") { push(@libs, "ssleay32.lib"); + } elsif ("$part" eq "-lcurl") { + push(@libs, "libcurl.lib"); } elsif ($part =~ /^-/) { push(@lflags, $part); } elsif ($part =~ /\.(a|lib)$/) { From 806cfd434d135ae66ff940ff547dc903301d6f28 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Oct 2016 01:45:49 -0700 Subject: [PATCH 080/102] contrib/buildsystems: handle libiconv, too Git's test suite shows tons of breakages unless Git is compiled *without* NO_ICONV. That means, in turn, that we need to generate build definitions *with* libiconv, which in turn implies that we have to handle the -liconv option properly. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index fddf2dc151..44adadb2e6 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -345,6 +345,8 @@ sub handleLinkLine push(@libs, "ssleay32.lib"); } elsif ("$part" eq "-lcurl") { push(@libs, "libcurl.lib"); + } elsif ("$part" eq "-liconv") { + push(@libs, "libiconv.lib"); } elsif ($part =~ /^-/) { push(@lflags, $part); } elsif ($part =~ /\.(a|lib)$/) { From 6be147917b25cdfd1e5cd63d14729cf359f5da43 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Oct 2016 07:25:00 -0700 Subject: [PATCH 081/102] contrib/buildsystems: handle options starting with a slash With the recent changes to allow building with MSVC=1, we now pass the /OPT:REF option to the compiler. This confuses the parser that wants to turn the output of a dry run into project definitions for QMake and Visual Studio: Unhandled link option @ line 213: /OPT:REF at [...] Let's just extend the code that passes through options that start with a dash, so that it passes through options that start with a slash, too. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 44adadb2e6..134a82d31f 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -347,7 +347,7 @@ sub handleLinkLine push(@libs, "libcurl.lib"); } elsif ("$part" eq "-liconv") { push(@libs, "libiconv.lib"); - } elsif ($part =~ /^-/) { + } elsif ($part =~ /^[-\/]/) { push(@lflags, $part); } elsif ($part =~ /\.(a|lib)$/) { $part =~ s/\.a$/.lib/; From 26e78d1774cb50bb4d4b57f6f76611959e30eba2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 4 Dec 2017 20:43:23 +0100 Subject: [PATCH 082/102] contrib/buildsystems: error out on unknown option One time too many did this developer call the `generate` script passing a `--make-out=` option that was happily ignored (because there should be a space, not an equal sign, between `--make-out` and the path). And one time too many, this script not only ignored it but did not even complain. Let's fix that. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 134a82d31f..9f4e7a2ccb 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -57,6 +57,8 @@ while (@ARGV) { open(F, "<$infile") || die "Couldn't open file $infile"; @makedry = ; close(F); + } else { + die "Unknown option: " . $arg; } } From 3366bf928eba62afc9cab07db8642609198c0bd2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 26 Oct 2016 02:28:10 -0700 Subject: [PATCH 083/102] contrib/buildsystems: add a backend for modern Visual Studio versions Based on the previous patch series to be able to compile Git using Visual C++ from the command-line, this patch offers to generate project definitions for Visual Studio, so that Git can be developed in a modern IDE. Based on the generator for Visual Studio versions <= 2008 (which used .sln/.vcproj files), this patch copy-edits the generator of the .vcproj files to a new generator that produces .vcxproj files ready for Visual Studio 2010 and later (or MSBuild). As the vcpkg system (which is used to build Git's dependencies) cannot run in parallel (it does not lock, wreaking havoc with files being accessed and written at the same time, letting the vcpkg processes stumble over each others' toes), we make libgit the root of the project dependency tree and initialize the vcpkg system in this project's PreBuildEvent. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcxproj.pm | 380 +++++++++++++++++++++ contrib/buildsystems/engine.pl | 2 + 2 files changed, 382 insertions(+) create mode 100644 contrib/buildsystems/Generators/Vcxproj.pm diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm new file mode 100644 index 0000000000..0c2a11c0f1 --- /dev/null +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -0,0 +1,380 @@ +package Generators::Vcxproj; +require Exporter; + +use strict; +use vars qw($VERSION); +use Digest::SHA qw(sha256_hex); + +our $VERSION = '1.00'; +our(@ISA, @EXPORT, @EXPORT_OK, @AVAILABLE); +@ISA = qw(Exporter); + +BEGIN { + push @EXPORT_OK, qw(generate); +} + +sub generate_guid ($) { + my $hex = sha256_hex($_[0]); + $hex =~ s/^(.{8})(.{4})(.{4})(.{4})(.{12}).*/{$1-$2-$3-$4-$5}/; + $hex =~ tr/a-z/A-Z/; + return $hex; +} + +sub generate { + my ($git_dir, $out_dir, $rel_dir, %build_structure) = @_; + my @libs = @{$build_structure{"LIBS"}}; + foreach (@libs) { + createProject($_, $git_dir, $out_dir, $rel_dir, \%build_structure, 1); + } + + my @apps = @{$build_structure{"APPS"}}; + foreach (@apps) { + createProject($_, $git_dir, $out_dir, $rel_dir, \%build_structure, 0); + } + + createGlueProject($git_dir, $out_dir, $rel_dir, %build_structure); + return 0; +} + +sub createProject { + my ($name, $git_dir, $out_dir, $rel_dir, $build_structure, $static_library) = @_; + my $label = $static_library ? "lib" : "app"; + my $prefix = $static_library ? "LIBS_" : "APPS_"; + my $config_type = $static_library ? "StaticLibrary" : "Application"; + print "Generate $name vcxproj $label project\n"; + my $cdup = $name; + $cdup =~ s/[^\/]+/../g; + $cdup =~ s/\//\\/g; + $rel_dir = $rel_dir eq "." ? $cdup : "$cdup\\$rel_dir"; + $rel_dir =~ s/\//\\/g; + + my $target = $name; + if ($static_library) { + $target =~ s/\.a//; + } else { + $target =~ s/\.exe//; + } + + my $uuid = generate_guid($name); + $$build_structure{"$prefix${target}_GUID"} = $uuid; + my $vcxproj = $target; + $vcxproj =~ s/(.*\/)?(.*)/$&\/$2.vcxproj/; + $vcxproj =~ s/([^\/]*)(\/lib)\/(lib.vcxproj)/$1$2\/$1_$3/; + $$build_structure{"$prefix${target}_VCXPROJ"} = $vcxproj; + + my @srcs = sort(map("$rel_dir\\$_", @{$$build_structure{"$prefix${name}_SOURCES"}})); + my @sources; + foreach (@srcs) { + $_ =~ s/\//\\/g; + push(@sources, $_); + } + my $defines = join(";", sort(@{$$build_structure{"$prefix${name}_DEFINES"}})); + my $includes= join(";", sort(map { s/^-I//; s/\//\\/g; File::Spec->file_name_is_absolute($_) ? $_ : "$rel_dir\\$_" } @{$$build_structure{"$prefix${name}_INCLUDES"}})); + my $cflags = join(" ", sort(map { s/^-[GLMOWZ].*//; s/.* .*/"$&"/; $_; } @{$$build_structure{"$prefix${name}_CFLAGS"}})); + $cflags =~ s//>/g; + + my $libs_release = "\n "; + my $libs_debug = "\n "; + if (!$static_library) { + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_debug = $libs_release; + $libs_debug =~ s/zlib\.lib/zlibd\.lib/; + } + + $defines =~ s/-D//g; + $defines =~ s//>/g; + $defines =~ s/\'//g; + + die "Could not create the directory $target for $label project!\n" unless (-d "$target" || mkdir "$target"); + + open F, ">$vcxproj" or die "Could not open $vcxproj for writing!\n"; + binmode F, ":crlf :utf8"; + print F chr(0xFEFF); + print F << "EOM"; + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + $uuid + Win32Proj + x86-windows + x64-windows + $cdup\\compat\\vcbuild\\vcpkg\\installed\\\$(VCPKGArch) + \$(VCPKGArchDirectory)\\debug\\bin + \$(VCPKGArchDirectory)\\debug\\lib + \$(VCPKGArchDirectory)\\bin + \$(VCPKGArchDirectory)\\lib + \$(VCPKGArchDirectory)\\include + $libs_debug + $libs_release + + + + true + true + + + false + true + + + $config_type + v140 + + ..\\ + + + + + + + + + + + + + false + true + + + + $cflags %(AdditionalOptions) + $cdup;$cdup\\compat;$cdup\\compat\\regex;$cdup\\compat\\win32;$cdup\\compat\\poll;$cdup\\compat\\vcbuild\\include;\$(VCPKGIncludeDirectory);%(AdditionalIncludeDirectories) + + OnlyExplicitInline + + ProgramDatabase + + + true + + + \$(VCPKGLibDirectory);%(AdditionalLibraryDirectories) + \$(VCPKGLibs);\$(AdditionalDependencies) + invalidcontinue.obj %(AdditionalOptions) + wmainCRTStartup + $cdup\\compat\\win32\\git.manifest + Console + +EOM + if ($target eq 'libgit') { + print F << "EOM"; + + Initialize VCPKG + del "$cdup\\compat\\vcbuild\\vcpkg" + call "$cdup\\compat\\vcbuild\\vcpkg_install.bat" + +EOM + } + print F << "EOM"; + + + + MachineX86 + + + + + Disabled + WIN32;_DEBUG;$defines;%(PreprocessorDefinitions) + MultiThreadedDebugDLL + + + true + + + + + MaxSpeed + true + WIN32;NDEBUG;$defines;%(PreprocessorDefinitions) + MultiThreadedDLL + true + Speed + + + true + true + true + + + +EOM + foreach(@sources) { + print F << "EOM"; + +EOM + } + print F << "EOM"; + +EOM + if (!$static_library || $target =~ 'vcs-svn') { + my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; + + print F << "EOM"; + + + $uuid_libgit + false + + + $uuid_xdiff_lib + false + +EOM + if ($name =~ /(test-(line-buffer|svn-fe)|^git-remote-testsvn)\.exe$/) { + my $uuid_vcs_svn_lib = $$build_structure{"LIBS_vcs-svn/lib_GUID"}; + print F << "EOM"; + + $uuid_vcs_svn_lib + false + +EOM + } + print F << "EOM"; + +EOM + } + print F << "EOM"; + +EOM + if (!$static_library) { + print F << "EOM"; + + + + + + +EOM + } + print F << "EOM"; + +EOM + close F; +} + +sub createGlueProject { + my ($git_dir, $out_dir, $rel_dir, %build_structure) = @_; + print "Generate solutions file\n"; + $rel_dir = "..\\$rel_dir"; + $rel_dir =~ s/\//\\/g; + my $SLN_HEAD = "Microsoft Visual Studio Solution File, Format Version 11.00\n# Visual Studio 2010\n"; + my $SLN_PRE = "Project(\"{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}\") = "; + my $SLN_POST = "\nEndProject\n"; + + my @libs = @{$build_structure{"LIBS"}}; + my @tmp; + foreach (@libs) { + $_ =~ s/\.a//; + push(@tmp, $_); + } + @libs = @tmp; + + my @apps = @{$build_structure{"APPS"}}; + @tmp = (); + foreach (@apps) { + $_ =~ s/\.exe//; + if ($_ eq "git" ) { + unshift(@tmp, $_); + } else { + push(@tmp, $_); + } + } + @apps = @tmp; + + open F, ">git.sln" || die "Could not open git.sln for writing!\n"; + binmode F, ":crlf :utf8"; + print F chr(0xFEFF); + print F "$SLN_HEAD"; + + foreach (@apps) { + my $appname = $_; + my $uuid = $build_structure{"APPS_${appname}_GUID"}; + print F "$SLN_PRE"; + my $vcxproj = $build_structure{"APPS_${appname}_VCXPROJ"}; + $vcxproj =~ s/\//\\/g; + $appname =~ s/.*\///; + print F "\"${appname}\", \"${vcxproj}\", \"${uuid}\""; + print F "$SLN_POST"; + } + foreach (@libs) { + my $libname = $_; + my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + print F "$SLN_PRE"; + my $vcxproj = $build_structure{"LIBS_${libname}_VCXPROJ"}; + $vcxproj =~ s/\//\\/g; + $libname =~ s/\//_/g; + print F "\"${libname}\", \"${vcxproj}\", \"${uuid}\""; + print F "$SLN_POST"; + } + + print F << "EOM"; +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection +EOM + print F << "EOM"; + GlobalSection(ProjectConfigurationPlatforms) = postSolution +EOM + foreach (@apps) { + my $appname = $_; + my $uuid = $build_structure{"APPS_${appname}_GUID"}; + print F "\t\t${uuid}.Debug|x64.ActiveCfg = Debug|x64\n"; + print F "\t\t${uuid}.Debug|x64.Build.0 = Debug|x64\n"; + print F "\t\t${uuid}.Debug|x86.ActiveCfg = Debug|Win32\n"; + print F "\t\t${uuid}.Debug|x86.Build.0 = Debug|Win32\n"; + print F "\t\t${uuid}.Release|x64.ActiveCfg = Release|x64\n"; + print F "\t\t${uuid}.Release|x64.Build.0 = Release|x64\n"; + print F "\t\t${uuid}.Release|x86.ActiveCfg = Release|Win32\n"; + print F "\t\t${uuid}.Release|x86.Build.0 = Release|Win32\n"; + } + foreach (@libs) { + my $libname = $_; + my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + print F "\t\t${uuid}.Debug|x64.ActiveCfg = Debug|x64\n"; + print F "\t\t${uuid}.Debug|x64.Build.0 = Debug|x64\n"; + print F "\t\t${uuid}.Debug|x86.ActiveCfg = Debug|Win32\n"; + print F "\t\t${uuid}.Debug|x86.Build.0 = Debug|Win32\n"; + print F "\t\t${uuid}.Release|x64.ActiveCfg = Release|x64\n"; + print F "\t\t${uuid}.Release|x64.Build.0 = Release|x64\n"; + print F "\t\t${uuid}.Release|x86.ActiveCfg = Release|Win32\n"; + print F "\t\t${uuid}.Release|x86.Build.0 = Release|Win32\n"; + } + + print F << "EOM"; + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal +EOM + close F; +} + +1; diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 9f4e7a2ccb..8bb07e8e25 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -347,6 +347,8 @@ sub handleLinkLine push(@libs, "ssleay32.lib"); } elsif ("$part" eq "-lcurl") { push(@libs, "libcurl.lib"); + } elsif ("$part" eq "-lexpat") { + push(@libs, "expat.lib"); } elsif ("$part" eq "-liconv") { push(@libs, "libiconv.lib"); } elsif ($part =~ /^[-\/]/) { From 7006085f2113582577c368dbc83ef6291877dd00 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 28 Nov 2016 15:54:59 +0100 Subject: [PATCH 084/102] msvc: add a Makefile target to pre-generate the VS solution The entire idea of generating the VS solution makes only sense if we generate it via Continuous Integration; otherwise potential users would still have to download the entire Git for Windows SDK. So let's just add a target in the Makefile that can be used to generate said solution; The generated files will then be committed so that they can be pushed to a branch ready to check out by Visual Studio users. To make things even more useful, we also generate and commit other files that are required to run the test suite, such as templates and bin-wrappers: with this, developers can run the test suite in a regular Git Bash (that is part of a regular Git for Windows installation) after building the solution in Visual Studio. Signed-off-by: Johannes Schindelin --- config.mak.uname | 61 ++++++++++++++++++++++++++++++++++ contrib/buildsystems/engine.pl | 3 +- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index e0a0ae59cc..d9cefa7b9c 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -25,10 +25,12 @@ include compat/vcbuild/MSVC-DEFS-GEN # See if vcpkg and the vcpkg-build versions of the third-party # libraries that we use are installed. We include the result # to get $(vcpkg_*) variables defined for the Makefile. +ifeq (,$(SKIP_VCPKG)) compat/vcbuild/VCPKG-DEFS: compat/vcbuild/vcpkg_install.bat @"$<" include compat/vcbuild/VCPKG-DEFS endif +endif # We choose to avoid "if .. else if .. else .. endif endif" # because maintaining the nesting to match is a pain. If @@ -659,3 +661,62 @@ ifeq ($(uname_S),QNX) NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease endif + +vcxproj: + # Require clean work tree + git update-index -q --refresh && \ + git diff-files --quiet && \ + git diff-index --cached --quiet HEAD -- + + # Make .vcxproj files and add them + unset QUIET_GEN QUIET_BUILT_IN; \ + perl contrib/buildsystems/generate -g Vcxproj + git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj + + # Add command-list.h + $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 command-list.h + git add -f command-list.h + + # Add scripts + rm -f perl/perl.mak + $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 \ + $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) + # Strip out the sane tool path, needed only for building + sed -i '/^git_broken_path_fix ".*/d' git-sh-setup + git add -f $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) + + # Add Perl module + $(MAKE) $(LIB_PERL_GEN) + git add -f perl/build + + # Add bin-wrappers, for testing + rm -rf bin-wrappers/ + $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 $(test_bindir_programs) + # Ensure that the GIT_EXEC_PATH is a Unix-y one, and that the absolute + # path of the repository is not hard-coded (GIT_EXEC_PATH will be set + # by test-lib.sh according to the current setup) + sed -i -e 's/^\(GIT_EXEC_PATH\)=.*/test -n "$${\1##*:*}" ||\ + \1="$$(cygpath -u "$$\1")"/' \ + -e "s|'$$(pwd)|\"\$$GIT_EXEC_PATH\"'|g" bin-wrappers/* + # Ensure that test-* helpers find the .dll files copied to top-level + sed -i 's|^PATH=.*|&:"$$GIT_EXEC_PATH"|' bin-wrappers/test-* + # We do not want to force hard-linking builtins + sed -i 's|\(git\)-\([-a-z]*\)\.exe"|\1.exe" \2|g' \ + bin-wrappers/git-{receive-pack,upload-archive} + git add -f $(test_bindir_programs) + # remote-ext is a builtin, but invoked as if it were external + sed 's|receive-pack|remote-ext|g' \ + bin-wrappers/git-remote-ext + git add -f bin-wrappers/git-remote-ext + + # Add templates + $(MAKE) -C templates + git add -f templates/boilerplates.made templates/blt/ + + # Add build options + $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 GIT-BUILD-OPTIONS + git add -f GIT-BUILD-OPTIONS + + # Commit the whole shebang + git commit -m "Generate Visual Studio solution" \ + -m "Auto-generated by \`$(MAKE)$(MAKEFLAGS) $@\`" diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 8bb07e8e25..fba8a3f056 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -82,7 +82,8 @@ EOM # Capture the make dry stderr to file for review (will be empty for a release build). my $ErrsFile = "msvc-build-makedryerrors.txt"; -@makedry = `make -C $git_dir -n MSVC=1 V=1 2>$ErrsFile` if !@makedry; +@makedry = `make -C $git_dir -n MSVC=1 SKIP_VCPKG=1 V=1 2>$ErrsFile` +if !@makedry; # test for an empty Errors file and remove it unlink $ErrsFile if -f -z $ErrsFile; From 87591a5f9b4fb406db7d34b343be73db55cb53e7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 19 Dec 2017 13:54:14 +0100 Subject: [PATCH 085/102] vcxproj: also link-or-copy builtins The problem with not having, say, git-receive-pack.exe after a full build is that the test suite will then happily use the *installed* git-receive-pack.exe because it finds nothing else. Absolutely not what we want. We want to have confidence that our test covers the MSVC-built Git executables, and not some random stuff. Signed-off-by: Johannes Schindelin --- config.mak.uname | 15 +++++++++++++++ contrib/buildsystems/Generators/Vcxproj.pm | 3 +++ 2 files changed, 18 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index d9cefa7b9c..de914f4593 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -673,6 +673,21 @@ vcxproj: perl contrib/buildsystems/generate -g Vcxproj git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj + # Generate the LinkOrCopyBuiltins.targets file + (echo '' && \ + echo ' ' && \ + for name in $(BUILT_INS);\ + do \ + echo ' '; \ + done && \ + for name in $(REMOTE_CURL_ALIASES); \ + do \ + echo ' '; \ + done && \ + echo ' ' && \ + echo '') >git/LinkOrCopyBuiltins.targets + git add -f git/LinkOrCopyBuiltins.targets + # Add command-list.h $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 command-list.h git add -f command-list.h diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index 0c2a11c0f1..00ef26fddd 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -269,6 +269,9 @@ EOM EOM } + if ($target eq 'git') { + print F " \n"; + } print F << "EOM"; EOM From 4415c12c9e4bf0ca2b7b6cf770e35a2f49ab8463 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 16:02:40 +0100 Subject: [PATCH 086/102] .gitignore: touch up the entries regarding Visual Studio Add the Microsoft .manifest pattern, and do not anchor the 'Debug' and 'Release' entries at the top-level directory, to allow for multiple projects (one per target). Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 81be1c907b..797605d2ef 100644 --- a/.gitignore +++ b/.gitignore @@ -232,5 +232,6 @@ *.ipdb *.dll .vs/ -/Debug/ -/Release/ +*.manifest +Debug/ +Release/ From 63496fa6305b4b3a0a709af0e3e17fd42c6384f0 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Mon, 23 Feb 2015 12:50:35 +0000 Subject: [PATCH 087/102] WIP .gitignore: ignore library directories created by MSVC VS2008 buildsystem TODO: test whether we can drop this. Signed-off-by: Philip Oakley --- .gitignore | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.gitignore b/.gitignore index 797605d2ef..07ff921f79 100644 --- a/.gitignore +++ b/.gitignore @@ -190,6 +190,42 @@ /gitweb/static/gitweb.js /gitweb/static/gitweb.min.* /command-list.h +/libgit +/test-chmtime +/test-ctype +/test-config +/test-date +/test-delta +/test-dump-cache-tree +/test-dump-split-index +/test-dump-untracked-cache +/test-fake-ssh +/test-scrap-cache-tree +/test-genrandom +/test-hashmap +/test-index-version +/test-line-buffer +/test-match-trees +/test-mergesort +/test-mktemp +/test-parse-options +/test-path-utils +/test-prio-queue +/test-read-cache +/test-regex +/test-revision-walking +/test-run-command +/test-sha1 +/test-sha1-array +/test-sigchain +/test-string-list +/test-submodule-config +/test-subprocess +/test-svn-fe +/test-urlmatch-normalization +/test-wildmatch +/vcs-svn_lib +/xdiff_lib *.tar.gz *.dsc *.deb From 03ac31268828aa2e2c953ea6f7c4adfdbd85c2dd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Oct 2016 06:06:10 -0700 Subject: [PATCH 088/102] .gitignore: ignore Visual Studio's temporary/generated files Signed-off-by: Johannes Schindelin --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 07ff921f79..56a7fefa76 100644 --- a/.gitignore +++ b/.gitignore @@ -271,3 +271,6 @@ *.manifest Debug/ Release/ +/UpgradeLog*.htm +/git.VC.VC.opendb +/git.VC.db From 9f177f45b00142da0aab86da91c9cfcf54788be9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 25 Nov 2016 18:29:51 +0100 Subject: [PATCH 089/102] bin-wrappers: append `.exe` to target paths if necessary When compiling with Visual Studio, the projects' names are identical to the executables modulo the extensions. Read: there will exist both a directory called `git` as well as an executable called `git.exe` in the end. Which means that the bin-wrappers *need* to target the `.exe` files lest they try to execute directories. Signed-off-by: Johannes Schindelin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 51934084a9..3531ff367f 100644 --- a/Makefile +++ b/Makefile @@ -2650,7 +2650,7 @@ bin-wrappers/%: wrap-for-bin.sh @mkdir -p bin-wrappers $(QUIET_GEN)sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \ -e 's|@@BUILD_DIR@@|$(shell pwd)|' \ - -e 's|@@PROG@@|$(patsubst test-%,t/helper/test-%,$(@F))|' < $< > $@ && \ + -e 's|@@PROG@@|$(patsubst test-%,t/helper/test-%$(X),$(@F))$(patsubst git%,$(X),$(filter $(@F),$(BINDIR_PROGRAMS_NEED_X)))|' < $< > $@ && \ chmod +x $@ # GNU make supports exporting all variables by "export" without parameters. From aa62003775ce5b9264b18e7ff0326dd23ac83a0c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Nov 2016 12:00:59 +0100 Subject: [PATCH 090/102] t5505,t5516: create .git/branches/ when needed It is a real old anachronism from the Cogito days to have a .git/branches/ directory. And to have tests that ensure that Cogito users can migrate away from using that directory. But so be it, let's continue testing it. Let's make sure, however, that git init does not need to create that directory. This bug was noticed when testing with templates that had been pre-committed, skipping the empty branches/ directory of course because Git does not track empty directories. Signed-off-by: Johannes Schindelin --- t/t5505-remote.sh | 2 ++ t/t5516-fetch-push.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index d2a2cdd453..b26732a080 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -824,6 +824,7 @@ test_expect_success 'migrate a remote from named file in $GIT_DIR/branches' ' ( cd six && git remote rm origin && + mkdir -p .git/branches && echo "$origin_url" >.git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && @@ -838,6 +839,7 @@ test_expect_success 'migrate a remote from named file in $GIT_DIR/branches (2)' ( cd seven && git remote rm origin && + mkdir -p .git/branches && echo "quux#foom" > .git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 37e8e80893..d6ad36f7af 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -866,6 +866,7 @@ test_expect_success 'fetch with branches' ' mk_empty testrepo && git branch second $the_first_commit && git checkout second && + mkdir -p testrepo/.git/branches && echo ".." > testrepo/.git/branches/branch1 && ( cd testrepo && @@ -879,6 +880,7 @@ test_expect_success 'fetch with branches' ' test_expect_success 'fetch with branches containing #' ' mk_empty testrepo && + mkdir -p testrepo/.git/branches && echo "..#second" > testrepo/.git/branches/branch2 && ( cd testrepo && @@ -893,6 +895,7 @@ test_expect_success 'fetch with branches containing #' ' test_expect_success 'push with branches' ' mk_empty testrepo && git checkout second && + mkdir -p .git/branches && echo "testrepo" > .git/branches/branch1 && git push branch1 && ( @@ -905,6 +908,7 @@ test_expect_success 'push with branches' ' test_expect_success 'push with branches containing #' ' mk_empty testrepo && + mkdir -p .git/branches && echo "testrepo#branch3" > .git/branches/branch2 && git push branch2 && ( From a7924b655e940b06cb547c235d6bed9767929673 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 28 Nov 2016 18:17:49 +0100 Subject: [PATCH 091/102] git: avoid calling aliased builtins via their dashed form This is one of the few places where Git violates its own deprecation of the dashed form. It is not necessary, either. As of 595d59e2b53 (git.c: ignore pager.* when launching builtin as dashed external, 2017-08-02), Git wants to ignore the pager.* config setting when expanding aliases. So let's strip out the check_pager_config() call from the copy-edited code. Signed-off-by: Johannes Schindelin --- git.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/git.c b/git.c index 2f604a41ea..a4a6942e16 100644 --- a/git.c +++ b/git.c @@ -699,6 +699,31 @@ static int run_argv(int *argcp, const char ***argv) */ if (!done_alias) handle_builtin(*argcp, *argv); + else if (get_builtin(**argv)) { + struct argv_array args = ARGV_ARRAY_INIT; + int i; + + if (get_super_prefix()) + die("%s doesn't support --super-prefix", **argv); + + commit_pager_choice(); + + argv_array_push(&args, "git"); + for (i = 0; i < *argcp; i++) + argv_array_push(&args, (*argv)[i]); + + trace_argv_printf(args.argv, "trace: exec:"); + + /* + * if we fail because the command is not found, it is + * OK to return. Otherwise, we just pass along the status code. + */ + i = run_command_v_opt(args.argv, RUN_SILENT_EXEC_FAILURE | + RUN_CLEAN_ON_EXIT); + if (i >= 0 || errno != ENOENT) + exit(i); + die("could not execute builtin %s", **argv); + } /* .. then try the external ones */ execv_dashed_external(*argv); From 4b6219d6f65e004922376a162f080f3c8083405d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 6 Jul 2013 02:09:35 +0200 Subject: [PATCH 092/102] Win32: make FILETIME conversion functions public We will use them in the upcoming "FSCache" patches (to accelerate sequential lstat() calls). Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 18 ------------------ compat/mingw.h | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 5487c8bfca..1d511ff1f4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -637,24 +637,6 @@ int mingw_chmod(const char *filename, int mode) return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) -{ - long long hnsec = filetime_to_hnsec(ft); - ts->tv_sec = (time_t)(hnsec / 10000000); - ts->tv_nsec = (hnsec % 10000000) * 100; -} - /** * Verifies that safe_create_leading_directories() would succeed. */ diff --git a/compat/mingw.h b/compat/mingw.h index 90a9e84a60..57b7633c79 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -353,6 +353,17 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows, * including our own struct stat with 64 bit st_size and nanosecond-precision @@ -369,6 +380,13 @@ struct timespec { #endif #endif +static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) +{ + long long hnsec = filetime_to_hnsec(ft); + ts->tv_sec = (time_t)(hnsec / 10000000); + ts->tv_nsec = (hnsec % 10000000) * 100; +} + struct mingw_stat { _dev_t st_dev; _ino_t st_ino; From bbef263558c19a51350e7fe62d9e48b59396d7e3 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:17:31 +0200 Subject: [PATCH 093/102] Win32: dirent.c: Move opendir down Move opendir down in preparation for the next patch. Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 52420ec7d4..2603a0fa39 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -18,40 +18,6 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - struct dirent *readdir(DIR *dir) { if (!dir) { @@ -90,3 +56,37 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *opendir(const char *name) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + DIR *dir; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((len = xutftowcs_path(pattern, name)) < 0) + return NULL; + + /* append optional '/' and wildcard '*' */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '/'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(DIR)); + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return dir; +} From 71e152d34dfbc3fc6348c6a5bc9303aeb1749065 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:18:40 +0200 Subject: [PATCH 094/102] Win32: Make the dirent implementation pluggable Emulating the POSIX dirent API on Windows via FindFirstFile/FindNextFile is pretty staightforward, however, most of the information provided in the WIN32_FIND_DATA structure is thrown away in the process. A more sophisticated implementation may cache this data, e.g. for later reuse in calls to lstat. Make the dirent implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Define a base DIR structure with pointers to readdir/closedir that match the opendir implementation (i.e. similar to vtable pointers in OOP). Define readdir/closedir so that they call the function pointers in the DIR structure. This allows to choose the opendir implementation on a call-by-call basis. Move the fixed sized dirent.d_name buffer to the dirent-specific DIR structure, as d_name may be implementation specific (e.g. a caching implementation may just set d_name to point into the cache instead of copying the entire file name string). Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 27 +++++++++++++++++---------- compat/win32/dirent.h | 26 +++++++++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 2603a0fa39..6b87042182 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,19 @@ #include "../../git-compat-util.h" -struct DIR { +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ struct dirent dd_dir; /* includes d_type */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + char dd_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ +} dirent_DIR; + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) @@ -18,7 +22,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -45,7 +49,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -57,13 +61,13 @@ int closedir(DIR *dir) return 0; } -DIR *opendir(const char *name) +DIR *dirent_opendir(const char *name) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; HANDLE h; int len; - DIR *dir; + dirent_DIR *dir; /* convert name to UTF-16 and check length < MAX_PATH */ if ((len = xutftowcs_path(pattern, name)) < 0) @@ -84,9 +88,12 @@ DIR *opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); + dir = xmalloc(sizeof(dirent_DIR)); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; + dir->dd_dir.d_name = dir->dd_name; dir->dd_handle = h; dir->dd_stat = 0; finddata2dirent(&dir->dd_dir, &fdata); - return dir; + return (DIR*) dir; } diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bf..6b3ddee51b 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,32 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char *d_name; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ From 180a10a8e590c3e78007b4e6178b79c6fdc740b5 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:21:30 +0200 Subject: [PATCH 095/102] Win32: make the lstat implementation pluggable Emulating the POSIX lstat API on Windows via GetFileAttributes[Ex] is quite slow. Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. A caching implementation may improve performance by bulk-reading entire directories or reusing data obtained via opendir / readdir. Make the lstat implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 2 ++ compat/mingw.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 1d511ff1f4..3244086e6d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -776,6 +776,8 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return do_lstat(follow, alt_name, buf); } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) { BY_HANDLE_FILE_INFORMATION fdata; diff --git a/compat/mingw.h b/compat/mingw.h index 57b7633c79..416092dd8f 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -419,7 +419,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); int mingw_utime(const char *file_name, const struct utimbuf *times); From d91ee8efdc6304be798cd779eff7153d2b4e03a3 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:23:27 +0200 Subject: [PATCH 096/102] add infrastructure for read-only file system level caches Add a macro to mark code sections that only read from the file system, along with a config option and documentation. This facilitates implementation of relatively simple file system level caches without the need to synchronize with the file system. Enable read-only sections for 'git status' and preload_index. Signed-off-by: Karsten Blees --- Documentation/config/core.txt | 6 ++++++ builtin/commit.c | 1 + compat/mingw.c | 6 ++++++ compat/mingw.h | 2 ++ git-compat-util.h | 15 +++++++++++++++ preload-index.c | 2 ++ 6 files changed, 32 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index d0e6635fe0..f6025349a1 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -548,6 +548,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/builtin/commit.c b/builtin/commit.c index c021b119bb..4e9e180ad6 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1363,6 +1363,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(1); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) progress_flag = REFRESH_PROGRESS; diff --git a/compat/mingw.c b/compat/mingw.c index 3244086e6d..2a9522b89a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -213,6 +213,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; +int core_fscache; int mingw_core_config(const char *var, const char *value, void *cb) { @@ -224,6 +225,11 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); diff --git a/compat/mingw.h b/compat/mingw.h index 416092dd8f..68a2998da9 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -11,6 +11,8 @@ typedef _sigset_t sigset_t; #undef _POSIX_THREAD_SAFE_FUNCTIONS #endif +extern int core_fscache; + extern int mingw_core_config(const char *var, const char *value, void *cb); #define platform_core_config mingw_core_config diff --git a/git-compat-util.h b/git-compat-util.h index 43e401b886..0f1d106d2b 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1244,6 +1244,21 @@ static inline int is_missing_file_error(int errno_) return (errno_ == ENOENT || errno_ == ENOTDIR); } +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + extern int cmd_main(int, const char **); /* diff --git a/preload-index.c b/preload-index.c index c7dc3f2b9f..4ec7feedd2 100644 --- a/preload-index.c +++ b/preload-index.c @@ -119,6 +119,7 @@ void preload_index(struct index_state *index, pthread_mutex_init(&pd.mutex, NULL); } + enable_fscache(1); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; int err; @@ -144,6 +145,7 @@ void preload_index(struct index_state *index, stop_progress(&pd.progress); trace_performance_leave("preload index"); + enable_fscache(0); } int read_index_preload(struct index_state *index, From 75da2506ceb1e261be4b88ae6cdadb1200ddbf1a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 1 Oct 2013 12:51:54 +0200 Subject: [PATCH 097/102] Win32: add a cache below mingw's lstat and dirent implementations Checking the work tree status is quite slow on Windows, due to slow lstat emulation (git calls lstat once for each file in the index). Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. Add an lstat implementation that uses a cache for lstat data. Cache misses read the entire parent directory and add it to the cache. Subsequent lstat calls for the same directory are served directly from the cache. Also implement opendir / readdir / closedir so that they create and use directory listings in the cache. The cache doesn't track file system changes and doesn't plug into any modifying file APIs, so it has to be explicitly enabled for git functions that don't modify the working copy. Note: in an earlier version of this patch, the cache was always active and tracked file system changes via ReadDirectoryChangesW. However, this was much more complex and had negative impact on the performance of modifying git commands such as 'git checkout'. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 444 +++++++++++++++++++++++++++++++++++++++++ compat/win32/fscache.h | 10 + config.mak.uname | 4 +- git-compat-util.h | 2 + 4 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 compat/win32/fscache.c create mode 100644 compat/win32/fscache.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 0000000000..0c5490c8f2 --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,444 @@ +#include "../../cache.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" + +static int initialized; +static volatile long enabled; +static struct hashmap map; +static CRITICAL_SECTION mutex; + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + const char *name; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + struct timespec st_atim; + struct timespec st_mtim; + struct timespec st_ctim; + }; + }; +}; + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(void *unused_cmp_data, + const struct fsentry *fse1, const struct fsentry *fse2, + void *unused_keydata) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && + (res = fsentry_cmp(NULL, fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2, NULL))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return strnicmp(fse1->name, fse2->name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + fse->name = name; + fse->len = len; + hashmap_entry_init(fse, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + char *nm = ((char*) fse) + sizeof(struct fsentry); + memcpy(nm, name, len); + nm[len] = 0; + /* init the rest of the structure */ + fsentry_init(fse, list, nm, len); + fse->next = NULL; + fse->refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->refcnt)); +} + +/* + * Release the reference to an fsentry, frees the memory if its the last ref. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + if (InterlockedDecrement(&(fse->refcnt))) + return; + + while (fse) { + struct fsentry *next = fse->next; + free(fse); + fse = next; + } +} + +/* + * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + */ +static struct fsentry *fseentry_create_entry(struct fsentry *list, + const WIN32_FIND_DATAW *fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + fse = fsentry_alloc(list, buf, len); + + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) + | fdata->nFileSizeLow; + filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->st_atim)); + filetime_to_timespec(&(fdata->ftLastWriteTime), &(fse->st_mtim)); + filetime_to_timespec(&(fdata->ftCreationTime), &(fse->st_ctim)); + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(const struct fsentry *dir) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { + if (errno == ERANGE) + errno = ENAMETOOLONG; + return NULL; + } + + /* append optional '/' and wildcard '*' */ + if (wlen) + pattern[wlen++] = '/'; + pattern[wlen++] = '*'; + pattern[wlen] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(NULL, dir->name, dir->len); + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + do { + *phead = fseentry_create_entry(list, &fdata); + phead = &(*phead)->next; + } while (FindNextFileW(h, &fdata)); + + /* remember result of last FindNextFile, then close find handle */ + err = GetLastError(); + FindClose(h); + + /* return the list if we've got all the files */ + if (err == ERROR_NO_MORE_FILES) + return list; + + /* otherwise free the list and return error */ + fsentry_release(list); + errno = err_win_to_posix(err); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&map, fse); +} + +/* + * Clears the cache. + */ +static void fscache_clear(void) +{ + hashmap_free(&map, 1); + hashmap_init(&map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); +} + +/* + * Checks if the cache is enabled for the given path. + */ +static inline int fscache_enabled(const char *path) +{ + return enabled > 0 && !is_absolute_path(path); +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fsentry *key) +{ + struct fsentry *fse; + + EnterCriticalSection(&mutex); + /* check if entry is in cache */ + fse = hashmap_get(&map, key, NULL); + if (fse) { + fsentry_addref(fse); + LeaveCriticalSection(&mutex); + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get(&map, key->list, NULL); + if (fse) { + LeaveCriticalSection(&mutex); + /* dir entry without file entry -> file doesn't exist */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing (outside mutex!) */ + LeaveCriticalSection(&mutex); + fse = fsentry_create_list(key->list ? key->list : key); + if (!fse) + return NULL; + + EnterCriticalSection(&mutex); + /* add directory listing if it hasn't been added by some other thread */ + if (!hashmap_get(&map, key, NULL)) + fscache_add(fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get(&map, key, NULL); + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + LeaveCriticalSection(&mutex); + return fse; +} + +/* + * Enables or disables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(int enable) +{ + int result; + + if (!initialized) { + /* allow the cache to be disabled entirely */ + if (!core_fscache) + return 0; + + InitializeCriticalSection(&mutex); + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, 0); + initialized = 1; + } + + result = enable ? InterlockedIncrement(&enabled) + : InterlockedDecrement(&enabled); + + if (enable && result == 1) { + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } else if (!enable && !result) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } + return result; +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; + struct fsentry key[2], *fse; + + if (!fscache_enabled(filename)) + return mingw_lstat(filename, st); + + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(key, NULL, filename, dirlen); + fsentry_init(key + 1, key, filename + base, len - base); + fse = fscache_get(key + 1); + if (!fse) + return -1; + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->st_size; + st->st_atim = fse->st_atim; + st->st_mtim = fse->st_mtim; + st->st_ctim = fse->st_ctim; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_name = (char*) next->name; + return &(dir->dirent); +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct fsentry key, *list; + fscache_DIR *dir; + int len; + + if (!fscache_enabled(dirname)) + return dirent_opendir(dirname); + + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key, NULL, dirname, len); + list = fscache_get(&key); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 0000000000..ed518b422d --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,10 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +int fscache_enable(int enable); +#define enable_fscache(x) fscache_enable(x) + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); + +#endif diff --git a/config.mak.uname b/config.mak.uname index de914f4593..6b17c6110d 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -424,7 +424,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS = -nologo -I. -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -ENTRY:wmainCRTStartup -SUBSYSTEM:CONSOLE # invalidcontinue.obj allows Git's source code to close the same file @@ -587,7 +587,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DWIN32 -DPROTECT_NTFS_DEFAULT=1 EXTLIBS += -lws2_32 GITLIBS += git.res diff --git a/git-compat-util.h b/git-compat-util.h index 0f1d106d2b..3ea303cfca 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -207,8 +207,10 @@ #if defined(__MINGW32__) /* pull in Windows compatibility stuff */ #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/msvc.h" +#include "compat/win32/fscache.h" #else #include #include From 2e649bc09994a7df932e81cb22928c5416391974 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 24 Jun 2014 13:22:35 +0200 Subject: [PATCH 098/102] fscache: load directories only once If multiple threads access a directory that is not yet in the cache, the directory will be loaded by each thread. Only one of the results is added to the cache, all others are leaked. This wastes performance and memory. On cache miss, add a future object to the cache to indicate that the directory is currently being loaded. Subsequent threads register themselves with the future object and wait. When the first thread has loaded the directory, it replaces the future object with the result and notifies waiting threads. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 67 +++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 0c5490c8f2..70435df680 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -33,6 +33,8 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; + /* Handle to wait on the loading thread. */ + HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -245,16 +247,43 @@ static inline int fscache_enabled(const char *path) return enabled > 0 && !is_absolute_path(path); } +/* + * Looks up a cache entry, waits if its being loaded by another thread. + * The mutex must be owned by the calling thread. + */ +static struct fsentry *fscache_get_wait(struct fsentry *key) +{ + struct fsentry *fse = hashmap_get(&map, key, NULL); + + /* return if its a 'real' entry (future entries have refcnt == 0) */ + if (!fse || fse->list || fse->refcnt) + return fse; + + /* create an event and link our key to the future entry */ + key->hwait = CreateEvent(NULL, TRUE, FALSE, NULL); + key->next = fse->next; + fse->next = key; + + /* wait for the loading thread to signal us */ + LeaveCriticalSection(&mutex); + WaitForSingleObject(key->hwait, INFINITE); + CloseHandle(key->hwait); + EnterCriticalSection(&mutex); + + /* repeat cache lookup */ + return hashmap_get(&map, key, NULL); +} + /* * Looks up or creates a cache entry for the specified key. */ static struct fsentry *fscache_get(struct fsentry *key) { - struct fsentry *fse; + struct fsentry *fse, *future, *waiter; EnterCriticalSection(&mutex); /* check if entry is in cache */ - fse = hashmap_get(&map, key, NULL); + fse = fscache_get_wait(key); if (fse) { fsentry_addref(fse); LeaveCriticalSection(&mutex); @@ -262,7 +291,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = hashmap_get(&map, key->list, NULL); + fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); /* dir entry without file entry -> file doesn't exist */ @@ -271,16 +300,34 @@ static struct fsentry *fscache_get(struct fsentry *key) } } + /* add future entry to indicate that we're loading it */ + future = key->list ? key->list : key; + future->next = NULL; + future->refcnt = 0; + hashmap_add(&map, future); + /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(key->list ? key->list : key); - if (!fse) - return NULL; - + fse = fsentry_create_list(future); EnterCriticalSection(&mutex); - /* add directory listing if it hasn't been added by some other thread */ - if (!hashmap_get(&map, key, NULL)) - fscache_add(fse); + + /* remove future entry and signal waiting threads */ + hashmap_remove(&map, future, NULL); + waiter = future->next; + while (waiter) { + HANDLE h = waiter->hwait; + waiter = waiter->next; + SetEvent(h); + } + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + LeaveCriticalSection(&mutex); + return NULL; + } + + /* add directory listing to the cache */ + fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) From b95958b32239c26d66efd6b857590e41ec06f869 Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 099/102] pack-objects (mingw): demonstrate a segmentation fault with large deltas There is a problem in the way 9ac3f0e5b3e4 (pack-objects: fix performance issues on packing large deltas, 2018-07-22) initializes that mutex in the `packing_data` struct. The problem manifests in a segmentation fault on Windows, when a mutex (AKA critical section) is accessed without being initialized. (With pthreads, you apparently do not really have to initialize them?) This was reported in https://github.com/git-for-windows/git/issues/1839. Signed-off-by: Johannes Schindelin --- t/t7419-submodule-long-path.sh | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 t/t7419-submodule-long-path.sh diff --git a/t/t7419-submodule-long-path.sh b/t/t7419-submodule-long-path.sh new file mode 100755 index 0000000000..9f9d2ea446 --- /dev/null +++ b/t/t7419-submodule-long-path.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +longpath=${longpath:0:90}; export longpath + +test_expect_failure 'submodule with a long path' ' + git init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin master + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout master && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + git init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git submodule add ../parent foo && + git submodule update --init --recursive && + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin master + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From dd7f4ea7f9fe8df2a086730afdf7b2f973915895 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 28 Jul 2015 21:07:41 +0200 Subject: [PATCH 100/102] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts, added support for chdir(), etc] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- Documentation/config/core.txt | 7 ++ compat/mingw.c | 141 ++++++++++++++++++++++++++------- compat/mingw.h | 75 ++++++++++++++++-- compat/win32/dirent.c | 14 ++-- compat/win32/fscache.c | 17 ++-- t/t2031-checkout-long-paths.sh | 102 ++++++++++++++++++++++++ t/t7419-submodule-long-path.sh | 24 +++--- 7 files changed, 323 insertions(+), 57 deletions(-) create mode 100755 t/t2031-checkout-long-paths.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index f6025349a1..e909b26001 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -554,6 +554,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/compat/mingw.c b/compat/mingw.c index 2a9522b89a..59a452a1e3 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -214,6 +214,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; int core_fscache; +int core_long_paths; int mingw_core_config(const char *var, const char *value, void *cb) { @@ -230,6 +231,11 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); @@ -267,8 +273,8 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; /* read-only files cannot be removed */ @@ -297,7 +303,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -318,8 +324,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -394,9 +400,12 @@ static int set_hidden_flag(const wchar_t *path, int set) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; - if (xutftowcs_path(wpath, path) < 0) + wchar_t wpath[MAX_LONG_PATH]; + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); @@ -469,7 +478,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; open_fn_t open_fn; va_start(args, oflags); @@ -484,7 +493,7 @@ int mingw_open (const char *filename, int oflags, ...) else open_fn = _wopen; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = open_fn(wfilename, oflags, mode); @@ -541,10 +550,10 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -563,10 +572,10 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -620,25 +629,31 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { - wchar_t wdirname[MAX_PATH]; - if (xutftowcs_path(wdirname, dirname) < 0) + int result; + wchar_t wdirname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -686,8 +701,8 @@ static int has_valid_directory_prefix(wchar_t *wfilename) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -858,8 +873,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -920,6 +935,7 @@ char *mingw_mktemp(char *template) wchar_t wtemplate[MAX_PATH]; int offset = 0; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; @@ -1428,6 +1444,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdOutput = winansi_get_osfhandle(fhout); si.hStdError = winansi_get_osfhandle(fherr); + /* executables and the current directory don't support long paths */ if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -1996,8 +2013,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -2306,9 +2324,9 @@ int mingw_raise(int sig) int link(const char *oldpath, const char *newpath) { - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!CreateHardLinkW(wnewpath, woldpath, NULL)) { @@ -2511,6 +2529,68 @@ static void setup_windows_environment(void) setenv("TERM", "cygwin", 1); } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + #if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from @@ -2666,6 +2746,9 @@ int wmain(int argc, const wchar_t **wargv) /* initialize Unicode console */ winansi_init(); + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + /* invoke the real main() using our utf8 version of argv. */ exit_status = main(argc, argv); diff --git a/compat/mingw.h b/compat/mingw.h index 68a2998da9..4aa84ab783 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -12,6 +12,7 @@ typedef _sigset_t sigset_t; #endif extern int core_fscache; +extern int core_long_paths; extern int mingw_core_config(const char *var, const char *value, void *cb); #define platform_core_config mingw_core_config @@ -501,6 +502,42 @@ extern char *mingw_query_user_email(void); #include #endif +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -558,17 +595,45 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) return xutftowcsn(wcs, utf, wcslen, -1); } +/** + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). + */ +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) +{ + int result = xutftowcsn(wcs, utf, wcslen, utflen); + if (result < 0 && errno == ERANGE) + errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); + return result; +} + /** * Simplified file system specific variant of xutftowcsn, assumes output * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... */ static inline int xutftowcs_path(wchar_t *wcs, const char *utf) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); - if (result < 0 && errno == ERANGE) - errno = ENAMETOOLONG; - return result; + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); } /** diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 6b87042182..b3bd8d7af7 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -63,19 +63,23 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, core_long_paths)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 70435df680..4ebd15e426 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -166,23 +166,24 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, */ static struct fsentry *fsentry_create_list(const struct fsentry *dir) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int wlen; struct fsentry *list, **phead; DWORD err; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, + dir->len, MAX_PATH - 2, core_long_paths)) < 0) return NULL; - } - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (wlen) - pattern[wlen++] = '/'; + pattern[wlen++] = '\\'; pattern[wlen++] = '*'; pattern[wlen] = 0; diff --git a/t/t2031-checkout-long-paths.sh b/t/t2031-checkout-long-paths.sh new file mode 100755 index 0000000000..f30f8920ca --- /dev/null +++ b/t/t2031-checkout-long-paths.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test ! -d longpa* +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa*/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >>$(ls longpa*/longtestfile) && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + if test -z "$debug" + then + rm -rf longpa~1 + fi +' + +# check that the template used in the test won't be too long: +abspath="$(pwd)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7419-submodule-long-path.sh b/t/t7419-submodule-long-path.sh index 9f9d2ea446..2ca9794ca5 100755 --- a/t/t7419-submodule-long-path.sh +++ b/t/t7419-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -longpath=${longpath:0:90}; export longpath +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' git init --bare remote && test_create_repo bundle1 && ( @@ -56,7 +61,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' git init --bare super && test_create_repo child && ( @@ -96,6 +101,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From 03e16328d3b937251d7d056dd92c2431fd128648 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 5 Jul 2014 00:00:36 +0200 Subject: [PATCH 101/102] Win32: fix 'lstat("dir/")' with long paths Use a suffciently large buffer to strip the trailing slash. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 59a452a1e3..941ebad6ee 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -773,7 +773,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) static int do_stat_internal(int follow, const char *file_name, struct stat *buf) { int namelen; - char alt_name[PATH_MAX]; + char alt_name[MAX_LONG_PATH]; if (!do_lstat(follow, file_name, buf)) return 0; @@ -789,7 +789,7 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return -1; while (namelen && file_name[namelen-1] == '/') --namelen; - if (!namelen || namelen >= PATH_MAX) + if (!namelen || namelen >= MAX_LONG_PATH) return -1; memcpy(alt_name, file_name, namelen); From 45f1117092efb70bb877fb6f6a26e3962404f55a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 2 Apr 2016 13:05:08 +0200 Subject: [PATCH 102/102] mingw: support spawning programs containing spaces in their names The CreateProcessW() function does not really support spaces in its first argument, lpApplicationName. But it supports passing NULL as lpApplicationName, which makes it figure out the application from the (possibly quoted) first argument of lpCommandLine. Let's use that trick (if we are certain that the first argument matches the executable's path) to support launching programs whose path contains spaces. This fixes https://github.com/git-for-windows/git/issue/692 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 941ebad6ee..3c474ad1ce 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1445,7 +1445,9 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdError = winansi_get_osfhandle(fherr); /* executables and the current directory don't support long paths */ - if (xutftowcs_path(wcmd, cmd) < 0) + if (*argv && !strcmp(cmd, *argv)) + wcmd[0] = L'\0'; + else if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) return -1; @@ -1474,8 +1476,8 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen wenvblk = make_environment_block(deltaenv); memset(&pi, 0, sizeof(pi)); - ret = CreateProcessW(wcmd, wargs, NULL, NULL, TRUE, flags, - wenvblk, dir ? wdir : NULL, &si, &pi); + ret = CreateProcessW(*wcmd ? wcmd : NULL, wargs, NULL, NULL, TRUE, + flags, wenvblk, dir ? wdir : NULL, &si, &pi); free(wenvblk); free(wargs);