From 90c4a96c0b55a1a05267e722732e30447b1b5769 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 15:14:58 +0100 Subject: [PATCH 1/6] ci: use the brand-new GitHub Action to download git-sdk-64-minimal In our continuous builds, Windows is the odd cookie that requires a complete development environment to be downloaded because it is not installed by default. Side note: technically, there _is_ a development environment: MSYS2. But it differs from Git for Windows' SDK in subtle points, enough so to prevent Git's test suite from running without failures. Traditionally, we support downloading this environment (which we nicknamed `git-sdk-64-minimal`) via a PowerShell scriptlet that accesses the build artifacts of a dedicated Azure Pipeline (which packages a tiny subset of the full Git for Windows SDK, containing just enough to build Git and run its test suite). This PowerShell script is unfortunately not very robust and sometimes due to network issues. Instead of doing all of this in Git's own `.github/workflows/`, let's offload this logic to the brand-new GitHub Action at https://github.com/marketplace/actions/setup-git-for-windows-sdk This Action not only downloads and extracts git-sdk-64-minimal _outside_ the worktree (making it no longer necessary to meddle with `.gitignore`), it also adds the `bash.exe` to the `PATH` and sets the environment variable `MSYSTEM` (an implementation detail that Git's workflow should never have needed to know about). This allows us to convert all those funny PowerShell tasks that wanted to call git-sdk-64-minimal's `bash.exe`: they all are now regular `bash` scriptlets. This finally lets us get rid of the funny quoting and escaping where we had to pay attention not only to quote and escape in the Bash scriptlets properly, but also to add a second level of escaping (with backslashes for double quotes and backticks for dollar signs) so that PowerShell would not do unintended things. Further, this Action uses a fast caching strategy native to GitHub Actions that is not only very fast, but should accelerate the download across CI runs: git-sdk-64-minimal is usually updated once per 24h, and needs to be cached only once within that period. With this we can drop the homerolled caching where we try to accelerate the test phase by uploading git-sdk-64-minimal as a workflow artifact after using it to build Git, and then download it as workflow artifact in the test phase. Even better: the `vs-test` job no longer needs to depend on the `windows-build` job. The only reason it depended on it was to ensure that the workflow artifact was available. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 103 +++++++------------------------------ 1 file changed, 19 insertions(+), 84 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f2f884b92..d0eaec6abd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,43 +82,18 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - shell: bash - run: | - ## Get artifact - urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=$(curl "$urlbase?definitions=22&statusFilter=completed&resultFilter=succeeded&\$top=1" | - jq -r ".value[] | .id") - download_url="$(curl "$urlbase/$id/artifacts" | - jq -r '.value[] | select(.name == "git-sdk-64-minimal").resource.downloadUrl')" - curl --connect-timeout 10 --retry 5 --retry-delay 0 --retry-max-time 240 \ - -o artifacts.zip "$download_url" - - ## Unzip and remove the artifact - unzip artifacts.zip - rm artifacts.zip + - uses: git-for-windows/setup-git-for-windows-sdk@v0 - name: build - shell: powershell + shell: bash env: HOME: ${{runner.workspace}} - MSYSTEM: MINGW64 NO_PERL: 1 - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude - - ci/make-test-artifacts.sh artifacts - "@ + run: ci/make-test-artifacts.sh artifacts - name: upload build artifacts uses: actions/upload-artifact@v1 with: name: windows-artifacts path: artifacts - - name: upload git-sdk-64-minimal - uses: actions/upload-artifact@v1 - with: - name: git-sdk-64-minimal - path: git-sdk-64-minimal windows-test: runs-on: windows-latest needs: [windows-build] @@ -136,25 +111,14 @@ jobs: - name: extract build artifacts shell: bash run: tar xf artifacts.tar.gz - - name: download git-sdk-64-minimal - uses: actions/download-artifact@v1 - with: - name: git-sdk-64-minimal - path: ${{github.workspace}}/git-sdk-64-minimal/ + - uses: git-for-windows/setup-git-for-windows-sdk@v0 - name: test - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - # Let Git ignore the SDK - printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude - - ci/run-test-slice.sh ${{matrix.nr}} 10 - "@ + shell: bash + run: ci/run-test-slice.sh ${{matrix.nr}} 10 - name: ci/print-test-failures.sh if: failure() - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh + shell: bash + run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v1 @@ -165,27 +129,12 @@ jobs: needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' env: - MSYSTEM: MINGW64 NO_PERL: 1 GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - shell: bash - run: | - ## Get artifact - urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=$(curl "$urlbase?definitions=22&statusFilter=completed&resultFilter=succeeded&\$top=1" | - jq -r ".value[] | .id") - download_url="$(curl "$urlbase/$id/artifacts" | - jq -r '.value[] | select(.name == "git-sdk-64-minimal").resource.downloadUrl')" - curl --connect-timeout 10 --retry 5 --retry-delay 0 --retry-max-time 240 \ - -o artifacts.zip "$download_url" - - ## Unzip and remove the artifact - unzip artifacts.zip - rm artifacts.zip + - uses: git-for-windows/setup-git-for-windows-sdk@v0 - name: download vcpkg artifacts shell: powershell run: | @@ -206,19 +155,17 @@ jobs: shell: bash run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DMSGFMT_EXE=`pwd`/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + -DMSGFMT_EXE=C:/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON - name: MSBuild run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 - name: bundle artifact tar - shell: powershell + shell: bash env: MSVC: 1 VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg run: | - & git-sdk-64-minimal\usr\bin\bash.exe -lc @" - mkdir -p artifacts && - eval \"`$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)\" - "@ + mkdir -p artifacts && + eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - name: upload build artifacts uses: actions/upload-artifact@v1 with: @@ -226,18 +173,14 @@ jobs: path: artifacts vs-test: runs-on: windows-latest - needs: [vs-build, windows-build] + needs: vs-build strategy: fail-fast: false matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - uses: actions/download-artifact@v1 - with: - name: git-sdk-64-minimal - path: ${{github.workspace}}/git-sdk-64-minimal/ + - uses: git-for-windows/setup-git-for-windows-sdk@v0 - name: download build artifacts uses: actions/download-artifact@v1 with: @@ -247,23 +190,15 @@ jobs: shell: bash run: tar xf artifacts.tar.gz - name: test - shell: powershell + shell: bash env: - MSYSTEM: MINGW64 NO_SVN_TESTS: 1 GIT_TEST_SKIP_REBASE_P: 1 - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - # Let Git ignore the SDK and the test-cache - printf '%s\n' /git-sdk-64-minimal/ /test-cache/ >>.git/info/exclude - - ci/run-test-slice.sh ${{matrix.nr}} 10 - "@ + run: ci/run-test-slice.sh ${{matrix.nr}} 10 - name: ci/print-test-failures.sh if: failure() - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh + shell: bash + run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v1 From dc244b26e5de3e1f28470c964ef648ca41d29bf4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 15:35:08 +0100 Subject: [PATCH 2/6] ci (vs-build): use `cmd` to copy the DLLs, not `powershell` We use a `.bat` script to copy the DLLs in the `vs-build` job, and those type of scripts are native to CMD, not to PowerShell. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d0eaec6abd..c9dafa34c6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,10 +147,8 @@ jobs: - name: add msbuild to PATH uses: microsoft/setup-msbuild@v1 - name: copy dlls to root - shell: powershell - run: | - & compat\vcbuild\vcpkg_copy_dlls.bat release - if (!$?) { exit(1) } + shell: cmd + run: compat\vcbuild\vcpkg_copy_dlls.bat release - name: generate Visual Studio solution shell: bash run: | From a81ddc41d6525ef04a0510a4c790948f648481c0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 15:58:45 +0100 Subject: [PATCH 3/6] ci: upgrade to using actions/{up,down}load-artifacts v2 The GitHub Actions to upload/download workflow artifacts saw a major upgrade since Git's GitHub workflow was established. Let's use it. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9dafa34c6..60e366564a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,7 +90,7 @@ jobs: NO_PERL: 1 run: ci/make-test-artifacts.sh artifacts - name: upload build artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: windows-artifacts path: artifacts @@ -104,7 +104,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: download build artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: windows-artifacts path: ${{github.workspace}} @@ -121,7 +121,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-windows path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -165,7 +165,7 @@ jobs: mkdir -p artifacts && eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - name: upload build artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: vs-artifacts path: artifacts @@ -180,7 +180,7 @@ jobs: - uses: actions/checkout@v1 - uses: git-for-windows/setup-git-for-windows-sdk@v0 - name: download build artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: vs-artifacts path: ${{github.workspace}} @@ -199,7 +199,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-windows path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -237,7 +237,7 @@ jobs: if: failure() - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -264,7 +264,7 @@ jobs: if: failure() - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} From 8c0de1a6bfc31c2a4f924269294378bfc9615538 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 16:00:59 +0100 Subject: [PATCH 4/6] ci(windows): transfer the tracked files to the test jobs Git's test suite is excruciatingly slow on Windows, mainly due to the fact that it executes a lot of shell script code, and that's simply not native to Windows. To help with that, we established the pattern where the artifacts are first built in one job, and then multiple test jobs run in parallel using the artifacts built in the first job. We take pains in transferring only the build outputs, and letting `actions/checkout` fill in the rest of the files. One major downside of that strategy is that the test jobs might fail to check out the intended revision (e.g. because the branch has been updated while the build was running, as is frequently the case with the `seen` branch). Let's transfer also the files tracked by Git, and skip the checkout step in the test jobs. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 60e366564a..c921413bdc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,7 +89,9 @@ jobs: HOME: ${{runner.workspace}} NO_PERL: 1 run: ci/make-test-artifacts.sh artifacts - - name: upload build artifacts + - name: zip up tracked files + run: git archive -o artifacts/tracked.tar.gz HEAD + - name: upload tracked files and build artifacts uses: actions/upload-artifact@v2 with: name: windows-artifacts @@ -102,15 +104,14 @@ jobs: matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - uses: actions/checkout@v1 - - name: download build artifacts + - name: download tracked files and build artifacts uses: actions/download-artifact@v2 with: name: windows-artifacts path: ${{github.workspace}} - - name: extract build artifacts + - name: extract tracked files and build artifacts shell: bash - run: tar xf artifacts.tar.gz + run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz - uses: git-for-windows/setup-git-for-windows-sdk@v0 - name: test shell: bash @@ -164,7 +165,9 @@ jobs: run: | mkdir -p artifacts && eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - - name: upload build artifacts + - name: zip up tracked files + run: git archive -o artifacts/tracked.tar.gz HEAD + - name: upload tracked files and build artifacts uses: actions/upload-artifact@v2 with: name: vs-artifacts @@ -177,16 +180,15 @@ jobs: matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - uses: actions/checkout@v1 - uses: git-for-windows/setup-git-for-windows-sdk@v0 - - name: download build artifacts + - name: download tracked files and build artifacts uses: actions/download-artifact@v2 with: name: vs-artifacts path: ${{github.workspace}} - - name: extract build artifacts + - name: extract tracked files and build artifacts shell: bash - run: tar xf artifacts.tar.gz + run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz - name: test shell: bash env: From 7491e752df48ae6c7dd881dc6cac1641ecfa53fd Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 11 Feb 2021 15:32:02 +0100 Subject: [PATCH 5/6] ci(vs-build): build with NO_GETTEXT We already build Git for Windows with `NO_GETTEXT` when compiling with GCC. Let's do the same with Visual C, too. Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c921413bdc..857c0b4dd5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,7 +154,7 @@ jobs: shell: bash run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DMSGFMT_EXE=C:/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON - name: MSBuild run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 - name: bundle artifact tar From fdf70a8bd3e1ed3d55b6cd2121d35ba417a54f16 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 18 Feb 2021 16:15:49 +0100 Subject: [PATCH 6/6] ci: accelerate the checkout By upgrading from v1 to v2 of `actions/checkout`, we avoid fetching all the tags and the entire history: v2 only fetches one revision by default. This should make things a lot faster. Note that `actions/checkout@v2` seems to be incompatible with running in containers: https://github.com/actions/checkout/issues/151. Therefore, we stick with v1 there. Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 857c0b4dd5..3b0d6618e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,7 +81,7 @@ jobs: if: needs.ci-config.outputs.enabled == 'yes' runs-on: windows-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: git-for-windows/setup-git-for-windows-sdk@v0 - name: build shell: bash @@ -134,7 +134,7 @@ jobs: GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: git-for-windows/setup-git-for-windows-sdk@v0 - name: download vcpkg artifacts shell: powershell @@ -232,7 +232,7 @@ jobs: jobname: ${{matrix.vector.jobname}} runs-on: ${{matrix.vector.pool}} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - run: ci/print-test-failures.sh @@ -277,7 +277,7 @@ jobs: jobname: StaticAnalysis runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh documentation: @@ -287,6 +287,6 @@ jobs: jobname: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/test-documentation.sh