Merge pull request #2586 from dscho/build-git-artifacts

Add a GitHub workflow to generate Git for Windows' installer, portable Git, MinGit, etc

.github/workflows/git-artifacts.yml

@@ -0,0 +1,500 @@
+name: git-artifacts
+
+on:
+  # This workflow can be triggered manually in the Actions tab, see
+  # https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/
+  workflow_dispatch:
+    inputs:
+      build_only:
+        description: 'Optionally restrict what artifacts to build'
+      ref:
+        description: 'Optionally override which branch to build'
+      repository:
+        description: 'Optionally override from where to fetch the specified ref'
+
+env:
+  GPG_OPTIONS: "--batch --yes --no-tty --list-options no-show-photos --verify-options no-show-photos --pinentry-mode loopback"
+  HOME: "${{github.workspace}}\\home"
+  MSYSTEM: MINGW64
+  USERPROFILE: "${{github.workspace}}\\home"
+  BUILD_ONLY: "${{github.event.inputs.build_only}}"
+  REPOSITORY: "${{github.event.inputs.repository}}"
+  REF: "${{github.event.inputs.ref}}"
+
+jobs:
+  bundle-artifacts:
+    runs-on: windows-latest
+    outputs:
+      latest-sdk64-extra-build-id: ${{ steps.determine-latest-sdk64-extra-build-id.outputs.id }}
+    steps:
+      - name: Configure user
+        shell: bash
+        run:
+          USER_NAME="${{github.actor}}" &&
+          USER_EMAIL="${{github.actor}}@users.noreply.github.com" &&
+          mkdir "$HOME" &&
+          git config --global user.name "$USER_NAME" &&
+          git config --global user.email "$USER_EMAIL" &&
+          echo "PACKAGER=$USER_NAME <$USER_EMAIL>" >>$GITHUB_ENV
+      - name: Determine latest git-sdk-64-extra-artifacts build ID
+        id: determine-latest-sdk64-extra-build-id
+        shell: bash
+        run: |
+          urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
+          id=$(curl "$urlbase?definitions=29&statusFilter=completed&resultFilter=succeeded&\$top=1" |
+            jq -r '.value[0].id')
+
+          echo "Latest ID is ${id}"
+          echo "::set-output name=id::$id"
+      - name: Cache git-sdk-64-build-installers
+        id: cache-sdk-build-installers
+        uses: actions/cache@v2
+        with:
+          path: git-sdk-64-build-installers
+          key: build-installers-64-${{ steps.determine-latest-sdk64-extra-build-id.outputs.id }}
+      - name: Download git-sdk-64-build-installers
+        if: steps.cache-sdk-build-installers.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          # Use Git Bash to download and unpack the artifact
+
+          ## Get artifact
+          urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
+          id=${{ steps.determine-latest-sdk64-extra-build-id.outputs.id }}
+          download_url=$(curl "$urlbase/$id/artifacts" |
+            jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl')
+
+          curl -o artifacts.zip "$download_url"
+
+          ## Unpack artifact
+          unzip artifacts.zip
+      - name: Clone build-extra
+        shell: bash
+        run: |
+          d=git-sdk-64-build-installers/usr/src/build-extra &&
+          if test ! -d $d/.git
+          then
+            git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d
+          else
+            git -C $d fetch https://github.com/git-for-windows/build-extra main &&
+            git -C $d switch -C main FETCH_HEAD
+          fi
+      - name: Prepare home directory for GPG signing
+        if: env.GPGKEY != ''
+        shell: bash
+        run: |
+          echo '${{secrets.PRIVGPGKEY}}' | tr % '\n' | gpg $GPG_OPTIONS --import &&
+          mkdir -p home &&
+          git config --global gpg.program "$PWD/git-sdk-64-build-installers/usr/src/build-extra/gnupg-with-gpgkey.sh" &&
+          info="$(gpg --list-keys --with-colons "${GPGKEY%% *}" | cut -d : -f 1,10 | sed -n '/^uid/{s|uid:||p;q}')" &&
+          git config --global user.name "${info% <*}" &&
+          git config --global user.email "<${info#*<}"
+        env:
+          GPGKEY: ${{secrets.GPGKEY}}
+      - name: Generate bundle artifacts
+        env:
+          GPGKEY: ${{secrets.GPGKEY}}
+        shell: powershell
+        run: |
+          & .\git-sdk-64-build-installers\git-cmd.exe --command=usr\bin\bash.exe -lc @"
+            printf '#!/bin/sh\n\nexec /mingw64/bin/git.exe "`$@"\n' >/usr/bin/git &&
+            mkdir -p bundle-artifacts &&
+
+            { test -n \"`$REPOSITORY\" || REPOSITORY='${{github.repository}}'; } &&
+            { test -n \"`$REF\" || REF='${{github.ref}}'; } &&
+            git -c init.defaultBranch=main init --bare &&
+            git remote add -f origin https://github.com/git-for-windows/git &&
+            git fetch \"https://github.com/`$REPOSITORY\" \"`$REF:`$REF\" &&
+
+            tag_name=\"`$(git describe --match 'v[0-9]*' FETCH_HEAD)-`$(date +%Y%m%d%H%M%S)\" &&
+            echo \"prerelease-`${tag_name#v}\" >bundle-artifacts/ver &&
+            echo \"`${tag_name#v}\" >bundle-artifacts/display_version &&
+            echo \"`$tag_name\" >bundle-artifacts/next_version &&
+            git tag `$(test -z \"`$GPGKEY\" || echo \" -s\") -m \"Snapshot build\" \"`$tag_name\" FETCH_HEAD &&
+            git bundle create bundle-artifacts/git.bundle origin/main..\"`$tag_name\" &&
+
+            sh -x /usr/src/build-extra/please.sh mention feature \"Snapshot of `$(git show -s  --pretty='tformat:%h (%s, %ad)' --date=short FETCH_HEAD)\" &&
+            git -C /usr/src/build-extra bundle create \"`$PWD/bundle-artifacts/build-extra.bundle\" origin/main..main
+          "@
+      - name: Clean up temporary files
+        if: always()
+        shell: bash
+        run: rm -rf home
+      - name: 'Publish Pipeline Artifact: bundle-artifacts'
+        uses: actions/upload-artifact@v1
+        with:
+          name: bundle-artifacts
+          path: bundle-artifacts
+  pkg:
+    runs-on: windows-latest
+    needs: bundle-artifacts
+    outputs:
+      latest-sdk64-extra-build-id: ${{ needs.bundle-artifacts.outputs.latest-sdk64-extra-build-id }}
+    strategy:
+      matrix:
+        arch:
+          - name: x86_64
+            bitness: 64
+            bin: /amd64
+          - name: i686
+            bitness: 32
+            bin: ''
+    steps:
+      - name: Configure user
+        shell: bash
+        run:
+          USER_NAME="${{github.actor}}" &&
+          USER_EMAIL="${{github.actor}}@users.noreply.github.com" &&
+          mkdir "$HOME" &&
+          git config --global user.name "$USER_NAME" &&
+          git config --global user.email "$USER_EMAIL" &&
+          echo "PACKAGER=$USER_NAME <$USER_EMAIL>" >>$GITHUB_ENV
+      - name: Cache git-sdk-64-build-installers
+        id: cache-sdk-build-installers
+        uses: actions/cache@v2
+        with:
+          path: git-sdk-64-build-installers
+          key: build-installers-64-${{ needs.bundle-artifacts.outputs.latest-sdk64-extra-build-id }}
+      - name: Download git-sdk-64-build-installers
+        if: steps.cache-sdk-build-installers.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          # Use Git Bash to download and unpack the artifact
+
+          ## Get artifact
+          urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
+          id=${{ needs.pkg.outputs.latest-sdk64-extra-build-id }}
+          download_url=$(curl "$urlbase/$id/artifacts" |
+            jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl')
+
+          curl -o artifacts.zip "$download_url"
+
+          ## Unpack artifact
+          unzip artifacts.zip
+      - name: Download bundle-artifacts
+        uses: actions/download-artifact@v1
+        with:
+          name: bundle-artifacts
+          path: bundle-artifacts
+      - name: Clone and update build-extra
+        shell: bash
+        run: |
+          d=git-sdk-64-build-installers/usr/src/build-extra &&
+          if test ! -d $d/.git
+          then
+            git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d
+          else
+            git -C $d fetch https://github.com/git-for-windows/build-extra main &&
+            git -C $d switch -C main FETCH_HEAD
+          fi &&
+          git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main
+      - name: Check out git/git
+        shell: bash
+        run: |
+          git -c init.defaultBranch=main init &&
+          git remote add -f origin https://github.com/git-for-windows/git &&
+          git fetch --tags bundle-artifacts/git.bundle $(cat bundle-artifacts/next_version) &&
+          git reset --hard $(cat bundle-artifacts/next_version)
+      - name: Prepare home directory for code-signing
+        env:
+          CODESIGN_P12: ${{secrets.CODESIGN_P12}}
+          CODESIGN_PASS: ${{secrets.CODESIGN_PASS}}
+        if: env.CODESIGN_P12 != '' && env.CODESIGN_PASS != ''
+        shell: bash
+        run: |
+          cd home &&
+          mkdir -p .sig &&
+          echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >.sig/codesign.p12 &&
+          echo -n "$CODESIGN_PASS" >.sig/codesign.pass
+          git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"'
+      - name: Prepare home directory for GPG signing
+        if: env.GPGKEY != ''
+        shell: bash
+        run: |
+          echo '${{secrets.PRIVGPGKEY}}' | tr % '\n' | gpg $GPG_OPTIONS --import &&
+          info="$(gpg --list-keys --with-colons "${GPGKEY%% *}" | cut -d : -f 1,10 | sed -n '/^uid/{s|uid:||p;q}')" &&
+          git config --global user.name "${info% <*}" &&
+          git config --global user.email "<${info#*<}"
+        env:
+          GPGKEY: ${{secrets.GPGKEY}}
+      - name: Build mingw-w64-${{matrix.arch.name}}-git
+        env:
+          GPGKEY: "${{secrets.GPGKEY}}"
+        shell: powershell
+        run: |
+          & git-sdk-64-build-installers\usr\bin\sh.exe -lc @"
+            set -x
+            # Let `cv2pdb` find the DLLs
+            PATH=\"`$PATH:/C/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin${{matrix.arch.bin}}\"
+            type -p mspdb140.dll || exit 1
+            sh -x /usr/src/build-extra/please.sh build-mingw-w64-git --only-${{matrix.arch.bitness}}-bit --build-src-pkg -o artifacts HEAD &&
+            cp bundle-artifacts/ver artifacts/ &&
+            if test -n \"`$GPGKEY\"
+            then
+              for tar in artifacts/*.tar*
+              do
+                /usr/src/build-extra/gnupg-with-gpgkey.sh --detach-sign --no-armor `$tar
+              done
+            fi &&
+
+            b=`$PWD/artifacts &&
+            version=`$(cat bundle-artifacts/next_version) &&
+            (cd /usr/src/MINGW-packages/mingw-w64-git &&
+            cp PKGBUILD.`$version PKGBUILD &&
+            git commit -s -m \"mingw-w64-git: new version (`$version)\" PKGBUILD &&
+            git bundle create \"`$b\"/MINGW-packages.bundle origin/main..main)
+          "@
+      - name: Clean up temporary files
+        if: always()
+        shell: bash
+        run: rm -rf home
+      - name: Publish mingw-w64-${{matrix.arch.name}}-git
+        uses: actions/upload-artifact@v1
+        with:
+          name: pkg-${{matrix.arch.name}}
+          path: artifacts
+  artifacts:
+    runs-on: windows-latest
+    needs: pkg
+    strategy:
+      matrix:
+        artifact:
+          - name: installer
+            fileprefix: Git
+            fileextension: exe
+          - name: portable
+            fileprefix: PortableGit
+            fileextension: exe
+          - name: archive
+            fileprefix: Git
+            fileextension: tar.bz2
+          - name: mingit
+            fileprefix: MinGit
+            fileextension: zip
+          - name: mingit-busybox
+            fileprefix: MinGit
+            fileextension: zip
+        arch:
+          - name: x86_64
+            bitness: 64
+          - name: i686
+            bitness: 32
+      fail-fast: false
+    env:
+      MSYSTEM: MINGW${{matrix.arch.bitness}}
+    steps:
+      - name: Determine whether this job should be skipped
+        shell: bash
+        run: |
+          case " $BUILD_ONLY " in
+          '  ') ;; # not set; build all
+          *" ${{matrix.artifact.name}} "*|*" ${{matrix.artifact.name}}-${{matrix.arch.name}} "*) ;; # build this artifact
+          *) echo "SKIP=true" >>$GITHUB_ENV;;
+          esac
+      - name: Download pkg-${{matrix.arch.name}}
+        if: env.SKIP != 'true'
+        uses: actions/download-artifact@v1
+        with:
+          name: pkg-${{matrix.arch.name}}
+          path: pkg-${{matrix.arch.name}}
+      - name: Download bundle-artifacts
+        if: env.SKIP != 'true'
+        uses: actions/download-artifact@v1
+        with:
+          name: bundle-artifacts
+          path: bundle-artifacts
+      - name: Cache git-sdk-64-build-installers
+        if: env.SKIP != 'true' && matrix.arch.bitness == '64'
+        id: cache-sdk64-build-installers
+        uses: actions/cache@v2
+        with:
+          path: git-sdk-64-build-installers
+          key: build-installers-64-${{ needs.pkg.outputs.latest-sdk64-extra-build-id }}
+      - name: Download git-sdk-64-build-installers
+        if: env.SKIP != 'true' && matrix.arch.bitness == '64' && steps.cache-sdk64-build-installers.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          # Use Git Bash to download and unpack the artifact
+
+          ## Get artifact
+          urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
+          id=${{ needs.pkg.outputs.latest-sdk64-extra-build-id }}
+          download_url="$(curl "$urlbase/$id/artifacts" |
+            jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl')"
+
+          curl -o artifacts.zip "$download_url"
+
+          ## Unpack artifact
+          unzip artifacts.zip
+      - name: Determine latest git-sdk-32-extra-artifacts build ID
+        if: env.SKIP != 'true' && matrix.arch.bitness == '32'
+        id: determine-latest-sdk32-extra-build-id
+        shell: bash
+        run: |
+          urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
+          id=$(curl "$urlbase?definitions=30&statusFilter=completed&resultFilter=succeeded&\$top=1" |
+            jq -r '.value[0].id')
+
+          echo "Latest ID is ${id}"
+          echo "::set-output name=id::$id"
+      - name: Cache git-sdk-32-build-installers
+        if: env.SKIP != 'true' && matrix.arch.bitness == '32'
+        id: cache-sdk32-build-installers
+        uses: actions/cache@v2
+        with:
+          path: git-sdk-32-build-installers
+          key: build-installers-32-${{ steps.determine-latest-sdk32-extra-build-id.outputs.id }}
+      - name: Download git-sdk-32-build-installers
+        if: env.SKIP != 'true' && matrix.arch.bitness == '32' && steps.cache-sdk32-build-installers.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          # Use Git Bash to download and unpack the artifact
+
+          ## Get artifact
+          urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
+          id=${{ steps.determine-latest-sdk32-extra-build-id.outputs.id }}
+          download_url=$(curl "$urlbase/$id/artifacts" |
+            jq -r '.value[] | select(.name == "git-sdk-32-build-installers").resource.downloadUrl')
+
+          curl -o artifacts.zip "$download_url"
+
+          ## Unpack artifact
+          unzip artifacts.zip
+      - name: Clone and update build-extra
+        if: env.SKIP != 'true'
+        shell: bash
+        run: |
+          d=git-sdk-${{matrix.arch.bitness}}-build-installers/usr/src/build-extra &&
+          if test ! -d $d/.git
+          then
+            git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d
+          else
+            git -C $d fetch https://github.com/git-for-windows/build-extra main &&
+            git -C $d switch -C main FETCH_HEAD
+          fi &&
+          git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main
+      - name: Prepare home directory for code-signing
+        env:
+          CODESIGN_P12: ${{secrets.CODESIGN_P12}}
+          CODESIGN_PASS: ${{secrets.CODESIGN_PASS}}
+        if: env.SKIP != 'true' && (matrix.artifact.name == 'installer' || matrix.artifact.name == 'portable') && env.CODESIGN_P12 != '' && env.CODESIGN_PASS != ''
+        shell: bash
+        run: |
+          mkdir -p home/.sig &&
+          echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >home/.sig/codesign.p12 &&
+          echo -n "$CODESIGN_PASS" >home/.sig/codesign.pass &&
+          git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"'
+      - name: Build ${{matrix.arch.bitness}}-bit ${{matrix.artifact.name}}
+        if: env.SKIP != 'true'
+        shell: powershell
+        run: |
+          & .\git-sdk-${{matrix.arch.bitness}}-build-installers\usr\bin\bash.exe -lc @"
+            set -x
+            /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-${{matrix.arch.name}}/ver) -o artifacts --${{matrix.artifact.name}} --pkg=pkg-${{matrix.arch.name}}/mingw-w64-${{matrix.arch.name}}-git-[0-9]*.tar.xz --pkg=pkg-${{matrix.arch.name}}/mingw-w64-${{matrix.arch.name}}-git-doc-html-[0-9]*.tar.xz &&
+            if test portable = '${{matrix.artifact.name}}' && test -n \"`$(git config alias.signtool)\"
+            then
+              git signtool artifacts/PortableGit-*.exe
+            fi &&
+            openssl dgst -sha256 artifacts/${{matrix.artifact.fileprefix}}-*.${{matrix.artifact.fileextension}} | sed \"s/.* //\" >artifacts/sha-256.txt
+          "@
+      - name: Copy package-versions and pdbs
+        if: env.SKIP != 'true' && matrix.artifact.name == 'installer'
+        shell: powershell
+        run: |
+          & .\git-sdk-${{matrix.arch.bitness}}-build-installers\usr\bin\bash.exe -lc @"
+            cp /usr/src/build-extra/installer/package-versions.txt artifacts/ &&
+
+            a=`$PWD/artifacts &&
+            p=`$PWD/pkg-${{matrix.arch.name}} &&
+            (cd /usr/src/build-extra &&
+            mkdir -p cached-source-packages &&
+            cp \"`$p\"/*-pdb* cached-source-packages/ &&
+            GIT_CONFIG_PARAMETERS=\"'windows.sdk${{matrix.arch.bitness}}.path='\" ./please.sh bundle_pdbs --arch=${{matrix.arch.name}} --directory=\"`$a\" installer/package-versions.txt)
+          "@
+      - name: Clean up temporary files
+        if: always() && env.SKIP != 'true'
+        shell: bash
+        run: rm -rf home
+      - name: Publish ${{matrix.artifact.name}}-${{matrix.arch.name}}
+        if: env.SKIP != 'true'
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{matrix.artifact.name}}-${{matrix.arch.name}}
+          path: artifacts
+  nuget:
+    runs-on: windows-latest
+    needs: pkg
+    steps:
+      - name: Determine whether this job should be skipped
+        shell: bash
+        run: |
+          case " $BUILD_ONLY " in
+          '  ') ;; # not set; build all
+          *" nuget "*) ;; # build this artifact
+          *) echo "SKIP=true" >>$GITHUB_ENV;;
+          esac
+      - name: Download pkg-x86_64
+        if: env.SKIP != 'true'
+        uses: actions/download-artifact@v1
+        with:
+          name: pkg-x86_64
+          path: pkg-x86_64
+      - name: Download bundle-artifacts
+        if: env.SKIP != 'true'
+        uses: actions/download-artifact@v1
+        with:
+          name: bundle-artifacts
+          path: bundle-artifacts
+      - name: Cache git-sdk-64-build-installers
+        if: env.SKIP != 'true'
+        id: cache-sdk-build-installers
+        uses: actions/cache@v2
+        with:
+          path: git-sdk-64-build-installers
+          key: build-installers-64-${{ needs.pkg.outputs.latest-sdk64-extra-build-id }}
+      - name: Download git-sdk-64-build-installers
+        if: env.SKIP != 'true' && steps.cache-sdk-build-installers.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          # Use Git Bash to download and unpack the artifact
+
+          ## Get artifact
+          urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
+          id=${{ needs.pkg.outputs.latest-sdk64-extra-build-id }}
+          download_url=$(curl "$urlbase/$id/artifacts" |
+            jq -r '.value[] | select(.name == "git-sdk-64-build-installers").resource.downloadUrl')
+
+          curl -o artifacts.zip "$download_url"
+
+          ## Unpack artifact
+          unzip artifacts.zip
+      - name: Clone and update build-extra
+        if: env.SKIP != 'true'
+        shell: bash
+        run: |
+          d=git-sdk-64-build-installers/usr/src/build-extra &&
+          if test ! -d $d/.git
+          then
+            git clone --single-branch -b main https://github.com/git-for-windows/build-extra $d
+          else
+            git -C $d fetch https://github.com/git-for-windows/build-extra main &&
+            git -C $d switch -C main FETCH_HEAD
+          fi &&
+          git -C $d pull "$PWD"/bundle-artifacts/build-extra.bundle main
+      - uses: nuget/setup-nuget@v1
+        if: env.SKIP != 'true'
+      - name: Build 64-bit NuGet packages
+        if: env.SKIP != 'true'
+        shell: powershell
+        run: |
+          & .\git-sdk-64-build-installers\usr\bin\bash.exe -lc @"
+            /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-x86_64/ver) -o artifacts --nuget --pkg=pkg-x86_64/mingw-w64-x86_64-git-[0-9]*.tar.xz --pkg=pkg-x86_64/mingw-w64-x86_64-git-doc-html-[0-9]*.tar.xz &&
+            /usr/src/build-extra/please.sh make_installers_from_mingw_w64_git --version=`$(cat pkg-x86_64/ver) -o artifacts --nuget-mingit &&
+            openssl dgst -sha256 artifacts/Git*.nupkg | sed \"s/.* //\" >artifacts/sha-256.txt
+          "@
+      - name: Publish nuget-x86_64
+        if: env.SKIP != 'true'
+        uses: actions/upload-artifact@v1
+        with:
+          name: nuget-x86_64
+          path: artifacts