From 90581c9d66173ab1e1b92626a4177620a97f6cf2 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 20 Feb 2025 14:00:48 -0800 Subject: [PATCH] Setup uploading PRs, tags, and branches to IDS --- .github/workflows/build.yml | 9 +++- .github/workflows/release-branches.yml | 20 ++++++++ .github/workflows/release-prs.yml | 30 +++++++++++ .github/workflows/release-tags.yml | 18 +++++++ .github/workflows/upload-release.yml | 71 ++++++++++++++++++++++++++ flake.nix | 31 +++++++++++ 6 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release-branches.yml create mode 100644 .github/workflows/release-prs.yml create mode 100644 .github/workflows/release-tags.yml create mode 100644 .github/workflows/upload-release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e3c9872d..f04126747 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,4 +18,11 @@ jobs: with: flakehub: true - uses: DeterminateSystems/flakehub-cache-action@main - - run: nix build . .#binaryTarball -L + - run: echo "system=$(nix eval --impure --raw --expr 'builtins.currentSystem')" >> "$GITHUB_OUTPUT" + id: system + - run: nix build .# .#binaryTarball --no-link -L + - run: nix build .#binaryTarball --out-link tarball + - uses: actions/upload-artifact@v4 + with: + name: ${{ steps.system.outputs.system }} + path: ./tarball/*.xz diff --git a/.github/workflows/release-branches.yml b/.github/workflows/release-branches.yml new file mode 100644 index 000000000..38e4044ed --- /dev/null +++ b/.github/workflows/release-branches.yml @@ -0,0 +1,20 @@ +name: Release Branch + +concurrency: + group: release + +on: + push: + branches: + # NOTE: make sure any branches here are also valid directory names, + # otherwise creating the directory and uploading to s3 will fail + - "main" + +permissions: + id-token: "write" + contents: "read" + +jobs: + release-branch: + uses: ./.github/workflows/upload-release.yml + secrets: inherit diff --git a/.github/workflows/release-prs.yml b/.github/workflows/release-prs.yml new file mode 100644 index 000000000..818083c68 --- /dev/null +++ b/.github/workflows/release-prs.yml @@ -0,0 +1,30 @@ +name: Release PR + +concurrency: + group: release + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + +permissions: + id-token: "write" + contents: "read" + +jobs: + release-pr: + # Only intra-repo PRs are allowed to have PR artifacts uploaded + # We only want to trigger once the upload once in the case the upload label is added, not when any label is added + if: | + always() && !failure() && !cancelled() + && github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-priv' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'upload to s3') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'upload to s3')) + ) + uses: ./.github/workflows/upload-release.yml + secrets: inherit diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml new file mode 100644 index 000000000..709fbb92a --- /dev/null +++ b/.github/workflows/release-tags.yml @@ -0,0 +1,18 @@ +name: Release Tags + +concurrency: + group: release + +on: + push: + tags: + - "v*.*.*" + +permissions: + contents: write # In order to upload artifacts to GitHub releases + id-token: write # In order to request a JWT for AWS auth + +jobs: + release-tag: + uses: ./.github/workflows/upload-release.yml + secrets: inherit diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml new file mode 100644 index 000000000..bec5816be --- /dev/null +++ b/.github/workflows/upload-release.yml @@ -0,0 +1,71 @@ +name: Upload release + +concurrency: + group: upload-release + +on: + workflow_call: + +permissions: + id-token: "write" + contents: "read" + +jobs: + build-x86_64-linux: + uses: ./.github/workflows/build.yml + with: + os: blacksmith-32vcpu-ubuntu-2204 + build-aarch64-linux: + uses: ./.github/workflows/build.yml + with: + os: blacksmith-32vcpu-ubuntu-2204-arm + build-x86_64-darwin: + uses: ./.github/workflows/build.yml + with: + os: macos-13 + build-aarch64-darwin: + uses: ./.github/workflows/build.yml + with: + os: macos-latest + + release: + runs-on: ubuntu-latest + needs: + - build-x86_64-linux + - build-aarch64-linux + - build-x86_64-darwin + - build-aarch64-darwin + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: "DeterminateSystems/nix-installer-action@main" + with: + determinate: true + + - name: Create artifacts directory + run: mkdir -p ./artifacts + + - name: Fetch artifacts + uses: actions/download-artifact@v4 + with: + path: downloaded + - name: Move downloaded artifacts to artifacts directory + run: | + for dir in ./downloaded/*; do + arch="$(basename "$dir")" + mv "$dir"/*.xz ./artifacts/"${arch}" + done + + - name: Build fallback-paths.nix + run: | + nix build .#fallbackPathsNix --out-link fallback + cat fallback > ./artifacts/fallback-paths.nix + + - uses: DeterminateSystems/push-artifact-ids@main + with: + s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE_ARN }} + bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET_NAME }} + directory: ./artifacts + ids_project_name: determinate-nix + ids_binary_prefix: determinate-nix + skip_acl: true diff --git a/flake.nix b/flake.nix index 29111b453..a499c0dcb 100644 --- a/flake.nix +++ b/flake.nix @@ -294,6 +294,37 @@ nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + + fallbackPathsNix = + let + pkgs = nixpkgsFor.${system}.native; + + # NOTE(cole-h): discard string context so that it doesn't try to build, we just care about the outPaths + closures = forAllSystems (system: builtins.unsafeDiscardStringContext self.packages.${system}.default.outPath); + + closures_json = pkgs.runCommand "versions.json" + { + buildInputs = [ pkgs.jq ]; + passAsFile = [ "json" ]; + json = builtins.toJSON closures; + } '' + cat "$jsonPath" | jq . > $out + ''; + + closures_nix = pkgs.runCommand "versions.nix" + { + buildInputs = [ pkgs.jq ]; + passAsFile = [ "template" ]; + jsonPath = closures_json; + template = '' + builtins.fromJSON('''@closures@''') + ''; + } '' + export closures=$(cat "$jsonPath"); + substituteAll "$templatePath" "$out" + ''; + in + closures_nix; } # We need to flatten recursive attribute sets of derivations to pass `flake check`. //