From bf0eb84f8bc54063797f450bdb85d61ce22f2683 Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Mon, 25 Jan 2021 12:02:22 +0100 Subject: Fix static build cache by using cachix It looks like we always hit the caching issue in Cirrus CI described within #8313. A solution around that is to use cachix, which has been pre-populated from my local machine. To push all (runtime and build) dependencies, we can leverage a pre-populated store by: ``` > nix-store -qR --include-outputs $(nix-instantiate nix/default.nix) | cachix push podman ``` The cache can be re-used by everybody to rapidly build static Podman binaries: https://app.cachix.org/cache/podman [NO TESTS NEEDED] Signed-off-by: Sascha Grunert --- .cirrus.yml | 15 +++++++-------- contrib/cirrus/required_host_ports.txt | 1 + contrib/cirrus/runner.sh | 17 ++++++++--------- nix/nixpkgs.json | 8 ++++---- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7c797cdf2..e1ee88da7 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -344,14 +344,13 @@ static_alt_build_task: ALT_NAME: 'Static build' # Do not use 'latest', fixed-version tag for runtime stability. CTR_FQIN: "docker.io/nixos/nix:2.3.6" - # This is critical, it helps to avoid a very lengthy process of - # statically building every dependency needed to build podman. - # Assuming the dependency and build description hasn't changed, - # this cache ensures only the static podman binary is built. - nix_cache: - folder: '/var/cache/nix' - # Cirrus will calculate/use sha of this output as the cache key - fingerprint_script: echo "${IMAGE_SUFFIX}" && cat nix/* + # Authentication token for pushing the build cache to cachix. + # This is critical, it helps to avoid a very lengthy process of + # statically building every dependency needed to build podman. + # Assuming the pinned nix dependencies in nix/nixpkgs.json have not + # changed, this cache will ensure that only the static podman binary is + # built. + CACHIX_AUTH_TOKEN: ENCRYPTED[df0d4d0a67474e8ea49cc503221dcb912b7e2ba45c8ec4bf2e5fd9c49a18ac21c24bacee59b5393355ed9e4358d2baef] setup_script: *setup main_script: *main always: *binary_artifacts diff --git a/contrib/cirrus/required_host_ports.txt b/contrib/cirrus/required_host_ports.txt index 9248e497a..5f066e059 100644 --- a/contrib/cirrus/required_host_ports.txt +++ b/contrib/cirrus/required_host_ports.txt @@ -2,3 +2,4 @@ github.com 22 docker.io 443 quay.io 443 registry.fedoraproject.org 443 +podman.cachix.org 443 diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index 6e6747f28..55dd461e4 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -187,15 +187,14 @@ function _run_altbuild() { req_env_vars CTR_FQIN [[ "$UID" -eq 0 ]] || \ die "Static build must execute nixos container as root on host" - mkdir -p /var/cache/nix - podman run -i --rm -v /var/cache/nix:/mnt/nix:Z \ - $CTR_FQIN cp -rfT /nix /mnt/nix - podman run -i --rm -v /var/cache/nix:/nix:Z \ - -v $PWD:$PWD:Z -w $PWD $CTR_FQIN \ - nix --print-build-logs --option cores 4 --option max-jobs 4 \ - build --file ./nix/ - # result symlink is absolute from container perspective :( - cp /var/cache/$(readlink result)/bin/podman ./ # for cirrus-ci artifact + podman run -i --rm \ + -e CACHIX_AUTH_TOKEN \ + -v $PWD:$PWD:Z -w $PWD $CTR_FQIN sh -c \ + "nix-env -iA cachix -f https://cachix.org/api/v1/install && \ + cachix use podman && \ + nix-build nix && \ + nix-store -qR --include-outputs \$(nix-instantiate nix/default.nix) | grep -v podman | cachix push podman && \ + cp -R result/bin ." rm result # makes cirrus puke ;; *) diff --git a/nix/nixpkgs.json b/nix/nixpkgs.json index d304de536..0cfb251f2 100644 --- a/nix/nixpkgs.json +++ b/nix/nixpkgs.json @@ -1,9 +1,9 @@ { "url": "https://github.com/nixos/nixpkgs", - "rev": "4a75203f0270f96cbc87f5dfa5d5185690237d87", - "date": "2020-12-29T03:18:48+01:00", - "path": "/nix/store/scswsm6r4jnhp9ki0f6s81kpj5x6jkn7-nixpkgs", - "sha256": "0h70fm9aa7s06wkalbadw70z5rscbs3p6nblb47z523nhlzgjxk9", + "rev": "ce7b327a52d1b82f82ae061754545b1c54b06c66", + "date": "2021-01-25T11:28:05+01:00", + "path": "/nix/store/dpsa6a1sy8hwhwjkklc52brs9z1k5fx9-nixpkgs", + "sha256": "1rc4if8nmy9lrig0ddihdwpzg2s8y36vf20hfywb8hph5hpsg4vj", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false -- cgit v1.2.3-54-g00ecf From 6c595dd31155f5af377ece1f106e3bed4e395494 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Thu, 28 Jan 2021 11:47:05 -0500 Subject: Cirrus: Build static podman-remote Prior to this commit, the "Static Build" task only produced a `bin/podman`. Update this to also include a `bin/podman-remote` binary. Update the pr-should-include-tests checker to ignore the `nix` directory, which isn't applicable. Lastly, restore the static build task to 'required' for CI success. Leaving the comment inplace in case it needs to be bypassed in the future on short notice. Signed-off-by: Chris Evich --- .cirrus.yml | 2 +- nix/default.nix | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index e1ee88da7..fbff05bec 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -335,7 +335,7 @@ static_alt_build_task: - build # Community-maintained task, may fail on occasion. If so, uncomment # the next line and file an issue with details about the failure. - allow_failures: $CI == $CI + # allow_failures: $CI == $CI gce_instance: *bigvm env: <<: *stdenvars diff --git a/nix/default.nix b/nix/default.nix index 13b4585ea..7745d8b50 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -49,9 +49,11 @@ let buildPhase = '' patchShebangs . make bin/podman + make bin/podman-remote ''; installPhase = '' install -Dm755 bin/podman $out/bin/podman + install -Dm755 bin/podman-remote $out/bin/podman-remote ''; }; in self -- cgit v1.2.3-54-g00ecf