BoomMicrophone/nixpkgs-suyu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #238357 from amjoseph-nixpkgs/review/232576

Browse source 
minimal-bootstrap: make sources a non-tarballs.nixos.org FOD
This commit is contained in:
John Ericson 2023-06-26 17:10:17 -04:00 committed by GitHub
commit d409d42ce7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 123 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix
View file

@ -1,13 +1,14 @@
{
}:
rec {
name = "stage0-posix-${version}-${rev}-source";
# Pinned from https://github.com/oriansj/stage0-posix/commit/3189b5f325b7ef8b88e3edec7c1cde4fce73c76c
version = "unstable-2023-05-02";
rev = "3189b5f325b7ef8b88e3edec7c1cde4fce73c76c";
outputHashAlgo = "sha256";
# This 256 byte seed is the only pre-compiled binary in the bootstrap chain.
# While it is included in the stage0-posix source bundle and is synced with
# stage0-posix updates, we have split it out into its own derivation to highlight
# its unique status as a trusted binary seed.
hex0-seed = import <nix/fetchurl.nix> {
name = "hex0-seed-${version}";
url = "https://github.com/oriansj/bootstrap-seeds/raw/b1263ff14a17835f4d12539226208c426ced4fba/POSIX/x86/hex0-seed";
@ -15,20 +16,91 @@ rec {
executable = true;
};
# Packaged resources required for the first bootstrapping stage.
# Contains source code and 256-byte hex0 binary seed.
#
# We don't have access to utilities such as fetchgit and fetchzip since this
# is this is part of the bootstrap process and would introduce a circular
# dependency. The only tool we have to fetch source trees is `import <nix/fetchurl.nix>`
# with the unpack option, taking a NAR file as input. This requires source
# tarballs to be repackaged.
#
# To build see `make-bootstrap-sources.nix`
src = import <nix/fetchurl.nix> {
inherit name;
url = "https://github.com/emilytrau/bootstrap-tools-nar-mirror/releases/download/2023-05-02/${name}.nar.xz";
hash = "sha256-ZRG0k49MxL1UTZhuMTvPoEprdSpJRNVy8QhLE6k+etg=";
unpack = true;
/*
Since `make-minimal-bootstrap-sources` requires nixpkgs and nix it
will create a circular dependency if it is used in place of the
binary bootstrap-files. To break the circular dependency,
`minimal-bootstrap-sources` extends `make-minimal-bootstrap-sources`
by adding Fixed Output Derivation (FOD) attributes. These cause
the builder to be skipped if the expected output is found (by
its hash) in the store or on a substituter.
# How do I update the hash?
Run the following command:
```
nix hash file $(nix build --print-out-paths -f '<nixpkgs>' make-minimal-bootstrap-sources)
```
# Why do we need this `.nar` archive?
This archive exists only because of a quirk/limitation of Nix: in
restricted mode the builtin fetchers can download only single
files; they have no way to unpack multi-file archives except for
NAR archives:
https://github.com/NixOS/nixpkgs/pull/232576#issuecomment-1592415619
# Why don't we have to upload this to tarballs.nixos.org like the binary bootstrap-files did?
Unlike this archive, the binary bootstrap-files contained binaries,
which meant that we had to:
1. Make sure they came from a trusted builder (Hydra)
2. Keep careful track of exactly what toolchain (i.e. nixpkgs
commit) that builder used to create them.
3. Keep copies of the built binaries, in case the toolchains that
produced them failed to be perfectly deterministic.
The curated archives at tarballs.nixos.org exist in order to
satisfy these requirements.
The second point created a significant burden: since the nixpkgs
toolchain used to build a given copy of the binary bootstrap-files
itself used a *previous* copy of the bootstrap-files, this meant
we had to track the provenance of all bootstrap-files tarballs
ever used, for all eternity. There was no explanation of where
the "original" bootstrap-files came from: turtles all the way
down. In spite of all this effort we still can't be sure of our
ability to reproduce the binary bootstrap-files, since the
compilers that built them don't always produce exactly bit-for-bit
deterministic results.
Since this archive contains no binaries and uses a format (NAR)
specifically designed for bit-exact reproducibility, none of the
requirements above apply to `minimal-bootstrap-sources`.
*/
minimal-bootstrap-sources = derivation {
name = "${name}.nar.xz";
system = builtins.currentSystem;
outputHashMode = "flat";
inherit outputHashAlgo;
outputHash = "sha256-ig988BiRTz92hhZZgKQW1tVPoV4aQ2D69Cq3wHvVgHg=";
# This builder always fails, but fortunately Nix will print the
# "builder", which is really the error message that we want the
# user to see.
builder = ''
#
#
# Neither your store nor your substituters seems to have:
#
# ${name}.nar.xz
#
# Please obtain or create this file, give it exactly the name
# shown above, and then run the following command:
#
# nix-store --add-fixed ${outputHashAlgo} ${name}.nar.xz
#
# You can create this file from an already-bootstrapped nixpkgs
# using the following command:
#
# nix-build '<nixpkgs>' -A make-minimal-bootstrap-sources
#
# Or, if you prefer, you can create this file using only `git`,
# `nix`, and `xz`. For the commands needed in order to do this,
# see `make-bootstrap-sources.nix`.
#
'';
};
}

5
pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/default.nix
View file

@ -3,7 +3,10 @@
}:
lib.makeScope newScope (self: with self; {
inherit (import ./bootstrap-sources.nix) version hex0-seed src;
inherit (self.callPackage ./bootstrap-sources.nix {})
version hex0-seed minimal-bootstrap-sources;
src = minimal-bootstrap-sources;
m2libc = src + "/M2libc";

2
pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/hex0.nix
View file

@ -9,7 +9,7 @@ derivationWithMeta {
pname = "hex0";
builder = hex0-seed;
args = [
"${src}/bootstrap-seeds/POSIX/x86/hex0_x86.hex0"
"${src}/x86/hex0_x86.hex0"
(placeholder "out")
];

2
pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/kaem/minimal.nix
View file

@ -9,7 +9,7 @@ derivationWithMeta {
pname = "kaem-minimal";
builder = hex0;
args = [
"${src}/bootstrap-seeds/POSIX/x86/kaem-minimal.hex0"
"${src}/x86/kaem-minimal.hex0"
(placeholder "out")
];

37
pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix
View file

@ -1,5 +1,4 @@
# Packaged resources required for the first bootstrapping stage.
# Contains source code and 256-byte hex0 binary seed.
# Packaged source files for the first bootstrapping stage.
#
# We don't have access to utilities such as fetchgit and fetchzip since this
# is this is part of the bootstrap process and would introduce a circular
@ -9,23 +8,29 @@
#
# To build:
#
# nix-build pkgs/os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix
# => ./result/stage0-posix-$version-$rev-source.nar.xz
# nix-build '<nixpkgs>' -o sources.nar.xz -A make-minimal-bootstrap-sources
#
{ pkgs ? import ../../../../.. {} }:
{ lib
, fetchFromGitHub
, runCommand
, nix
, xz
}:
let
inherit (pkgs) callPackage runCommand fetchFromGitHub nix xz;
inherit (import ./bootstrap-sources.nix) name rev;
inherit (import ./bootstrap-sources.nix { }) name rev;
src = fetchFromGitHub {
owner = "oriansj";
repo = "stage0-posix";
inherit rev;
sha256 = "sha256-ZRG0k49MxL1UTZhuMTvPoEprdSpJRNVy8QhLE6k+etg=";
sha256 = "sha256-FpMp7z+B3cR3LkQ+PooH/b1/NlxH8NHVJNWifaPWt4U=";
fetchSubmodules = true;
postFetch = ''
# Seed binaries will be fetched separately
echo "Removing seed binaries"
rm -rf $out/bootstrap-seeds/*
# Remove vendored/duplicate M2libc's
echo "Removing duplicate M2libc"
rm -rf \
@ -35,12 +40,20 @@ let
$out/mescc-tools-extra/M2libc
'';
};
in
runCommand name {
runCommand "${name}.nar.xz" {
nativeBuildInputs = [ nix xz ];
passthru = { inherit src; };
meta = with lib; {
description = "Packaged sources for the first bootstrapping stage";
homepage = "https://github.com/oriansj/stage0-posix";
license = licenses.gpl3Plus;
maintainers = teams.minimal-bootstrap.members;
platforms = platforms.all;
};
} ''
mkdir $out
nix-store --dump ${src} | xz -c > "$out/${name}.nar.xz"
nix-store --dump ${src} | xz -c > $out
''

2
pkgs/top-level/all-packages.nix
View file

@ -27559,6 +27559,8 @@ with pkgs;
};
checkMeta = callPackage ../stdenv/generic/check-meta.nix { };
});
minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/bootstrap-sources.nix { };
make-minimal-bootstrap-sources = callPackage ../os-specific/linux/minimal-bootstrap/stage0-posix/make-bootstrap-sources.nix { };
mingetty = callPackage ../os-specific/linux/mingetty { };