chromium: Refactor updater entirely in Nix
The update.sh shell script now is only a call to nix-build, which does all the hard work of updating the Chromium source channels and the plugins. It results in a store path with the new sources.nix that replaces the already existing sources.nix. Along the way, this has led to a quite massive workaround, which abuses MD5 collisions to detect whether an URL is existing, because something like builtins.tryEval (builtins.fetchurl url) unfortunately doesn't work. Further explanations and implementation details are documented in the actual implementation. The drawback of this is that we don't have nice status messages anymore, but on the upside we have a more robust generation of the sources.nix file, which now also should work properly on missing upstream sources/binaries. This also makes it much easier to implement fetching non-GNU/Linux versions of Chromium and we have all values from omahaproxy available as an attribute set (see the csv2nix and channels attributes in the update attribute). Signed-off-by: aszlig <aszlig@redmoonstudios.org>
This commit is contained in:
aszlig
parent
f6c3b13c7c
commit
28b289efa6
2 changed files with 180 additions and 179 deletions
|
|
@ -3,17 +3,24 @@
|
|||
let
|
||||
inherit (import ../../../../../../. {
|
||||
inherit system;
|
||||
}) lib writeText stdenv;
|
||||
}) lib runCommand writeText stdenv curl cacert nix;
|
||||
|
||||
sources = if builtins.pathExists ./sources.nix
|
||||
then import ./sources.nix
|
||||
else null;
|
||||
else {};
|
||||
|
||||
bucketURL = "https://commondatastorage.googleapis.com/"
|
||||
+ "chromium-browser-official";
|
||||
|
||||
mkVerURL = version: "${bucketURL}/chromium-${version}.tar.xz";
|
||||
|
||||
debURL = "https://dl.google.com/linux/chrome/deb/pool/main/g";
|
||||
|
||||
getDebURL = channelName: version: arch: mirror: let
|
||||
packageSuffix = if channelName == "dev" then "unstable" else channelName;
|
||||
packageName = "google-chrome-${packageSuffix}";
|
||||
in "${mirror}/${packageName}/${packageName}_${version}-1_${arch}.deb";
|
||||
|
||||
# Untrusted mirrors, don't try to update from them!
|
||||
debMirrors = [
|
||||
"http://95.31.35.30/chrome/pool/main/g"
|
||||
|
|
@ -21,32 +28,6 @@ let
|
|||
"http://repo.fdzh.org/chrome/deb/pool/main/g"
|
||||
];
|
||||
|
||||
tryChannel = channel: let
|
||||
chan = builtins.getAttr channel sources;
|
||||
in if sources != null then ''
|
||||
oldver="${chan.version}";
|
||||
echo -n "Checking if $oldver ($channel) is up to date..." >&2;
|
||||
if [ "x$(get_newest_ver "$version" "$oldver")" != "x$oldver" ];
|
||||
then
|
||||
echo " no, getting sha256 for new version $version:" >&2;
|
||||
sha256="$(prefetch_sha "$channel" "$version")" || return 1;
|
||||
else
|
||||
echo " yes, keeping old sha256." >&2;
|
||||
sha256="${chan.sha256}";
|
||||
${if (chan ? sha256bin32 && chan ? sha256bin64) then ''
|
||||
sha256="$sha256.${chan.sha256bin32}.${chan.sha256bin64}";
|
||||
'' else ''
|
||||
sha256="$sha256.$(prefetch_deb_sha "$channel" "$version")";
|
||||
''}
|
||||
fi;
|
||||
'' else ''
|
||||
sha256="$(prefetch_sha "$channel" "$version")" || return 1;
|
||||
'';
|
||||
|
||||
caseChannel = channel: ''
|
||||
${channel}) ${tryChannel channel};;
|
||||
'';
|
||||
|
||||
in rec {
|
||||
getChannel = channel: let
|
||||
chanAttrs = builtins.getAttr channel sources;
|
||||
|
|
@ -54,78 +35,196 @@ in rec {
|
|||
inherit (chanAttrs) version;
|
||||
|
||||
main = {
|
||||
url = "${bucketURL}/chromium-${chanAttrs.version}.tar.xz";
|
||||
url = mkVerURL chanAttrs.version;
|
||||
inherit (chanAttrs) sha256;
|
||||
};
|
||||
|
||||
binary = let
|
||||
pname = if channel == "dev"
|
||||
then "google-chrome-unstable"
|
||||
else "google-chrome-${channel}";
|
||||
|
||||
mkUrls = arch: let
|
||||
relpath = "${pname}/${pname}_${chanAttrs.version}-1_${arch}.deb";
|
||||
in map (url: "${url}/${relpath}") ([ debURL ] ++ debMirrors);
|
||||
|
||||
mkURLForMirror = getDebURL channel chanAttrs.version arch;
|
||||
in map mkURLForMirror ([ debURL ] ++ debMirrors);
|
||||
in if stdenv.is64bit && chanAttrs ? sha256bin64 then {
|
||||
urls = mkUrls "amd64";
|
||||
sha256 = chanAttrs.sha256bin64;
|
||||
} else if stdenv.is32bit && chanAttrs ? sha256bin32 then {
|
||||
urls = mkUrls "i386";
|
||||
sha256 = chanAttrs.sha256bin64;
|
||||
sha256 = chanAttrs.sha256bin32;
|
||||
} else throw "No Chrome plugins are available for your architecture.";
|
||||
};
|
||||
|
||||
updateHelpers = writeText "update-helpers.sh" ''
|
||||
update = let
|
||||
csv2nix = name: src: import (runCommand "${name}.nix" {
|
||||
src = builtins.fetchurl src;
|
||||
} ''
|
||||
esc() { echo "\"$(echo "$1" | sed -e 's/"\\$/\\&/')\""; }
|
||||
IFS=, read -r -a headings <<< "$(head -n1 "$src")"
|
||||
echo "[" > "$out"
|
||||
tail -n +2 "$src" | while IFS=, read -r -a line; do
|
||||
echo " {"
|
||||
for idx in "''${!headings[@]}"; do
|
||||
echo " $(esc "''${headings[idx]}") = $(esc ''${line[$idx]});"
|
||||
done
|
||||
echo " }"
|
||||
done >> "$out"
|
||||
echo "]" >> "$out"
|
||||
'');
|
||||
|
||||
prefetch_main_sha()
|
||||
{
|
||||
nix-prefetch-url "${bucketURL}/chromium-$2.tar.xz";
|
||||
}
|
||||
channels = lib.fold lib.recursiveUpdate {} (map (attrs: {
|
||||
${attrs.os}.${attrs.channel} = attrs // {
|
||||
history = let
|
||||
drvName = "omahaproxy-${attrs.os}.${attrs.channel}-info";
|
||||
history = csv2nix drvName "http://omahaproxy.appspot.com/history";
|
||||
cond = h: attrs.os == h.os && attrs.channel == h.channel
|
||||
&& lib.versionOlder h.version attrs.current_version;
|
||||
# Note that this is a *reverse* sort!
|
||||
sorter = a: b: lib.versionOlder b.version a.version;
|
||||
sorted = builtins.sort sorter (lib.filter cond history);
|
||||
in map (lib.flip removeAttrs ["os" "channel"]) sorted;
|
||||
version = attrs.current_version;
|
||||
};
|
||||
}) (csv2nix "omahaproxy-info" "http://omahaproxy.appspot.com/all?csv=1"));
|
||||
|
||||
prefetch_deb_sha()
|
||||
{
|
||||
channel="$1";
|
||||
version="$2";
|
||||
/*
|
||||
XXX: This is essentially the same as:
|
||||
|
||||
case "$1" in
|
||||
dev) pname="google-chrome-unstable";;
|
||||
*) pname="google-chrome-$channel";;
|
||||
esac;
|
||||
builtins.tryEval (builtins.fetchurl url)
|
||||
|
||||
deb_pre="${debURL}/$pname/$pname";
|
||||
... except that tryEval on fetchurl isn't working and doesn't catch errors
|
||||
for fetchurl, so we go for a different approach.
|
||||
|
||||
deb32="$(nix-prefetch-url "''${deb_pre}_$version-1_i386.deb")" || :
|
||||
deb64="$(nix-prefetch-url "''${deb_pre}_$version-1_amd64.deb")" || :
|
||||
We only have fixed-output derivations that can have networking access, so
|
||||
we abuse MD5 and its weaknesses to forge a fixed-output derivation which
|
||||
is not so fixed, because it emits different contents that have the same
|
||||
MD5 hash.
|
||||
|
||||
if [ -n "$deb32" -o -n "$deb64" ]; then
|
||||
echo "$deb32.$deb64";
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
Using this method, we can distinguish whether the URL is available or
|
||||
whether it's not based on the actual content.
|
||||
|
||||
prefetch_sha()
|
||||
{
|
||||
main_sha="$(prefetch_main_sha "$@")" || return 1;
|
||||
deb_sha="$(prefetch_deb_sha "$@")" || return 1;
|
||||
echo "$main_sha.$deb_sha";
|
||||
return 0;
|
||||
}
|
||||
So let's use tryEval as soon as it's working with fetchurl in Nix.
|
||||
*/
|
||||
tryFetch = url: let
|
||||
mkBin = b: runCommand "binary-blurb" { inherit b; } ''
|
||||
h="$(echo "$b" | sed -e ':r;N;$!br;s/[^ \n][^ \n]/\\x&/g;s/[ \n]//g')"
|
||||
echo -ne "$h" > "$out"
|
||||
'';
|
||||
|
||||
get_sha256()
|
||||
{
|
||||
channel="$1";
|
||||
version="$2";
|
||||
# Both MD5 hash collision examples are from:
|
||||
# https://en.wikipedia.org/wiki/MD5#Collision_vulnerabilities
|
||||
hashCollTrue = mkBin ''
|
||||
d131dd02c5e6eec4 693d9a0698aff95c 2fcab58712467eab 4004583eb8fb7f89
|
||||
55ad340609f4b302 83e488832571415a 085125e8f7cdc99f d91dbdf280373c5b
|
||||
d8823e3156348f5b ae6dacd436c919c6 dd53e2b487da03fd 02396306d248cda0
|
||||
e99f33420f577ee8 ce54b67080a80d1e c69821bcb6a88393 96f9652b6ff72a70
|
||||
'';
|
||||
|
||||
case "$channel" in
|
||||
${lib.concatMapStrings caseChannel [ "stable" "dev" "beta" ]}
|
||||
esac;
|
||||
hashCollFalse = mkBin ''
|
||||
d131dd02c5e6eec4 693d9a0698aff95c 2fcab50712467eab 4004583eb8fb7f89
|
||||
55ad340609f4b302 83e4888325f1415a 085125e8f7cdc99f d91dbd7280373c5b
|
||||
d8823e3156348f5b ae6dacd436c919c6 dd53e23487da03fd 02396306d248cda0
|
||||
e99f33420f577ee8 ce54b67080280d1e c69821bcb6a88393 96f965ab6ff72a70
|
||||
'';
|
||||
|
||||
sha_insert "$version" "$sha256";
|
||||
echo "$sha256";
|
||||
return 0;
|
||||
}
|
||||
cacheVal = let
|
||||
urlHash = builtins.hashString "sha256" url;
|
||||
timeSlice = builtins.currentTime / 600;
|
||||
in "${urlHash}-${toString timeSlice}";
|
||||
|
||||
successBin = stdenv.mkDerivation {
|
||||
name = "tryfetch-${cacheVal}";
|
||||
inherit url;
|
||||
|
||||
outputHash = "79054025255fb1a26e4bc422aef54eb4";
|
||||
outputHashMode = "flat";
|
||||
outputHashAlgo = "md5";
|
||||
|
||||
buildInputs = [ curl ];
|
||||
preferLocalBuild = true;
|
||||
|
||||
buildCommand = ''
|
||||
if SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt" \
|
||||
curl -s -L -f -I "$url" > /dev/null; then
|
||||
cat "${hashCollTrue}" > "$out"
|
||||
else
|
||||
cat "${hashCollFalse}" > "$out"
|
||||
fi
|
||||
'';
|
||||
|
||||
impureEnvVars = [
|
||||
"http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
|
||||
];
|
||||
};
|
||||
|
||||
in {
|
||||
success = builtins.readFile successBin == builtins.readFile hashCollTrue;
|
||||
value = builtins.fetchurl url;
|
||||
};
|
||||
|
||||
fetchLatest = channel: let
|
||||
result = tryFetch (mkVerURL channel.version);
|
||||
in if result.success then result.value else fetchLatest (channel // {
|
||||
version = if channel.history != []
|
||||
then (lib.head channel.history).version
|
||||
else throw "Unfortunately there's no older version than " +
|
||||
"${channel.version} available for channel " +
|
||||
"${channel.channel} on ${channel.os}.";
|
||||
history = lib.tail channel.history;
|
||||
});
|
||||
|
||||
getHash = path: import (runCommand "gethash.nix" {
|
||||
inherit path;
|
||||
buildInputs = [ nix ];
|
||||
} ''
|
||||
sha256="$(nix-hash --flat --base32 --type sha256 "$path")"
|
||||
echo "\"$sha256\"" > "$out"
|
||||
'');
|
||||
|
||||
isLatest = channel: version: let
|
||||
ourVersion = sources.${channel}.version or null;
|
||||
in if ourVersion == null then false
|
||||
else lib.versionAtLeast version sources.${channel}.version;
|
||||
|
||||
# We only support GNU/Linux right now.
|
||||
linuxChannels = let
|
||||
genLatest = channelName: channel: let
|
||||
newUpstream = {
|
||||
inherit (channel) version;
|
||||
sha256 = getHash (fetchLatest channel);
|
||||
};
|
||||
keepOld = let
|
||||
oldChannel = sources.${channelName};
|
||||
in {
|
||||
inherit (oldChannel) version sha256;
|
||||
} // lib.optionalAttrs (oldChannel ? sha256bin32) {
|
||||
inherit (oldChannel) sha256bin32;
|
||||
} // lib.optionalAttrs (oldChannel ? sha256bin64) {
|
||||
inherit (oldChannel) sha256bin64;
|
||||
};
|
||||
in if isLatest channelName channel.version then keepOld else newUpstream;
|
||||
in lib.mapAttrs genLatest channels.linux;
|
||||
|
||||
getLinuxFlash = channelName: channel: let
|
||||
inherit (channel) version;
|
||||
fetchArch = arch: tryFetch (getDebURL channelName version arch debURL);
|
||||
packages = lib.genAttrs ["i386" "amd64"] fetchArch;
|
||||
isNew = arch: attr: !(builtins.hasAttr attr channel)
|
||||
&& packages.${arch}.success;
|
||||
in channel // lib.optionalAttrs (isNew "i386" "sha256bin32") {
|
||||
sha256bin32 = getHash (packages.i386.value);
|
||||
} // lib.optionalAttrs (isNew "amd64" "sha256bin64") {
|
||||
sha256bin64 = getHash (packages.amd64.value);
|
||||
};
|
||||
|
||||
newChannels = lib.mapAttrs getLinuxFlash linuxChannels;
|
||||
|
||||
dumpAttrs = indent: attrs: let
|
||||
mkVal = val: if lib.isAttrs val then dumpAttrs (indent + 1) val
|
||||
else "\"${lib.escape ["$" "\\" "\""] (toString val)}\"";
|
||||
mkIndent = level: lib.concatStrings (builtins.genList (_: " ") level);
|
||||
mkAttr = key: val: "${mkIndent (indent + 1)}${key} = ${mkVal val};\n";
|
||||
attrLines = lib.mapAttrsToList mkAttr attrs;
|
||||
in "{\n" + (lib.concatStrings attrLines) + (mkIndent indent) + "}";
|
||||
in writeText "chromium-new-sources.nix" ''
|
||||
# This file is autogenerated from update.sh in the parent directory.
|
||||
${dumpAttrs 0 newChannels}
|
||||
'';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,101 +1,3 @@
|
|||
#!/bin/sh
|
||||
|
||||
channels_url="http://omahaproxy.appspot.com/all?csv=1";
|
||||
history_url="http://omahaproxy.appspot.com/history";
|
||||
bucket_url="http://commondatastorage.googleapis.com/chromium-browser-official/";
|
||||
base_path="$(cd "$(dirname "$0")" && pwd)/source";
|
||||
|
||||
source "$(nix-build --no-out-link "$base_path/update.nix" -A updateHelpers)";
|
||||
|
||||
### poor mans key/value-store :-) ###
|
||||
|
||||
ver_sha_table=""; # list of version:sha256
|
||||
|
||||
sha_insert()
|
||||
{
|
||||
version="$1";
|
||||
sha256="$2";
|
||||
|
||||
ver_sha_table="$ver_sha_table $version:$sha256";
|
||||
}
|
||||
|
||||
get_newest_ver()
|
||||
{
|
||||
versions="$(for v in $@; do echo "$v"; done)";
|
||||
if oldest="$(echo "$versions" | sort -V 2> /dev/null | tail -n1)";
|
||||
then
|
||||
echo "$oldest";
|
||||
else
|
||||
echo "$versions" | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | tail -n1;
|
||||
fi;
|
||||
}
|
||||
|
||||
fetch_filtered_history()
|
||||
{
|
||||
curl -s "$history_url" | sed -nr 's/^'"linux,$1"',([^,]+).*$/\1/p';
|
||||
}
|
||||
|
||||
get_prev_sha256()
|
||||
{
|
||||
channel="$1";
|
||||
current_version="$2";
|
||||
|
||||
for version in $(fetch_filtered_history "$channel");
|
||||
do
|
||||
[ "x$version" = "x$current_version" ] && continue;
|
||||
sha256="$(get_sha256 "$channel" "$version")" || continue;
|
||||
echo "$sha256:$version";
|
||||
return 0;
|
||||
done;
|
||||
}
|
||||
|
||||
get_channel_exprs()
|
||||
{
|
||||
for chline in $1;
|
||||
do
|
||||
channel="${chline%%,*}";
|
||||
version="${chline##*,}";
|
||||
|
||||
sha256="$(get_sha256 "$channel" "$version")";
|
||||
if [ $? -ne 0 ];
|
||||
then
|
||||
echo "Whoops, failed to fetch $version, trying previous" \
|
||||
"versions:" >&2;
|
||||
|
||||
sha_ver="$(get_prev_sha256 "$channel" "$version")";
|
||||
sha256="${sha_ver%:*}";
|
||||
version="${sha_ver#*:}";
|
||||
fi;
|
||||
|
||||
sha_insert "$version" "$sha256";
|
||||
|
||||
main="${sha256%%.*}";
|
||||
deb="${sha256#*.}";
|
||||
deb32="${deb%.*}";
|
||||
deb64="${deb#*.}";
|
||||
|
||||
echo " $channel = {";
|
||||
echo " version = \"$version\";";
|
||||
echo " sha256 = \"$main\";";
|
||||
if [ "x${deb#[a-z0-9]}" != "x$deb" ]; then
|
||||
echo " sha256bin32 = \"$deb32\";";
|
||||
fi;
|
||||
if [ "x${deb#*.[a-z0-9]}" != "x$deb" ]; then
|
||||
echo " sha256bin64 = \"$deb64\";";
|
||||
fi;
|
||||
echo " };";
|
||||
done;
|
||||
}
|
||||
|
||||
cd "$(dirname "$0")";
|
||||
|
||||
omaha="$(curl -s "$channels_url")";
|
||||
versions="$(echo "$omaha" | sed -nr -e 's/^linux,([^,]+,[^,]+).*$/\1/p')";
|
||||
channel_exprs="$(get_channel_exprs "$versions")";
|
||||
|
||||
cat > "$base_path/sources.nix" <<-EOF
|
||||
# This file is autogenerated from update.sh in the parent directory.
|
||||
{
|
||||
$channel_exprs
|
||||
}
|
||||
EOF
|
||||
#!/bin/sh -e
|
||||
sp="$(nix-build -Q --no-out-link source/update.nix -A update)"
|
||||
cat "$sp" > source/sources.nix
|
||||
|
|
|
|||
Loading…
Reference in a new issue