diff --git a/pkgs/applications/networking/browsers/chromium/source/update.nix b/pkgs/applications/networking/browsers/chromium/source/update.nix index 77ebb9488e2e..3c489b7523e6 100644 --- a/pkgs/applications/networking/browsers/chromium/source/update.nix +++ b/pkgs/applications/networking/browsers/chromium/source/update.nix @@ -3,17 +3,24 @@ let inherit (import ../../../../../../. { inherit system; - }) lib writeText stdenv; + }) lib runCommand writeText stdenv curl cacert nix; sources = if builtins.pathExists ./sources.nix then import ./sources.nix - else null; + else {}; bucketURL = "https://commondatastorage.googleapis.com/" + "chromium-browser-official"; + mkVerURL = version: "${bucketURL}/chromium-${version}.tar.xz"; + debURL = "https://dl.google.com/linux/chrome/deb/pool/main/g"; + getDebURL = channelName: version: arch: mirror: let + packageSuffix = if channelName == "dev" then "unstable" else channelName; + packageName = "google-chrome-${packageSuffix}"; + in "${mirror}/${packageName}/${packageName}_${version}-1_${arch}.deb"; + # Untrusted mirrors, don't try to update from them! debMirrors = [ "http://95.31.35.30/chrome/pool/main/g" @@ -21,32 +28,6 @@ let "http://repo.fdzh.org/chrome/deb/pool/main/g" ]; - tryChannel = channel: let - chan = builtins.getAttr channel sources; - in if sources != null then '' - oldver="${chan.version}"; - echo -n "Checking if $oldver ($channel) is up to date..." >&2; - if [ "x$(get_newest_ver "$version" "$oldver")" != "x$oldver" ]; - then - echo " no, getting sha256 for new version $version:" >&2; - sha256="$(prefetch_sha "$channel" "$version")" || return 1; - else - echo " yes, keeping old sha256." >&2; - sha256="${chan.sha256}"; - ${if (chan ? sha256bin32 && chan ? sha256bin64) then '' - sha256="$sha256.${chan.sha256bin32}.${chan.sha256bin64}"; - '' else '' - sha256="$sha256.$(prefetch_deb_sha "$channel" "$version")"; - ''} - fi; - '' else '' - sha256="$(prefetch_sha "$channel" "$version")" || return 1; - ''; - - caseChannel = channel: '' - ${channel}) ${tryChannel channel};; - ''; - in rec { getChannel = channel: let chanAttrs = builtins.getAttr channel sources; @@ -54,78 +35,196 @@ in rec { inherit (chanAttrs) version; main = { - url = "${bucketURL}/chromium-${chanAttrs.version}.tar.xz"; + url = mkVerURL chanAttrs.version; inherit (chanAttrs) sha256; }; binary = let - pname = if channel == "dev" - then "google-chrome-unstable" - else "google-chrome-${channel}"; - mkUrls = arch: let - relpath = "${pname}/${pname}_${chanAttrs.version}-1_${arch}.deb"; - in map (url: "${url}/${relpath}") ([ debURL ] ++ debMirrors); - + mkURLForMirror = getDebURL channel chanAttrs.version arch; + in map mkURLForMirror ([ debURL ] ++ debMirrors); in if stdenv.is64bit && chanAttrs ? sha256bin64 then { urls = mkUrls "amd64"; sha256 = chanAttrs.sha256bin64; } else if stdenv.is32bit && chanAttrs ? sha256bin32 then { urls = mkUrls "i386"; - sha256 = chanAttrs.sha256bin64; + sha256 = chanAttrs.sha256bin32; } else throw "No Chrome plugins are available for your architecture."; }; - updateHelpers = writeText "update-helpers.sh" '' + update = let + csv2nix = name: src: import (runCommand "${name}.nix" { + src = builtins.fetchurl src; + } '' + esc() { echo "\"$(echo "$1" | sed -e 's/"\\$/\\&/')\""; } + IFS=, read -r -a headings <<< "$(head -n1 "$src")" + echo "[" > "$out" + tail -n +2 "$src" | while IFS=, read -r -a line; do + echo " {" + for idx in "''${!headings[@]}"; do + echo " $(esc "''${headings[idx]}") = $(esc ''${line[$idx]});" + done + echo " }" + done >> "$out" + echo "]" >> "$out" + ''); - prefetch_main_sha() - { - nix-prefetch-url "${bucketURL}/chromium-$2.tar.xz"; - } + channels = lib.fold lib.recursiveUpdate {} (map (attrs: { + ${attrs.os}.${attrs.channel} = attrs // { + history = let + drvName = "omahaproxy-${attrs.os}.${attrs.channel}-info"; + history = csv2nix drvName "http://omahaproxy.appspot.com/history"; + cond = h: attrs.os == h.os && attrs.channel == h.channel + && lib.versionOlder h.version attrs.current_version; + # Note that this is a *reverse* sort! + sorter = a: b: lib.versionOlder b.version a.version; + sorted = builtins.sort sorter (lib.filter cond history); + in map (lib.flip removeAttrs ["os" "channel"]) sorted; + version = attrs.current_version; + }; + }) (csv2nix "omahaproxy-info" "http://omahaproxy.appspot.com/all?csv=1")); - prefetch_deb_sha() - { - channel="$1"; - version="$2"; + /* + XXX: This is essentially the same as: - case "$1" in - dev) pname="google-chrome-unstable";; - *) pname="google-chrome-$channel";; - esac; + builtins.tryEval (builtins.fetchurl url) - deb_pre="${debURL}/$pname/$pname"; + ... except that tryEval on fetchurl isn't working and doesn't catch errors + for fetchurl, so we go for a different approach. - deb32="$(nix-prefetch-url "''${deb_pre}_$version-1_i386.deb")" || : - deb64="$(nix-prefetch-url "''${deb_pre}_$version-1_amd64.deb")" || : + We only have fixed-output derivations that can have networking access, so + we abuse MD5 and its weaknesses to forge a fixed-output derivation which + is not so fixed, because it emits different contents that have the same + MD5 hash. - if [ -n "$deb32" -o -n "$deb64" ]; then - echo "$deb32.$deb64"; - return 0 - else - return 1 - fi - } + Using this method, we can distinguish whether the URL is available or + whether it's not based on the actual content. - prefetch_sha() - { - main_sha="$(prefetch_main_sha "$@")" || return 1; - deb_sha="$(prefetch_deb_sha "$@")" || return 1; - echo "$main_sha.$deb_sha"; - return 0; - } + So let's use tryEval as soon as it's working with fetchurl in Nix. + */ + tryFetch = url: let + mkBin = b: runCommand "binary-blurb" { inherit b; } '' + h="$(echo "$b" | sed -e ':r;N;$!br;s/[^ \n][^ \n]/\\x&/g;s/[ \n]//g')" + echo -ne "$h" > "$out" + ''; - get_sha256() - { - channel="$1"; - version="$2"; + # Both MD5 hash collision examples are from: + # https://en.wikipedia.org/wiki/MD5#Collision_vulnerabilities + hashCollTrue = mkBin '' + d131dd02c5e6eec4 693d9a0698aff95c 2fcab58712467eab 4004583eb8fb7f89 + 55ad340609f4b302 83e488832571415a 085125e8f7cdc99f d91dbdf280373c5b + d8823e3156348f5b ae6dacd436c919c6 dd53e2b487da03fd 02396306d248cda0 + e99f33420f577ee8 ce54b67080a80d1e c69821bcb6a88393 96f9652b6ff72a70 + ''; - case "$channel" in - ${lib.concatMapStrings caseChannel [ "stable" "dev" "beta" ]} - esac; + hashCollFalse = mkBin '' + d131dd02c5e6eec4 693d9a0698aff95c 2fcab50712467eab 4004583eb8fb7f89 + 55ad340609f4b302 83e4888325f1415a 085125e8f7cdc99f d91dbd7280373c5b + d8823e3156348f5b ae6dacd436c919c6 dd53e23487da03fd 02396306d248cda0 + e99f33420f577ee8 ce54b67080280d1e c69821bcb6a88393 96f965ab6ff72a70 + ''; - sha_insert "$version" "$sha256"; - echo "$sha256"; - return 0; - } + cacheVal = let + urlHash = builtins.hashString "sha256" url; + timeSlice = builtins.currentTime / 600; + in "${urlHash}-${toString timeSlice}"; + + successBin = stdenv.mkDerivation { + name = "tryfetch-${cacheVal}"; + inherit url; + + outputHash = "79054025255fb1a26e4bc422aef54eb4"; + outputHashMode = "flat"; + outputHashAlgo = "md5"; + + buildInputs = [ curl ]; + preferLocalBuild = true; + + buildCommand = '' + if SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt" \ + curl -s -L -f -I "$url" > /dev/null; then + cat "${hashCollTrue}" > "$out" + else + cat "${hashCollFalse}" > "$out" + fi + ''; + + impureEnvVars = [ + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + }; + + in { + success = builtins.readFile successBin == builtins.readFile hashCollTrue; + value = builtins.fetchurl url; + }; + + fetchLatest = channel: let + result = tryFetch (mkVerURL channel.version); + in if result.success then result.value else fetchLatest (channel // { + version = if channel.history != [] + then (lib.head channel.history).version + else throw "Unfortunately there's no older version than " + + "${channel.version} available for channel " + + "${channel.channel} on ${channel.os}."; + history = lib.tail channel.history; + }); + + getHash = path: import (runCommand "gethash.nix" { + inherit path; + buildInputs = [ nix ]; + } '' + sha256="$(nix-hash --flat --base32 --type sha256 "$path")" + echo "\"$sha256\"" > "$out" + ''); + + isLatest = channel: version: let + ourVersion = sources.${channel}.version or null; + in if ourVersion == null then false + else lib.versionAtLeast version sources.${channel}.version; + + # We only support GNU/Linux right now. + linuxChannels = let + genLatest = channelName: channel: let + newUpstream = { + inherit (channel) version; + sha256 = getHash (fetchLatest channel); + }; + keepOld = let + oldChannel = sources.${channelName}; + in { + inherit (oldChannel) version sha256; + } // lib.optionalAttrs (oldChannel ? sha256bin32) { + inherit (oldChannel) sha256bin32; + } // lib.optionalAttrs (oldChannel ? sha256bin64) { + inherit (oldChannel) sha256bin64; + }; + in if isLatest channelName channel.version then keepOld else newUpstream; + in lib.mapAttrs genLatest channels.linux; + + getLinuxFlash = channelName: channel: let + inherit (channel) version; + fetchArch = arch: tryFetch (getDebURL channelName version arch debURL); + packages = lib.genAttrs ["i386" "amd64"] fetchArch; + isNew = arch: attr: !(builtins.hasAttr attr channel) + && packages.${arch}.success; + in channel // lib.optionalAttrs (isNew "i386" "sha256bin32") { + sha256bin32 = getHash (packages.i386.value); + } // lib.optionalAttrs (isNew "amd64" "sha256bin64") { + sha256bin64 = getHash (packages.amd64.value); + }; + + newChannels = lib.mapAttrs getLinuxFlash linuxChannels; + + dumpAttrs = indent: attrs: let + mkVal = val: if lib.isAttrs val then dumpAttrs (indent + 1) val + else "\"${lib.escape ["$" "\\" "\""] (toString val)}\""; + mkIndent = level: lib.concatStrings (builtins.genList (_: " ") level); + mkAttr = key: val: "${mkIndent (indent + 1)}${key} = ${mkVal val};\n"; + attrLines = lib.mapAttrsToList mkAttr attrs; + in "{\n" + (lib.concatStrings attrLines) + (mkIndent indent) + "}"; + in writeText "chromium-new-sources.nix" '' + # This file is autogenerated from update.sh in the parent directory. + ${dumpAttrs 0 newChannels} ''; } diff --git a/pkgs/applications/networking/browsers/chromium/update.sh b/pkgs/applications/networking/browsers/chromium/update.sh index 975e141e6685..05cc671d31ce 100755 --- a/pkgs/applications/networking/browsers/chromium/update.sh +++ b/pkgs/applications/networking/browsers/chromium/update.sh @@ -1,101 +1,3 @@ -#!/bin/sh - -channels_url="http://omahaproxy.appspot.com/all?csv=1"; -history_url="http://omahaproxy.appspot.com/history"; -bucket_url="http://commondatastorage.googleapis.com/chromium-browser-official/"; -base_path="$(cd "$(dirname "$0")" && pwd)/source"; - -source "$(nix-build --no-out-link "$base_path/update.nix" -A updateHelpers)"; - -### poor mans key/value-store :-) ### - -ver_sha_table=""; # list of version:sha256 - -sha_insert() -{ - version="$1"; - sha256="$2"; - - ver_sha_table="$ver_sha_table $version:$sha256"; -} - -get_newest_ver() -{ - versions="$(for v in $@; do echo "$v"; done)"; - if oldest="$(echo "$versions" | sort -V 2> /dev/null | tail -n1)"; - then - echo "$oldest"; - else - echo "$versions" | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | tail -n1; - fi; -} - -fetch_filtered_history() -{ - curl -s "$history_url" | sed -nr 's/^'"linux,$1"',([^,]+).*$/\1/p'; -} - -get_prev_sha256() -{ - channel="$1"; - current_version="$2"; - - for version in $(fetch_filtered_history "$channel"); - do - [ "x$version" = "x$current_version" ] && continue; - sha256="$(get_sha256 "$channel" "$version")" || continue; - echo "$sha256:$version"; - return 0; - done; -} - -get_channel_exprs() -{ - for chline in $1; - do - channel="${chline%%,*}"; - version="${chline##*,}"; - - sha256="$(get_sha256 "$channel" "$version")"; - if [ $? -ne 0 ]; - then - echo "Whoops, failed to fetch $version, trying previous" \ - "versions:" >&2; - - sha_ver="$(get_prev_sha256 "$channel" "$version")"; - sha256="${sha_ver%:*}"; - version="${sha_ver#*:}"; - fi; - - sha_insert "$version" "$sha256"; - - main="${sha256%%.*}"; - deb="${sha256#*.}"; - deb32="${deb%.*}"; - deb64="${deb#*.}"; - - echo " $channel = {"; - echo " version = \"$version\";"; - echo " sha256 = \"$main\";"; - if [ "x${deb#[a-z0-9]}" != "x$deb" ]; then - echo " sha256bin32 = \"$deb32\";"; - fi; - if [ "x${deb#*.[a-z0-9]}" != "x$deb" ]; then - echo " sha256bin64 = \"$deb64\";"; - fi; - echo " };"; - done; -} - -cd "$(dirname "$0")"; - -omaha="$(curl -s "$channels_url")"; -versions="$(echo "$omaha" | sed -nr -e 's/^linux,([^,]+,[^,]+).*$/\1/p')"; -channel_exprs="$(get_channel_exprs "$versions")"; - -cat > "$base_path/sources.nix" <<-EOF -# This file is autogenerated from update.sh in the parent directory. -{ -$channel_exprs -} -EOF +#!/bin/sh -e +sp="$(nix-build -Q --no-out-link source/update.nix -A update)" +cat "$sp" > source/sources.nix