fetchipfs: init

Fixes #18296

pkgs/applications/misc/hello/default.nix

@@ -1,11 +1,12 @@
-{ stdenv, fetchurl }:
+{ stdenv, fetchipfs }:
 
 stdenv.mkDerivation rec {
   name = "hello-2.10";
 
-  src = fetchurl {
+  src = fetchipfs {
-    url = "mirror://gnu/hello/${name}.tar.gz";
+    url    = "https://ftp.gnu.org/gnu/hello/hello-2.10.tar.gz";
-    sha256 = "0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i";
+    ipfs   = "QmWyj65ak3wd8kG2EvPCXKd6Tij15m4SwJz6g2yG2rQ7w8";
+    sha256 = "1im1gglfm4k10bh4mdaqzmx3lm3kivnsmxrvl6vyvmfqqzljq75l";
   };
 
   doCheck = true;

pkgs/build-support/fetchipfs/builder.sh

@@ -0,0 +1,87 @@
+source $stdenv/setup
+
+# Curl flags to handle redirects, not use EPSV, handle cookies for
+# servers to need them during redirects, and work on SSL without a
+# certificate (this isn't a security problem because we check the
+# cryptographic hash of the output anyway).
+
+set -o noglob
+
+curl="curl            \
+ --location           \
+ --max-redirs 20      \
+ --retry 2            \
+ --disable-epsv       \
+ --cookie-jar cookies \
+ --insecure           \
+ --speed-time 5       \
+ -#                   \
+ --fail               \
+ $curlOpts            \
+ $NIX_CURL_FLAGS"
+
+finish() {
+    runHook postFetch
+    set +o noglob
+    exit 0
+}
+
+ipfs_add() {
+    if curl --retry 0 --head --silent "localhost:5001" > /dev/null; then
+        echo "=IPFS= add $ipfs"
+        tar --owner=root --group=root -cWf "source.tar" $(echo *)
+        res=$(curl -# -F "file=@source.tar" "localhost:5001/api/v0/tar/add" | sed 's/.*"Hash":"\(.*\)".*/\1/')
+        if [ $ipfs != $res ]; then
+            echo "\`ipfs tar add' results in $res when $ipfs is expected"
+            exit 1
+        fi
+        rm "source.tar"
+    fi
+}
+
+echo
+
+mkdir download
+cd download
+
+if curl --retry 0 --head --silent "localhost:5001" > /dev/null; then
+    curlexit=18;
+    echo "=IPFS= get $ipfs"
+    # if we get error code 18, resume partial download
+    while [ $curlexit -eq 18 ]; do
+        # keep this inside an if statement, since on failure it doesn't abort the script
+        if $curl -C - "http://localhost:5001/api/v0/tar/cat?arg=$ipfs" --output "$ipfs.tar"; then
+            unpackFile "$ipfs.tar"
+            rm "$ipfs.tar"
+            set +o noglob
+            mv $(echo *) "$out"
+            finish
+        else
+            curlexit=$?;
+        fi
+    done
+fi
+
+if test -n "$url"; then
+    curlexit=18;
+    echo "Downloading $url"
+    while [ $curlexit -eq 18 ]; do
+        # keep this inside an if statement, since on failure it doesn't abort the script
+        if $curl "$url" -O; then
+            set +o noglob
+            tmpfile=$(echo *)
+            unpackFile $tmpfile
+            rm $tmpfile
+            ipfs_add
+            mv $(echo *) "$out"
+            finish
+        else
+            curlexit=$?;
+        fi
+    done
+fi
+
+echo "error: cannot download $ipfs from ipfs or the given url"
+echo
+set +o noglob
+exit 1

pkgs/build-support/fetchipfs/default.nix

@@ -0,0 +1,52 @@
+{ stdenv
+, curl
+}:
+
+{ ipfs
+, url            ? ""
+, curlOpts       ? ""
+, outputHash     ? ""
+, outputHashAlgo ? ""
+, md5            ? ""
+, sha1           ? ""
+, sha256         ? ""
+, sha512         ? ""
+, meta           ? {}
+, port           ? "8080"
+, postFetch      ? ""
+}:
+
+assert sha512 != "" -> builtins.compareVersions "1.11" builtins.nixVersion <= 0;
+
+let
+
+  hasHash = (outputHash != "" && outputHashAlgo != "")
+    || md5 != "" || sha1 != "" || sha256 != "" || sha512 != "";
+
+in
+
+if (!hasHash) then throw "Specify sha for fetchipfs fixed-output derivation" else stdenv.mkDerivation {
+  name = ipfs;
+  builder = ./builder.sh;
+  buildInputs = [ curl ];
+
+  # New-style output content requirements.
+  outputHashAlgo = if outputHashAlgo != "" then outputHashAlgo else
+      if sha512 != "" then "sha512" else if sha256 != "" then "sha256" else if sha1 != "" then "sha1" else "md5";
+  outputHash = if outputHash != "" then outputHash else
+      if sha512 != "" then sha512 else if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
+
+  outputHashMode = "recursive";
+
+  inherit curlOpts
+          postFetch
+          ipfs
+          url
+          port;
+
+  # Doing the download on a remote machine just duplicates network
+  # traffic, so don't do that.
+  preferLocalBuild = true;
+
+  inherit meta;
+}

pkgs/top-level/all-packages.nix

@@ -177,6 +177,10 @@ with pkgs;
 
   fetchRepoProject = callPackage ../build-support/fetchrepoproject { };
 
+  fetchipfs = import ../build-support/fetchipfs {
+    inherit curl stdenv;
+  };
+
   # fetchurlBoot is used for curl and its dependencies in order to
   # prevent a cyclic dependency (curl depends on curl.tar.bz2,
   # curl.tar.bz2 depends on fetchurl, fetchurl depends on curl).  It