Merge pull request #150898 from schnusch/invidious

invidious: unstable-2021-11-08 -> unstable-2022-03-16

pkgs/servers/invidious/default.nix

@@ -1,24 +1,27 @@
-{ lib, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, sqlite, lsquic, nixosTests }:
+{ lib, stdenv, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }:
 let
-  # When updating, always update the following:
-  #  * the git revision
-  #  * the version attribute
-  #  * the source hash (sha256)
-  # If the shards.lock file changed, also the following:
-  #  * shards.nix (by running `crystal2nix` in invidious’ source tree)
-  #  * If the lsquic.cr dependency changed: lsquic in lsquic.nix (version, sha256)
-  #  * If the lsquic version changed: boringssl' in lsquic.nix (version, sha256)
-  rev = "21879da80d2dfa97e789a13b90e82e466c4854e3";
+  # All versions, revisions, and checksums are stored in ./versions.json.
+  # The update process is the following:
+  #   * pick the latest commit
+  #   * update .invidious.rev, .invidious.version, and .invidious.sha256
+  #   * prefetch the videojs dependencies with scripts/fetch-player-dependencies.cr
+  #     and update .videojs.sha256 (they are normally fetched during build
+  #     but nix's sandboxing does not allow that)
+  #   * if shard.lock changed
+  #     * recreate shards.nix by running crystal2nix
+  #     * update lsquic and boringssl if necessarry, lsquic.cr depends on
+  #       the same version of lsquic and lsquic requires the boringssl
+  #       commit mentioned in its README
+  versions = builtins.fromJSON (builtins.readFile ./versions.json);
 in
 crystal.buildCrystalPackage rec {
   pname = "invidious";
-  version = "unstable-2021-11-08";
+  inherit (versions.invidious) version;
 
   src = fetchFromGitHub {
     owner = "iv-org";
     repo = pname;
-    inherit rev;
-    sha256 = "0jvnwjdh2l0hxfvzim00r3zbs528bb93y1nk0bjrbbrcfv5cn5ss";
+    inherit (versions.invidious) rev sha256;
   };
 
   postPatch =
@@ -33,13 +36,15 @@ crystal.buildCrystalPackage rec {
       assetCommitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}'';
     in
     ''
+      for d in ${videojs}/*; do ln -s "$d" assets/videojs; done
+
       # Use the version metadata from the derivation instead of using git at
       # build-time
       substituteInPlace src/invidious.cr \
           --replace ${lib.escapeShellArg branchTemplate} '"master"' \
-          --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 rev}"' \
+          --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' \
           --replace ${lib.escapeShellArg versionTemplate} '"${lib.replaceChars ["-"] ["."] (lib.substring 9 10 version)}"' \
-          --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 rev}"'
+          --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"'
 
       # Patch the assets and locales paths to be absolute
       substituteInPlace src/invidious.cr \
@@ -48,19 +53,22 @@ crystal.buildCrystalPackage rec {
           --replace 'File.read("locales/' 'File.read("${placeholder "out"}/share/invidious/locales/'
 
       # Reference sql initialisation/migration scripts by absolute path
-      substituteInPlace src/invidious/helpers/helpers.cr \
+      substituteInPlace src/invidious/database/base.cr \
             --replace 'config/sql' '${placeholder "out"}/share/invidious/config/sql'
 
-      substituteInPlace src/invidious/users.cr \
+      substituteInPlace src/invidious/user/captcha.cr \
           --replace 'Process.run(%(rsvg-convert' 'Process.run(%(${lib.getBin librsvg}/bin/rsvg-convert'
     '';
 
-  nativeBuildInputs = [ pkg-config ];
+  nativeBuildInputs = [ pkg-config shards ];
   buildInputs = [ libxml2 openssl sqlite ];
 
   format = "crystal";
   shardsFile = ./shards.nix;
-  crystalBinaries.invidious.src = "src/invidious.cr";
+  crystalBinaries.invidious = {
+    src = "src/invidious.cr";
+    options = [ "--release" "--progress" "--verbose" "--no-debug" "-Dskip_videojs_download" ];
+  };
 
   postConfigure = ''
     # lib includes nix store paths which can’t be patched, so the links have to
@@ -88,12 +96,17 @@ crystal.buildCrystalPackage rec {
     INVIDIOUS_CONFIG="database_url: sqlite3:///dev/null" $out/bin/invidious --help
   '';
 
-  passthru.tests = { inherit (nixosTests) invidious; };
+  passthru = {
+    inherit lsquic;
+    tests = { inherit (nixosTests) invidious; };
+    updateScript = ./update.sh;
+  };
 
   meta = with lib; {
     description = "An open source alternative front-end to YouTube";
     homepage = "https://invidious.io/";
     license = licenses.agpl3;
     maintainers = with maintainers; [ infinisil sbruder ];
+    broken = stdenv.isDarwin && stdenv.isAarch64;
   };
 }

pkgs/servers/invidious/lsquic.nix

@@ -1,24 +1,30 @@
 { lib, boringssl, stdenv, fetchgit, fetchFromGitHub, cmake, zlib, perl, libevent }:
 let
+  versions = builtins.fromJSON (builtins.readFile ./versions.json);
+
   # lsquic requires a specific boringssl version (noted in its README)
-  boringssl' = boringssl.overrideAttrs (old: rec {
-    version = "251b5169fd44345f455438312ec4e18ae07fd58c";
+  boringssl' = boringssl.overrideAttrs (old: {
+    version = versions.boringssl.rev;
     src = fetchgit {
       url = "https://boringssl.googlesource.com/boringssl";
-      rev = version;
-      sha256 = "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A=";
+      inherit (versions.boringssl) rev sha256;
     };
+
+    patches = [
+      # Use /etc/ssl/certs/ca-certificates.crt instead of /etc/ssl/cert.pem
+      ./use-etc-ssl-certs.patch
+    ];
   });
 in
 stdenv.mkDerivation rec {
   pname = "lsquic";
-  version = "2.18.1";
+  version = versions.lsquic.version;
 
   src = fetchFromGitHub {
     owner = "litespeedtech";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=";
+    inherit (versions.lsquic) sha256;
     fetchSubmodules = true;
   };
 
@@ -49,6 +55,8 @@ stdenv.mkDerivation rec {
     runHook postInstall
   '';
 
+  passthru.boringssl = boringssl';
+
   meta = with lib; {
     description = "A library for QUIC and HTTP/3 (version for Invidious)";
     homepage = "https://github.com/litespeedtech/lsquic";

pkgs/servers/invidious/shards.nix

@@ -59,10 +59,22 @@
     rev = "v0.4.1";
     sha256 = "1l08cydkdidq9yyil1wl240hvk41iycv04jrg6nx5mkvzw4z1bzg";
   };
+  spectator = {
+    owner = "icy-arctic-fox";
+    repo = "spectator";
+    rev = "v0.10.4";
+    sha256 = "0rcxq2nbslvwrd8m9ajw6dzaw3hagxmkdy9s8p34cgnr4c9dijdq";
+  };
   sqlite3 = {
     owner = "crystal-lang";
     repo = "crystal-sqlite3";
     rev = "v0.18.0";
     sha256 = "03nnvpchhq9f9ywsm3pk2rrj4a3figw7xs96zdziwgr5znkz6x93";
   };
+  ameba = {
+    owner = "crystal-ameba";
+    repo = "ameba";
+    rev = "v0.14.3";
+    sha256 = "1cfr95xi6hsyxw1wlrh571hc775xhwmssk3k14i8b7dgbwfmm5x1";
+  };
 }

pkgs/servers/invidious/update.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p curl crystal crystal2nix jq git moreutils nix nix-prefetch pkg-config
+git_url='https://github.com/iv-org/invidious.git'
+git_branch='master'
+git_dir='/var/tmp/invidious.git'
+pkg='invidious'
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+info() {
+    if [ -t 2 ]; then
+        set -- '\033[32m%s\033[39m\n' "$@"
+    else
+        set -- '%s\n' "$@"
+    fi
+    printf "$@" >&2
+}
+
+json_get() {
+    jq -r "$1" < 'versions.json'
+}
+
+json_set() {
+    jq --arg x "$2" "$1 = \$x" < 'versions.json' | sponge 'versions.json'
+}
+
+old_rev=$(json_get '.invidious.rev')
+old_version=$(json_get '.invidious.version')
+today=$(LANG=C date -u +'%Y-%m-%d')
+
+info "fetching $git_url..."
+if [ ! -d "$git_dir" ]; then
+    git init --initial-branch="$git_branch" "$git_dir"
+    git -C "$git_dir" remote add origin "$git_url"
+fi
+git -C "$git_dir" fetch origin "$git_branch"
+
+# use latest commit before today, we should not call the version *today*
+# because there might still be commits coming
+# use the day of the latest commit we picked as version
+new_rev=$(git -C "$git_dir" log -n 1 --format='format:%H' --before="${today}T00:00:00Z" "origin/$git_branch")
+new_version="unstable-$(git -C "$git_dir" log -n 1 --format='format:%cs' "$new_rev")"
+info "latest commit before $today: $new_rev"
+
+if [ "$new_rev" = "$old_rev" ]; then
+    info "$pkg is up-to-date."
+    exit
+fi
+
+json_set '.invidious.version' "$new_version"
+json_set '.invidious.rev' "$new_rev"
+new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "$pkg")
+json_set '.invidious.sha256' "$new_sha256"
+commit_msg="$pkg: $old_version -> $new_version"
+
+# fetch video.js dependencies
+info "Running scripts/fetch-player-dependencies.cr..."
+git -C "$git_dir" reset --hard "$new_rev"
+(cd "$git_dir" && crystal run scripts/fetch-player-dependencies.cr -- --minified)
+rm -f "$git_dir/assets/videojs/.gitignore"
+videojs_new_sha256=$(nix hash-path --type sha256 --base32 "$git_dir/assets/videojs")
+json_set '.videojs.sha256' "$videojs_new_sha256"
+
+if git -C "$git_dir" diff-tree --quiet "${old_rev}..${new_rev}" -- 'shard.lock'; then
+    info "shard.lock did not change since $old_rev."
+else
+    info "Updating shards.nix..."
+    crystal2nix -- "$git_dir/shard.lock"  # argv's index seems broken
+
+    lsquic_old_version=$(json_get '.lsquic.version')
+    # lsquic.cr's version tracks lsquic's, so lsquic must be updated to the
+    # version in the shards file
+    lsquic_new_version=$(nix eval --raw -f 'shards.nix' lsquic.rev \
+        | sed -e 's/^v//' -e 's/-[0-9]*$//')
+    if [ "$lsquic_old_version" != "$lsquic_new_version" ]; then
+        info "Updating lsquic to $lsquic_new_version..."
+        json_set '.lsquic.version' "$lsquic_new_version"
+        lsquic_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic")
+        json_set '.lsquic.sha256' "$lsquic_new_sha256"
+
+        info "Updating boringssl..."
+        # lsquic specifies the boringssl commit it requires in its README
+        boringssl_new_rev=$(curl -LSsf "https://github.com/litespeedtech/lsquic/raw/v${lsquic_new_version}/README.md" \
+            | grep -Pom1 '(?<=^git checkout ).*')
+        json_set '.boringssl.rev' "$boringssl_new_rev"
+        boringssl_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic.boringssl")
+        json_set '.boringssl.sha256' "$boringssl_new_sha256"
+        commit_msg="$commit_msg
+
+lsquic: $lsquic_old_version -> $lsquic_new_version"
+    fi
+fi
+
+git commit --verbose --message "$commit_msg" -- versions.json shards.nix

pkgs/servers/invidious/use-etc-ssl-certs.patch

@@ -0,0 +1,13 @@
+diff --git a/crypto/x509/x509_def.c b/crypto/x509/x509_def.c
+index d2bc3e5c1..329580075 100644
+--- a/crypto/x509/x509_def.c
++++ b/crypto/x509/x509_def.c
+@@ -67,7 +67,7 @@
+ 
+ #define X509_CERT_AREA          OPENSSLDIR
+ #define X509_CERT_DIR           OPENSSLDIR "/certs"
+-#define X509_CERT_FILE          OPENSSLDIR "/cert.pem"
++#define X509_CERT_FILE          "/etc/ssl/certs/ca-certificates.crt"
+ #define X509_PRIVATE_DIR        OPENSSLDIR "/private"
+ #define X509_CERT_DIR_EVP        "SSL_CERT_DIR"
+ #define X509_CERT_FILE_EVP       "SSL_CERT_FILE"

pkgs/servers/invidious/versions.json

@@ -0,0 +1,18 @@
+{
+  "boringssl": {
+    "rev": "251b5169fd44345f455438312ec4e18ae07fd58c",
+    "sha256": "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A="
+  },
+  "invidious": {
+    "rev": "ed265cfdcd131b9df5398d899cc5d7036a5b7846",
+    "sha256": "0hhnq4s0slwbgxra7gxapl7dcz60a7k71cndi4crqcikmazzac3b",
+    "version": "unstable-2022-03-16"
+  },
+  "lsquic": {
+    "sha256": "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=",
+    "version": "2.18.1"
+  },
+  "videojs": {
+    "sha256": "0b4vxd29kpvy60yhqm376r1872gds17s6wljqw0zlr16j762k50r"
+  }
+}

pkgs/servers/invidious/videojs.nix

@@ -0,0 +1,18 @@
+{ stdenvNoCC, cacert, crystal, openssl, pkg-config, invidious }:
+
+let
+  versions = builtins.fromJSON (builtins.readFile ./versions.json);
+in
+stdenvNoCC.mkDerivation {
+  name = "videojs";
+
+  inherit (invidious) src;
+
+  builder = ./videojs.sh;
+
+  nativeBuildInputs = [ cacert crystal openssl pkg-config ];
+
+  outputHashAlgo = "sha256";
+  outputHashMode = "recursive";
+  outputHash = versions.videojs.sha256;
+}

pkgs/servers/invidious/videojs.sh

@@ -0,0 +1,9 @@
+source $stdenv/setup
+
+unpackPhase
+cd source
+# this helper downloads the videojs files and checks their checksums
+# against videojs-dependencies.yml so it should be pure
+crystal run scripts/fetch-player-dependencies.cr -- --minified
+rm -f assets/videojs/.gitignore
+mv assets/videojs "$out"

pkgs/top-level/all-packages.nix

@@ -6775,6 +6775,8 @@ with pkgs;
   invidious = callPackage ../servers/invidious {
     # needs a specific version of lsquic
     lsquic = callPackage ../servers/invidious/lsquic.nix { };
+    # normally video.js is downloaded at build time
+    videojs = callPackage ../servers/invidious/videojs.nix { };
   };
 
   invoice2data  = callPackage ../tools/text/invoice2data  { };