Merge pull request #296549 from TomaSajt/strip-java-archives-hook

add stripJavaArchivesHook and use treewide

doc/languages-frameworks/java.section.md

@@ -4,12 +4,31 @@ Ant-based Java packages are typically built from source as follows:
 
 ```nix
 stdenv.mkDerivation {
-  name = "...";
+  pname = "...";
+  version = "...";
+
   src = fetchurl { ... };
 
-  nativeBuildInputs = [ jdk ant ];
+  nativeBuildInputs = [
+    ant
+    jdk
+    stripJavaArchivesHook # removes timestamp metadata from jar files
+  ];
 
-  buildPhase = "ant";
+  buildPhase = ''
+    runHook preBuild
+    ant # build the project using ant
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    # copy generated jar file(s) to an appropriate location in $out
+    install -Dm644 build/foo.jar $out/share/java/foo.jar
+
+    runHook postInstall
+  '';
 }
 ```
 
@@ -17,6 +36,10 @@ Note that `jdk` is an alias for the OpenJDK (self-built where available,
 or pre-built via Zulu). Platforms with OpenJDK not (yet) in Nixpkgs
 (`Aarch32`, `Aarch64`) point to the (unfree) `oraclejdk`.
 
+Also note that not using `stripJavaArchivesHook` will likely cause the
+generated `.jar` files to be non-deterministic, which is not optimal.
+Using it, however, does not always guarantee reproducibility.
+
 JAR files that are intended to be used by other packages should be
 installed in `$out/share/java`. JDKs have a stdenv setup hook that add
 any JARs in the `share/java` directories of the build inputs to the

pkgs/applications/blockchains/bisq-desktop/default.nix

@@ -9,7 +9,7 @@
 , dpkg
 , writeScript
 , bash
-, strip-nondeterminism
+, stripJavaArchivesHook
 , tor
 , zip
 , xz
@@ -50,7 +50,7 @@ stdenv.mkDerivation rec {
     dpkg
     imagemagick
     makeWrapper
-    strip-nondeterminism
+    stripJavaArchivesHook
     xz
     zip
     findutils
@@ -89,7 +89,6 @@ stdenv.mkDerivation rec {
     tar --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cJf native/linux/x64/tor.tar.xz tor
     tor_jar_file=$(find ./opt/bisq/lib/app -name "tor-binary-linux64-*.jar")
     zip -r $tor_jar_file native
-    strip-nondeterminism ./opt/bisq/lib/app/*.jar
   '';
 
   installPhase = ''

pkgs/applications/misc/calcoo/default.nix

@@ -2,7 +2,7 @@
 , stdenv
 , fetchzip
 , ant
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 , jdk
 , makeWrapper
 }:
@@ -18,7 +18,7 @@ stdenv.mkDerivation (finalAttrs: {
 
   nativeBuildInputs = [
     ant
-    canonicalize-jars-hook
+    stripJavaArchivesHook
     jdk
     makeWrapper
   ];

pkgs/applications/misc/mkgmap/default.nix

@@ -6,6 +6,7 @@
 , jre
 , ant
 , makeWrapper
+, stripJavaArchivesHook
 , doCheck ? true
 , withExamples ? false
 }:
@@ -30,10 +31,6 @@ stdenv.mkDerivation rec {
   ];
 
   postPatch = with deps; ''
-    # Fix the output jar timestamps for reproducibility
-    substituteInPlace build.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-
     # Manually create version properties file for reproducibility
     mkdir -p build/classes
     cat > build/classes/mkgmap-version.properties << EOF
@@ -61,7 +58,7 @@ stdenv.mkDerivation rec {
     '') testInputs}
   '';
 
-  nativeBuildInputs = [ jdk ant makeWrapper ];
+  nativeBuildInputs = [ jdk ant makeWrapper stripJavaArchivesHook ];
 
   buildPhase = ''
     runHook preBuild

pkgs/applications/misc/mkgmap/splitter/default.nix

@@ -6,6 +6,7 @@
 , jre
 , ant
 , makeWrapper
+, stripJavaArchivesHook
 , doCheck ? true
 }:
 let
@@ -30,10 +31,6 @@ stdenv.mkDerivation rec {
   ];
 
   postPatch = with deps; ''
-    # Fix the output jar timestamps for reproducibility
-    substituteInPlace build.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-
     # Manually create version properties file for reproducibility
     mkdir -p build/classes
     cat > build/classes/splitter-version.properties << EOF
@@ -58,7 +55,7 @@ stdenv.mkDerivation rec {
     '') testInputs}
   '';
 
-  nativeBuildInputs = [ jdk ant makeWrapper ];
+  nativeBuildInputs = [ jdk ant makeWrapper stripJavaArchivesHook ];
 
   buildPhase = ''
     runHook preBuild

pkgs/applications/misc/pattypan/default.nix

@@ -7,7 +7,7 @@
 , wrapGAppsHook
 , makeDesktopItem
 , copyDesktopItems
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -27,7 +27,7 @@ stdenv.mkDerivation (finalAttrs: {
     makeWrapper
     wrapGAppsHook
     copyDesktopItems
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   dontWrapGApps = true;

pkgs/applications/networking/remote/dayon/default.nix

@@ -6,7 +6,7 @@
 , jre
 , makeWrapper
 , copyDesktopItems
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -25,7 +25,7 @@ stdenv.mkDerivation (finalAttrs: {
     jdk
     makeWrapper
     copyDesktopItems
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/applications/office/jameica/default.nix

@@ -4,6 +4,7 @@
 , makeDesktopItem
 , makeWrapper
 , wrapGAppsHook
+, stripJavaArchivesHook
 , ant
 , jdk
 , jre
@@ -46,13 +47,7 @@ stdenv.mkDerivation rec {
     hash = "sha256-MSVSd5DyVL+dcfTDv1M99hxickPwT2Pt6QGNsu6DGZI=";
   };
 
-  postPatch = ''
+  nativeBuildInputs = [ ant jdk wrapGAppsHook makeWrapper stripJavaArchivesHook ];
-    # Fix jar timestamps for reproducibility
-    substituteInPlace build/build.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-  '';
-
-  nativeBuildInputs = [ ant jdk wrapGAppsHook makeWrapper ];
   buildInputs = lib.optionals stdenv.isLinux [ gtk2 glib libXtst ]
     ++ lib.optional stdenv.isDarwin Cocoa;
 

pkgs/applications/science/biology/trimmomatic/default.nix

@@ -5,7 +5,7 @@
 , jdk
 , jre
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -29,7 +29,7 @@ stdenv.mkDerivation (finalAttrs: {
     ant
     jdk
     makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/build-support/java/canonicalize-jar.nix

@@ -1,9 +0,0 @@
-{ substituteAll, unzip, zip }:
-
-substituteAll {
-  name = "canonicalize-jar";
-  src = ./canonicalize-jar.sh;
-
-  unzip = "${unzip}/bin/unzip";
-  zip = "${zip}/bin/zip";
-}

pkgs/build-support/java/canonicalize-jar.sh

@@ -1,29 +0,0 @@
-# Canonicalize the manifest & repack with deterministic timestamps.
-canonicalizeJar() {
-    local input='' outer=''
-    input="$(realpath -sm -- "$1")"
-    outer="$(pwd)"
-    # -qq: even quieter
-    @unzip@ -qq "$input" -d "$input-tmp"
-    canonicalizeJarManifest "$input-tmp/META-INF/MANIFEST.MF"
-    # Sets all timestamps to Jan 1 1980, the earliest mtime zips support.
-    find -- "$input-tmp" -exec touch -t 198001010000.00 {} +
-    rm "$input"
-    pushd "$input-tmp" 2>/dev/null
-    # -q|--quiet, -r|--recurse-paths
-    # -o|--latest-time: canonicalizes overall archive mtime
-    # -X|--no-extra: don't store platform-specific extra file attribute fields
-    @zip@ -qroX "$outer/tmp-out.jar" . 2> /dev/null
-    popd 2>/dev/null
-    rm -rf "$input-tmp"
-    mv "$outer/tmp-out.jar" "$input"
-}
-
-# See also the Java specification's JAR requirements:
-# https://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Notes_on_Manifest_and_Signature_Files
-canonicalizeJarManifest() {
-    local input=''
-    input="$(realpath -sm -- "$1")"
-    (head -n 1 "$input" && tail -n +2 "$input" | sort | grep -v '^\s*$') > "$input-tmp"
-    mv "$input-tmp" "$input"
-}

pkgs/build-support/setup-hooks/canonicalize-jars.sh

@@ -1,16 +0,0 @@
-# This setup hook causes the fixup phase to repack all JAR files in a
-# canonical & deterministic fashion, e.g. resetting mtimes (like with normal
-# store files) and avoiding impure metadata.
-
-fixupOutputHooks+=('if [ -z "$dontCanonicalizeJars" -a -e "$prefix" ]; then canonicalizeJarsIn "$prefix"; fi')
-
-canonicalizeJarsIn() {
-  local dir="$1"
-  echo "canonicalizing jars in $dir"
-  dir="$(realpath -sm -- "$dir")"
-  while IFS= read -rd '' f; do
-    canonicalizeJar "$f"
-  done < <(find -- "$dir" -type f -name '*.jar' -print0)
-}
-
-source @canonicalize_jar@

pkgs/build-support/setup-hooks/strip-java-archives.sh

@@ -0,0 +1,16 @@
+# This setup hook makes the fixup phase to repack all java archives in a
+# deterministic fashion. The most important change being done is the resetting
+# of the modification times of the archive entries
+
+fixupOutputHooks+=('stripJavaArchivesIn $prefix')
+
+stripJavaArchivesIn() {
+    local dir="$1"
+    echo "stripping java archives in $dir"
+    find $dir -type f -regextype posix-egrep -regex ".*\.(jar|war|hpi|apk)$" -print0 |
+    while IFS= read -rd '' f; do
+        echo "stripping java archive $f"
+        strip-nondeterminism --type jar "$f"
+    done
+}
+

pkgs/by-name/jo/jogl/package.nix

@@ -5,6 +5,7 @@
 , jdk11
 , git
 , xmlstarlet
+, stripJavaArchivesHook
 , xcbuild
 , udev
 , xorg
@@ -42,13 +43,6 @@ stdenv.mkDerivation {
     substituteInPlace gluegen/src/java/com/jogamp/common/util/IOUtil.java \
       --replace-fail '#!/bin/true' '#!${coreutils}/bin/true'
   ''
-  # set timestamp of files in jar to a fixed point in time
-  + ''
-    xmlstarlet ed --inplace \
-      --append //jar --type attr -n modificationtime --value 1980-01-01T00:00Z \
-      gluegen/make/{build.xml,gluegen-cpptasks-base.xml} \
-      jogl/make/{build.xml,build-nativewindow.xml,build-jogl.xml}
-  ''
   # prevent looking for native libraries in /usr/lib
   + ''
     substituteInPlace jogl/make/build-*.xml \
@@ -72,6 +66,7 @@ stdenv.mkDerivation {
     jdk11
     git
     xmlstarlet
+    stripJavaArchivesHook
   ] ++ lib.optionals stdenv.isDarwin [
     xcbuild
   ];

pkgs/development/compilers/abcl/default.nix

@@ -6,7 +6,7 @@
 , jdk
 , jre
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 let
@@ -30,7 +30,7 @@ stdenv.mkDerivation (finalAttrs: {
     jdk
     fakeHostname
     makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/development/compilers/jasmin/default.nix

@@ -5,7 +5,7 @@
 , ant
 , jdk8
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 , callPackage
 }:
 
@@ -27,7 +27,7 @@ in stdenv.mkDerivation (finalAttrs: {
     ant
     jdk
     makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/development/libraries/freetts/default.nix

@@ -4,6 +4,7 @@
 , ant
 , jdk8
 , sharutils
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -20,16 +21,11 @@ stdenv.mkDerivation (finalAttrs: {
     ant
     jdk8
     sharutils
+    stripJavaArchivesHook
   ];
 
   sourceRoot = "${finalAttrs.src.name}/freetts-${finalAttrs.version}";
 
-  postPatch = ''
-    # Fix jar timestamps for reproducibility
-    substituteInPlace build.xml demo.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-  '';
-
   buildPhase = ''
     runHook preBuild
 

pkgs/development/libraries/java/cup/default.nix

@@ -4,7 +4,7 @@
 , ant
 , jdk
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -24,7 +24,7 @@ stdenv.mkDerivation (finalAttrs: {
     ant
     jdk
     makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/development/libraries/java/hydra-ant-logger/default.nix

@@ -3,7 +3,7 @@
 , fetchFromGitHub
 , ant
 , jdk
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation {
@@ -20,7 +20,7 @@ stdenv.mkDerivation {
   nativeBuildInputs = [
     ant
     jdk
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/development/libraries/java/swt/default.nix

@@ -1,6 +1,6 @@
 { lib
 , stdenv
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 , fetchzip
 , pkg-config
 , atk
@@ -58,7 +58,7 @@ in stdenv.mkDerivation rec {
   };
 
   nativeBuildInputs = [
-    canonicalize-jars-hook
+    stripJavaArchivesHook
     pkg-config
   ];
   buildInputs = [

pkgs/development/tools/analysis/jdepend/default.nix

@@ -4,7 +4,7 @@
 , ant
 , jdk
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -22,7 +22,7 @@ stdenv.mkDerivation (finalAttrs: {
     ant
     jdk
     makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/development/tools/parsing/javacc/default.nix

@@ -5,7 +5,7 @@
 , jdk
 , jre
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -23,7 +23,7 @@ stdenv.mkDerivation (finalAttrs: {
     ant
     jdk
     makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/games/prismlauncher/default.nix

@@ -1,7 +1,7 @@
 { lib
 , stdenv
 , fetchFromGitHub
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 , cmake
 , cmark
 , Cocoa
@@ -40,7 +40,7 @@ stdenv.mkDerivation (finalAttrs: {
     hash = "sha256-4VsoxZzi/EfEsnDvvwzg2xhj7j5B+k3gvaSqwJFDweE=";
   };
 
-  nativeBuildInputs = [ extra-cmake-modules cmake jdk17 ninja canonicalize-jars-hook ];
+  nativeBuildInputs = [ extra-cmake-modules cmake jdk17 ninja stripJavaArchivesHook ];
   buildInputs =
     [
       qtbase

pkgs/tools/games/jpsxdec/default.nix

@@ -7,7 +7,7 @@
 , makeWrapper
 , makeDesktopItem
 , copyDesktopItems
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -28,7 +28,7 @@ stdenv.mkDerivation (finalAttrs: {
     jdk8
     makeWrapper
     copyDesktopItems
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   buildPhase = ''

pkgs/tools/misc/ili2c/default.nix

@@ -5,7 +5,7 @@
 , jdk8
 , jre8
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:
 
 let
@@ -20,7 +20,7 @@ stdenv.mkDerivation (finalAttrs: {
     ant
     jdk
     makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
   ];
 
   src = fetchFromGitHub {

pkgs/tools/typesetting/fop/default.nix

@@ -5,6 +5,7 @@
 , jdk
 , jre
 , makeWrapper
+, stripJavaArchivesHook
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -16,16 +17,11 @@ stdenv.mkDerivation (finalAttrs: {
     hash = "sha256-b7Av17wu6Ar/npKOiwYqzlvBFSIuXTpqTacM1sxtBvc=";
   };
 
-  postPatch = ''
-    # Fix jar timestamps for reproducibility
-    substituteInPlace fop/build.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-  '';
-
   nativeBuildInputs = [
     ant
     jdk
     makeWrapper
+    stripJavaArchivesHook
   ];
 
   # Note: not sure if this is needed anymore

pkgs/top-level/aliases.nix

@@ -147,6 +147,7 @@ mapAliases ({
   callPackage_i686 = pkgsi686Linux.callPackage;
   cadence = throw "cadence has been removed from nixpkgs, as it was archived upstream"; # Added 2023-10-28
   cask = emacs.pkgs.cask; # Added 2022-11-12
+  canonicalize-jars-hook = stripJavaArchivesHook; # Added 2024-03-17
   cargo-embed = throw "cargo-embed is now part of the probe-rs package"; # Added 2023-07-03
   cargo-espflash = espflash;
   cargo-flash = throw "cargo-flash is now part of the probe-rs package"; # Added 2023-07-03

pkgs/top-level/all-packages.nix

@@ -212,11 +212,10 @@ with pkgs;
     inherit (darwin.apple_sdk.frameworks) Security;
   };
 
-  canonicalize-jar = callPackage ../build-support/java/canonicalize-jar.nix { };
+  stripJavaArchivesHook = makeSetupHook {
-  canonicalize-jars-hook = makeSetupHook {
+    name = "strip-java-archives-hook";
-    name = "canonicalize-jars-hook";
+    propagatedBuildInputs = [ strip-nondeterminism ];
-    substitutions = { canonicalize_jar = canonicalize-jar; };
+  } ../build-support/setup-hooks/strip-java-archives.sh;
-  } ../build-support/setup-hooks/canonicalize-jars.sh;
 
   ensureNewerSourcesHook = { year }: makeSetupHook {
     name = "ensure-newer-sources-hook";