From d03e4ffdbf9849f7e928c7a95be76aac69780c55 Mon Sep 17 00:00:00 2001 From: aszlig Date: Mon, 19 Nov 2018 17:18:27 +0100 Subject: [PATCH 1/7] autoPatchelfHook: Make easier to run autoPatchelf The autoPatchelf main function which is run against all of the outputs was pretty much tailored towards this specific setup-hook and was relying on $prefix to be set globally. So if you wanted to run autoPatchelf manually - let's say during buildPhase - you would have needed to run it like this: prefix=/some/directory autoPatchelf This is now more intuitive and all you need to do is run the following: autoPatchelf /some/directory Signed-off-by: aszlig --- pkgs/build-support/setup-hooks/auto-patchelf.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index d1ae317ff9a4..59e9a933211f 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -153,7 +153,7 @@ autoPatchelf() { # Add all shared objects of the current output path to the start of # cachedDependencies so that it's choosen first in findDependency. cachedDependencies+=( - $(find "$prefix" \! -type d \( -name '*.so' -o -name '*.so.*' \)) + $(find "$@" \! -type d \( -name '*.so' -o -name '*.so.*' \)) ) local elffile @@ -169,7 +169,7 @@ autoPatchelf() { LANG=C readelf -l "$file" | grep -q "^ *INTERP\\>" || continue fi autoPatchelfFile "$file" - done < <(find "$prefix" -type f -print0) + done < <(find "$@" -type f -print0) } # XXX: This should ultimately use fixupOutputHooks but we currently don't have @@ -181,5 +181,5 @@ autoPatchelf() { # behaviour as fixupOutputHooks because the setup hook for patchelf is run in # fixupOutput and the postFixup hook runs later. postFixupHooks+=( - 'for output in $outputs; do prefix="${!output}" autoPatchelf; done' + 'autoPatchelf $(for output in $outputs; do echo "${!output}"; done)' ) From e4fbb244ee313a3003144e8e148341c9e5c67295 Mon Sep 17 00:00:00 2001 From: aszlig Date: Mon, 19 Nov 2018 17:36:22 +0100 Subject: [PATCH 2/7] autoPatchelfHook: Allow to prevent automatic run If you want to only run autoPatchelf on a specific path and leave everything else alone, we now have a $dontAutoPatchelf environment variable, which causes the postFixup hook to not run at all. The name "dontAutoPatchelf" probably is a bit weird in conjunction with putting "autoPatchelfHook" in nativeBuildInputs, but unless someone comes up with a better name I keep it that way because it's consistent with all the other dontStrip, dontPatchShebangs, dontPatchELF and whatnot. A specific example where this is needed is when building the Android SDK emulator, which contains a few ARM binaries in subdirectories that should not be patched. If we were to run autoPatchelf on all outputs unconditionally we'd run into errors because some ARM libraries couldn't be found. Signed-off-by: aszlig --- pkgs/build-support/setup-hooks/auto-patchelf.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index 59e9a933211f..62348d71ed05 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -180,6 +180,11 @@ autoPatchelf() { # So what we do here is basically run in postFixup and emulate the same # behaviour as fixupOutputHooks because the setup hook for patchelf is run in # fixupOutput and the postFixup hook runs later. -postFixupHooks+=( - 'autoPatchelf $(for output in $outputs; do echo "${!output}"; done)' -) +postFixupHooks+=(' + if [ -z "$dontAutoPatchelf" ]; then + autoPatchelf $(for output in $outputs; do + [ -e "${!output}" ] || continue + echo "${!output}" + done) + fi +') From 3ca35ce0b2dd1adb11044e03a816c77a72f7135d Mon Sep 17 00:00:00 2001 From: aszlig Date: Mon, 19 Nov 2018 23:23:38 +0100 Subject: [PATCH 3/7] autoPatchelfHook: Add --no-recurse flag This is to be used with the autoPatchelf command and allows to only patch a specific file or directory without recursing into subdirectories. Apart from being able to run the command in a standalone way, as detailled in the previous commit this is also needed for the Android SDK emulator, because according to @svanderburg there are subdirectories we don't want to patch. The reason why I didn't use GNU getopt is that it might not be available on all operating systems and the getopts bash builtin doesn't support long arguments. Apart from that, the implementation for recognizing the flag is pretty trivial and it's also using bash builtins only, so if we want to do something really fancy someday, we can still change it. Signed-off-by: aszlig --- .../setup-hooks/auto-patchelf.sh | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index 62348d71ed05..43b1679670d4 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -148,12 +148,32 @@ autoPatchelfFile() { } autoPatchelf() { + local -a findOpts=() + + while [ $# -gt 0 ]; do + case "$1" in + --) shift; break;; + --no-recurse) shift; findOpts+=("-maxdepth" 1);; + --*) + echo "autoPatchelf: ERROR: Invalid command line" \ + "argument: $1" >&2 + return 1;; + *) break;; + esac + done + + if [ $# -eq 0 ]; then + echo "autoPatchelf: No paths to patch specified." >&2 + return 1 + fi + echo "automatically fixing dependencies for ELF files" >&2 # Add all shared objects of the current output path to the start of # cachedDependencies so that it's choosen first in findDependency. cachedDependencies+=( - $(find "$@" \! -type d \( -name '*.so' -o -name '*.so.*' \)) + $(find "$@" "${findOpts[@]}" \! -type d \ + \( -name '*.so' -o -name '*.so.*' \)) ) local elffile @@ -169,7 +189,7 @@ autoPatchelf() { LANG=C readelf -l "$file" | grep -q "^ *INTERP\\>" || continue fi autoPatchelfFile "$file" - done < <(find "$@" -type f -print0) + done < <(find "$@" "${findOpts[@]}" -type f -print0) } # XXX: This should ultimately use fixupOutputHooks but we currently don't have @@ -182,7 +202,7 @@ autoPatchelf() { # fixupOutput and the postFixup hook runs later. postFixupHooks+=(' if [ -z "$dontAutoPatchelf" ]; then - autoPatchelf $(for output in $outputs; do + autoPatchelf -- $(for output in $outputs; do [ -e "${!output}" ] || continue echo "${!output}" done) From 503b41b9b2d726f10372e3daac713cc2042b8cf0 Mon Sep 17 00:00:00 2001 From: aszlig Date: Mon, 19 Nov 2018 23:33:44 +0100 Subject: [PATCH 4/7] doc/stdenv: Document autoPatchelfHook changes First of all, this makes the existing documentation a bit more clear on what autoPatchelfHook is all about, because after discussing with @svanderburg - who wrote a similar implementation - the rationale about autoPatchelfHook wasn't very clear in the documentation. I also added the recent changes around being able to use autoPatchelf manually and the new --no-recurse flag. Signed-off-by: aszlig --- doc/stdenv.xml | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/doc/stdenv.xml b/doc/stdenv.xml index 9ed9a448c614..e1e8090dcb89 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -2421,12 +2421,31 @@ addEnvHooks "$hostOffset" myBashFunction This is a special setup hook which helps in packaging proprietary software in that it automatically tries to find missing shared library - dependencies of ELF files. All packages within the - runtimeDependencies environment variable are - unconditionally added to executables, which is useful for programs that - use - dlopen - 3 to load libraries at runtime. + dependencies of ELF files based on the given + buildInputs and nativeBuildInputs. + + + You can also specify a runtimeDependencies environment + variable which lists dependencies that are unconditionally added to all + executables. + + + This is useful for programs that use + dlopen + 3 + to load libraries at runtime. + + + In certain situations you may want to run the main command + (autoPatchelf) of the setup hook on a file or a set + of directories instead of unconditionally patching all outputs. This + can be done by setting the dontAutoPatchelf environment + variable to a non-empty value. + + + The autoPatchelf command also recognizes a + --no-recurse command line flag, + which prevents it from recursing into subdirectories. From 2faf905f98eeb9f674ef845ae9bf9fd7f1279d1e Mon Sep 17 00:00:00 2001 From: aszlig Date: Sun, 25 Nov 2018 16:17:14 +0100 Subject: [PATCH 5/7] autoPatchelfHook: Add addAutoPatchelfSearchPath This function is useful if autoPatchelf is invoked during some of the phases of a build and allows to add arbitrary shared objects to the search path. So far the same functionality was in autoPatchelf itself, but not available as a separate function, so when adding shared objects to the dependency cache one would have to do so manually. The function also has the --no-recurse flag, which prevents recursing into subdirectories. Signed-off-by: aszlig --- .../setup-hooks/auto-patchelf.sh | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index 43b1679670d4..61bdafa88c37 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -147,13 +147,38 @@ autoPatchelfFile() { fi } -autoPatchelf() { +# Can be used to manually add additional directories with shared object files +# to be included for the next autoPatchelf invocation. +addAutoPatchelfSearchPath() { local -a findOpts=() + # XXX: Somewhat similar to the one in the autoPatchelf function, maybe make + # it DRY someday... while [ $# -gt 0 ]; do case "$1" in --) shift; break;; --no-recurse) shift; findOpts+=("-maxdepth" 1);; + --*) + echo "addAutoPatchelfSearchPath: ERROR: Invalid command line" \ + "argument: $1" >&2 + return 1;; + *) break;; + esac + done + + cachedDependencies+=( + $(find "$@" "${findOpts[@]}" \! -type d \ + \( -name '*.so' -o -name '*.so.*' \)) + ) +} + +autoPatchelf() { + local -a norecurse= + + while [ $# -gt 0 ]; do + case "$1" in + --) shift; break;; + --no-recurse) shift; norecurse=1;; --*) echo "autoPatchelf: ERROR: Invalid command line" \ "argument: $1" >&2 @@ -171,11 +196,7 @@ autoPatchelf() { # Add all shared objects of the current output path to the start of # cachedDependencies so that it's choosen first in findDependency. - cachedDependencies+=( - $(find "$@" "${findOpts[@]}" \! -type d \ - \( -name '*.so' -o -name '*.so.*' \)) - ) - local elffile + addAutoPatchelfSearchPath ${norecurse:+--no-recurse} -- "$@" # Here we actually have a subshell, which also means that # $cachedDependencies is final at this point, so whenever we want to run @@ -189,7 +210,7 @@ autoPatchelf() { LANG=C readelf -l "$file" | grep -q "^ *INTERP\\>" || continue fi autoPatchelfFile "$file" - done < <(find "$@" "${findOpts[@]}" -type f -print0) + done < <(find "$@" ${norecurse:+-maxdepth 1} -type f -print0) } # XXX: This should ultimately use fixupOutputHooks but we currently don't have From 9f23a63f79e1a9d4a7f51444c12605d73ec770e1 Mon Sep 17 00:00:00 2001 From: aszlig Date: Mon, 26 Nov 2018 01:13:59 +0100 Subject: [PATCH 6/7] autoPatchelfHook: Fix type of norecurse variable While declaring it as an array doesn't do any harm in our usage, it might be a bit confusing when reading the code. Signed-off-by: aszlig --- pkgs/build-support/setup-hooks/auto-patchelf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index 61bdafa88c37..9a124f415b8c 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -173,7 +173,7 @@ addAutoPatchelfSearchPath() { } autoPatchelf() { - local -a norecurse= + local norecurse= while [ $# -gt 0 ]; do case "$1" in From 4a6e3e4185dcbe53c3d5eceaf2570d51516ad2b4 Mon Sep 17 00:00:00 2001 From: aszlig Date: Mon, 26 Nov 2018 01:58:36 +0100 Subject: [PATCH 7/7] autoPatchelfHook: Skip on missing segment headers If the file in question is not a shared object file but an ELF, we really want to skip the file, because we won't have anything to patch there. For example if the file is created via "gcc -c -o foo.o foo.c", we don't get a segment header and so far autoPatchelf was trying to patch such a file. By checking for missing segment headers, we're now no longer going to attempt patching such a file. Signed-off-by: aszlig Reported-by: Sander van der Burg --- pkgs/build-support/setup-hooks/auto-patchelf.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/setup-hooks/auto-patchelf.sh b/pkgs/build-support/setup-hooks/auto-patchelf.sh index 9a124f415b8c..5bedd1a9f9ca 100644 --- a/pkgs/build-support/setup-hooks/auto-patchelf.sh +++ b/pkgs/build-support/setup-hooks/auto-patchelf.sh @@ -205,9 +205,12 @@ autoPatchelf() { # outside of this function. while IFS= read -r -d $'\0' file; do isELF "$file" || continue + segmentHeaders="$(LANG=C readelf -l "$file")" + # Skip if the ELF file doesn't have segment headers (eg. object files). + echo "$segmentHeaders" | grep -q '^Program Headers:' || continue if isExecutable "$file"; then # Skip if the executable is statically linked. - LANG=C readelf -l "$file" | grep -q "^ *INTERP\\>" || continue + echo "$segmentHeaders" | grep -q "^ *INTERP\\>" || continue fi autoPatchelfFile "$file" done < <(find "$@" ${norecurse:+-maxdepth 1} -type f -print0)