diff --git a/pkgs/development/libraries/ctranslate2/default.nix b/pkgs/development/libraries/ctranslate2/default.nix
index 83eb9bf3a895..58906b608231 100644
--- a/pkgs/development/libraries/ctranslate2/default.nix
+++ b/pkgs/development/libraries/ctranslate2/default.nix
@@ -60,7 +60,7 @@ stdenv.mkDerivation rec {
     cudaPackages.cuda_cudart
     cudaPackages.libcublas
     cudaPackages.libcurand
-  ] ++ lib.optionals withCuDNN [
+  ] ++ lib.optionals (withCUDA && withCuDNN) [
     cudaPackages.cudnn
   ] ++ lib.optionals withOneDNN [
     oneDNN
@@ -87,5 +87,8 @@ stdenv.mkDerivation rec {
     changelog = "https://github.com/OpenNMT/CTranslate2/blob/${src.rev}/CHANGELOG.md";
     license = licenses.mit;
     maintainers = with maintainers; [ hexa misuzu ];
+    broken =
+      (lib.versionOlder cudaPackages.cudaVersion "11.4")
+      || !(withCuDNN -> withCUDA);
   };
 }
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 8075dae6f091..6679e1dee8e4 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -20935,7 +20935,8 @@ with pkgs;
   ctranslate2 = callPackage ../development/libraries/ctranslate2 rec {
     stdenv = if withCUDA then gcc11Stdenv else pkgs.stdenv;
     withCUDA = pkgs.config.cudaSupport;
-    withCuDNN = pkgs.config.cudaSupport;
+    withCuDNN = withCUDA && (cudaPackages ? cudnn);
+    cudaPackages = pkgs.cudaPackages;
   };
 
   ubus = callPackage ../development/libraries/ubus { };