diff --git a/nixos/modules/programs/ssh.nix b/nixos/modules/programs/ssh.nix
index a66679dff90d..89333533465e 100644
--- a/nixos/modules/programs/ssh.nix
+++ b/nixos/modules/programs/ssh.nix
@@ -31,7 +31,7 @@ in
 
       setXAuthLocation = mkOption {
         type = types.bool;
-        default = true;
+        default = config.services.xserver.enable;
         description = ''
           Whether to set the path to <command>xauth</command> for X11-forwarded connections.
           This causes a dependency on X11 packages.