diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.nix b/pkgs/test/nixpkgs-check-by-name/src/eval.nix
index f13dbd167a00..87c54b6444ee 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/eval.nix
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.nix
@@ -79,34 +79,37 @@ let
         };
       };
 
-  byNameAttrs = map (name: [
-    name
-    {
-      ByName =
-        if ! pkgs ? ${name} then
-          { Missing = null; }
-        else
-          { Existing = attrInfo name pkgs.${name}; };
-    }
-  ]) attrs;
+  byNameAttrs = builtins.listToAttrs (map (name: {
+    inherit name;
+    value.ByName =
+      if ! pkgs ? ${name} then
+        { Missing = null; }
+      else
+        { Existing = attrInfo name pkgs.${name}; };
+  }) attrs);
 
   # Information on all attributes that exist but are not in pkgs/by-name.
   # We need this to enforce pkgs/by-name for new packages
-  nonByNameAttrs = map (name:
+  nonByNameAttrs = builtins.mapAttrs (name: value:
     let
-      output = attrInfo name pkgs.${name};
+      output = attrInfo name value;
       result = builtins.tryEval (builtins.deepSeq output null);
     in
-    [
-      name
-      {
-        NonByName =
-          if result.success then
-            { EvalSuccess = output; }
-          else
-            { EvalFailure = null; };
-      }
-    ]
-  ) (builtins.attrNames (builtins.removeAttrs pkgs attrs));
+    {
+      NonByName =
+        if result.success then
+          { EvalSuccess = output; }
+        else
+          { EvalFailure = null; };
+    }
+  ) (builtins.removeAttrs pkgs attrs);
+
+  # All attributes
+  attributes = byNameAttrs // nonByNameAttrs;
 in
-byNameAttrs ++ nonByNameAttrs
+# We output them in the form [ [ <name> <value> ] ]` such that the Rust side
+# doesn't need to sort them again to get deterministic behavior (good for testing)
+map (name: [
+  name
+  attributes.${name}
+]) (builtins.attrNames attributes)
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/all-packages.nix
new file mode 100644
index 000000000000..688f52b9358f
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/all-packages.nix
@@ -0,0 +1,6 @@
+self: super: {
+  a = self.callPackage ./pkgs/by-name/a/a/package.nix { };
+  b = self.callPackage ({ someDrv }: someDrv) { };
+  c = self.callPackage ./pkgs/by-name/c/c/package.nix { };
+  d = self.callPackage ({ someDrv }: someDrv) { };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/default.nix
new file mode 100644
index 000000000000..af25d1450122
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/default.nix
@@ -0,0 +1 @@
+import ../mock-nixpkgs.nix { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/expected b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/expected
new file mode 100644
index 000000000000..349e9ad47c41
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/expected
@@ -0,0 +1,4 @@
+pkgs.a: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/a/a/package.nix { ... }` with a non-empty second argument.
+pkgs.b: This is a new top-level package of the form `callPackage ... { }`. Please define it in pkgs/by-name/b/b/package.nix instead. See `pkgs/by-name/README.md` for more details. Since the second `callPackage` argument is `{ }`, no manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is needed anymore.
+pkgs.c: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/c/c/package.nix { ... }` with a non-empty second argument.
+pkgs.d: This is a new top-level package of the form `callPackage ... { }`. Please define it in pkgs/by-name/d/d/package.nix instead. See `pkgs/by-name/README.md` for more details. Since the second `callPackage` argument is `{ }`, no manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is needed anymore.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/a/a/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/a/a/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/a/a/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/c/c/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/c/c/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/c/c/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv