lib.packagesFromDirectoryRecursive: use explicit recursion, support nested scopes

lib/filesystem.nix

@@ -12,11 +12,6 @@ let
     toString
     ;
 
-  inherit (lib.attrsets)
-    mapAttrs'
-    filterAttrs
-    ;
-
   inherit (lib.filesystem)
     pathIsDirectory
     pathIsRegularFile
@@ -26,7 +21,6 @@ let
 
   inherit (lib.strings)
     hasSuffix
-    removeSuffix
     ;
 in
 
@@ -296,26 +290,27 @@ in
       `callPackage <directory>/package.nix { }` is returned.
     - Otherwise, the input directory's contents are listed and transformed into
       an attribute set.
-      - If a file name has the `.nix` extension, it is turned into attribute
+      - If a regular file's name has the `.nix` extension, it is turned into attribute
         where:
         - The attribute name is the file name without the `.nix` extension
         - The attribute value is `callPackage <file path> { }`
-      - Other files are ignored.
       - Directories are turned into an attribute where:
         - The attribute name is the name of the directory
         - The attribute value is the result of calling
           `packagesFromDirectoryRecursive { ... }` on the directory.
 
         As a result, directories with no `.nix` files (including empty
         directories) will be transformed into empty attribute sets.
+      - Other files are ignored, including symbolic links to directories and to regular `.nix`
+        files; this is because nixlang code cannot distinguish the type of a link's target.
 
     # Type
 
     ```
     packagesFromDirectoryRecursive :: {
       callPackage :: Path -> {} -> a,
+      newScope? :: AttrSet -> scope,
       directory :: Path,
-      ...
     } -> AttrSet
     ```
 
@@ -325,10 +320,14 @@ in
     : The function used to convert a Nix file's path into a leaf of the attribute set.
       It is typically the `callPackage` function, taken from either `pkgs` or a new scope corresponding to the `directory`.
 
+    `newScope`
+    : If present, this function is used when recursing into a directory, to generate a new scope.
+      The arguments are updated with the scope's `callPackage` and `newScope` functions, so packages can require
+      anything in their scope, or in an ancestor of their scope.
+
     `directory`
     : The directory to read package files from.
 
-
     # Examples
     :::{.example}
     ## Basic use of `lib.packagesFromDirectoryRecursive`
@@ -348,12 +347,10 @@ in
     ::::{.example}
     ## Create a scope for the nix files found in a directory
     ```nix
-    lib.makeScope pkgs.newScope (
-      self: packagesFromDirectoryRecursive {
-        inherit (self) callPackage;
-        directory = ./my-packages;
-      }
-    )
+    packagesFromDirectoryRecursive {
+      inherit (pkgs) callPackage newScope;
+      directory = ./my-packages;
+    }
     => { ... }
     ```
 
@@ -372,46 +369,59 @@ in
     :::{.note}
     `a.nix` cannot directly take as inputs packages defined in a child directory, such as `b1`.
     :::
-
-    :::{.warning}
-    As of now, `lib.packagesFromDirectoryRecursive` cannot create nested scopes for sub-directories.
-
-    In particular, files under `b/` can only require (as inputs) other files under `my-packages`,
-    but not to those in the same directory, nor those in a parent directory; e.g, `b2.nix` cannot directly
-    require `b1`.
-    :::
     ::::
   */
   packagesFromDirectoryRecursive =
+    let
+      inherit (lib) concatMapAttrs id makeScope recurseIntoAttrs removeSuffix;
+      inherit (lib.path) append;
+
+      # Generate an attrset corresponding to a given directory.
+      # This function is outside `packagesFromDirectoryRecursive`'s lambda expression,
+      #  to prevent accidentally using its parameters.
+      processDir = { callPackage, directory, ... }@args:
+        concatMapAttrs (name: type:
+          # for each directory entry
+          let path = append directory name; in
+          if type == "directory" then {
+            # recurse into directories
+            "${name}" = packagesFromDirectoryRecursive (args // {
+              directory = path;
+            });
+          } else if type == "regular" && hasSuffix ".nix" name then {
+            # call .nix files
+            "${removeSuffix ".nix" name}" = callPackage path {};
+          } else if type == "regular" then {
+            # ignore non-nix files
+          } else throw ''
+            lib.filesystem.packagesFromDirectoryRecursive: Unsupported file type ${type} at path ${toString path}
+          ''
+        ) (builtins.readDir directory);
+    in
     {
       callPackage,
+      newScope ? throw "lib.packagesFromDirectoryRecursive: newScope wasn't passed in args",
       directory,
-      ...
-    }:
+    }@args:
     let
-      inherit (lib) concatMapAttrs removeSuffix;
-      inherit (lib.path) append;
       defaultPath = append directory "package.nix";
     in
     if pathExists defaultPath then
       # if `${directory}/package.nix` exists, call it directly
       callPackage defaultPath {}
-    else concatMapAttrs (name: type:
-      # otherwise, for each directory entry
-      let path = append directory name; in
-      if type == "directory" then {
-        # recurse into directories
-        "${name}" = packagesFromDirectoryRecursive {
-          inherit callPackage;
-          directory = path;
-        };
-      } else if type == "regular" && hasSuffix ".nix" name then {
-        # call .nix files
-        "${removeSuffix ".nix" name}" = callPackage path {};
-      } else if type == "regular" then {
-        # ignore non-nix files
-      } else throw ''
-        lib.filesystem.packagesFromDirectoryRecursive: Unsupported file type ${type} at path ${toString path}
-      ''
-    ) (builtins.readDir directory);
+    else if args ? newScope then
+      # Create a new scope and mark it `recurseForDerivations`.
+      # This lets the packages refer to each other.
+      # See:
+      #  [lib.makeScope](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.customisation.makeScope) and
+      #  [lib.recurseIntoAttrs](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.customisation.makeScope)
+      recurseIntoAttrs (makeScope newScope (self:
+        # generate the attrset representing the directory, using the new scope's `callPackage` and `newScope`
+        processDir (args // {
+          inherit (self) callPackage newScope;
+        })
+      ))
+    else
+      processDir args
+  ;
 }

lib/tests/misc.nix

@@ -2559,7 +2559,7 @@ runTests {
   testPackagesFromDirectoryRecursive = {
     expr = packagesFromDirectoryRecursive {
       callPackage = path: overrides: import path overrides;
-      directory = ./packages-from-directory;
+      directory = ./packages-from-directory/plain;
     };
     expected = {
       a = "a";
@@ -2584,8 +2584,36 @@ runTests {
   testPackagesFromDirectoryRecursiveTopLevelPackageNix = {
     expr = packagesFromDirectoryRecursive {
       callPackage = path: overrides: import path overrides;
-      directory = ./packages-from-directory/c;
+      directory = ./packages-from-directory/plain/c;
     };
     expected = "c";
   };
+
+  # Check that `packagesFromDirectoryRecursive` can be used to create scopes
+  # for sub-directories
+  testPackagesFromDirectoryNestedScopes = let
+    inherit (lib) makeScope recurseIntoAttrs;
+    emptyScope = makeScope lib.callPackageWith (_: {});
+  in {
+    expr = lib.filterAttrsRecursive (name: value: !lib.elem name [ "callPackage" "newScope" "overrideScope" "packages" ]) (packagesFromDirectoryRecursive {
+      inherit (emptyScope) callPackage newScope;
+      directory = ./packages-from-directory/scope;
+    });
+    expected = lib.recurseIntoAttrs {
+      a = "a";
+      b = "b";
+      # Note: Other files/directories in `./test-data/c/` are ignored and can be
+      # used by `package.nix`.
+      c = "c";
+      my-namespace = lib.recurseIntoAttrs {
+        d = "d";
+        e = "e";
+        f = "f";
+        my-sub-namespace = lib.recurseIntoAttrs {
+          g = "g";
+          h = "h";
+        };
+      };
+    };
+  };
 }

lib/tests/packages-from-directory/scope/a.nix

@@ -0,0 +1 @@
+{ }: "a"

lib/tests/packages-from-directory/scope/b.nix

@@ -0,0 +1,3 @@
+{ a }:
+assert a == "a";
+"b"

lib/tests/packages-from-directory/scope/c/not-a-namespace/not-a-package.nix

@@ -0,0 +1 @@
+{ }

lib/tests/packages-from-directory/scope/c/package.nix

@@ -0,0 +1 @@
+{ }: "c"

lib/tests/packages-from-directory/scope/c/support-definitions.nix

@@ -0,0 +1 @@
+{ }

lib/tests/packages-from-directory/scope/my-namespace/d.nix

@@ -0,0 +1,5 @@
+{ a, e }:
+# Check we can get parameter from the parent scope(s) as well as the current one
+assert a == "a";
+assert e == "e";
+"d"

lib/tests/packages-from-directory/scope/my-namespace/e.nix

@@ -0,0 +1,3 @@
+{ d }:
+# Check that mutual recursion is possible
+"e"

lib/tests/packages-from-directory/scope/my-namespace/f/package.nix

@@ -0,0 +1 @@
+{ }: "f"

lib/tests/packages-from-directory/scope/my-namespace/my-sub-namespace/g.nix

@@ -0,0 +1,7 @@
+{
+  a,
+  d,
+  h,
+}:
+# Check we can get parameters from ancestral scopes (e.g. the scope's grandparent)
+"g"

lib/tests/packages-from-directory/scope/my-namespace/my-sub-namespace/h.nix

@@ -0,0 +1 @@
+{ }: "h"

nixos/doc/manual/redirects.json

@@ -1892,6 +1892,9 @@
   "sec-release-25.05-lib-breaking": [
     "release-notes.html#sec-release-25.05-lib-breaking"
   ],
+  "sec-release-25.05-lib-additions-improvements": [
+    "release-notes.html#sec-release-25.05-lib-additions-improvements"
+  ],
   "sec-release-24.11": [
     "release-notes.html#sec-release-24.11"
   ],

nixos/doc/manual/release-notes/rl-2505.section.md

@@ -310,3 +310,11 @@
 - Structure of the `functor` of some types has changed. `functor` is an implementation detail and should not be relied upon. If you did rely on it let us know in this [PR](https://github.com/NixOS/nixpkgs/pull/363565).
   - [`lib.types.enum`](https://nixos.org/manual/nixos/unstable/#sec-option-types-basic): Previously the `functor.payload` was the list of enum values directly. Now it is an attribute set containing the values in the `values` attribute.
   - [`lib.types.separatedString`](https://nixos.org/manual/nixos/unstable/#sec-option-types-string): Previously the `functor.payload` was the seperator directly. Now it is an attribute set containing the seperator in the `sep` attribute.
+
+- [`lib.packagesFromDirectoryRecursive`] now rejects unknown arguments, and applies [`lib.recurseIntoAttrs`] when recursing into a directory.
+  [`lib.packagesFromDirectoryRecursive`]: https://nixos.org/manual/nixpkgs/stable/#function-library-lib.filesystem.packagesFromDirectoryRecursive
+  [`lib.recurseIntoAttrs`]: https://nixos.org/manual/nixpkgs/stable/#function-library-lib.attrsets.recurseIntoAttrs
+
+### Additions and Improvements {#sec-release-25.05-lib-additions-improvements}
+
+- [`lib.packagesFromDirectoryRecursive`] can now construct nested scopes matching the directory tree passed as input.