refactor({modules,pkgs}/by-name-overlay): De-duplicate in a `nixLib`

7 files changed, 132 insertions, 132 deletions

diff --git a/flake.nix b/flake.nix
index 0fddec86..399e0577 100644
--- a/flake.nix
+++ b/flake.nix
@@ -285,10 +285,12 @@
     system = "x86_64-linux";
     sysLib = shell_library.lib.${system};
 
+    nixLib = import ./lib {};
+
     pkgsStable = nixpkgs-stable.legacyPackages.${system};
     pkgs = nixpkgs.legacyPackages.${system};
     myPkgs = import ./pkgs {
-      inherit sysLib pkgs;
+      inherit sysLib pkgs nixLib;
     };
 
     nixpkgs_as_input = nixpkgs;
diff --git a/lib/by-name-overlay.nix b/lib/by-name-overlay.nix
new file mode 100644
index 00000000..cc8bcb67
--- /dev/null
+++ b/lib/by-name-overlay.nix
@@ -0,0 +1,65 @@
+{warn}:
+# Adapted from this: https://github.com/NixOS/nixpkgs/blob/1814b56453c91192f6d5a6276079948f9fe96c18/pkgs/top-level/by-name-overlay.nix
+# This file should not depend on `pkgs` and thus not use `lib`.
+{
+  baseDirectory,
+  fileName,
+  finalizeFunction,
+}: let
+  # Takes a list of attrs as input and returns one merged attr set.
+  flattenAttrs = list:
+    if builtins.isList list
+    then
+      builtins.foldl' (acc: elem:
+        if builtins.isList elem
+        # Merging them with `//` is okay here, as we can be sure that the attr names are
+        # unique (they were separate dictionary after all)
+        then acc // (flattenAttrs elem)
+        else acc // elem) {}
+      list
+    else list;
+
+  # From nixpkgs/lib {{{
+  # These functions are taken straight out of the `nixpkgs/lib`.
+  # We can't depended on `pkgs` (and thus on `lib`), because the `pkgs` module argument
+  # is only defined in the `nixpkgs` module (which is imported through this function).
+  mapAttrsToList = f: attrs:
+    builtins.map (name: f name attrs.${name}) (builtins.attrNames attrs);
+
+  nameValuePair = name: value: {inherit name value;};
+  filterAttrs = pred: set:
+    builtins.listToAttrs (builtins.concatMap (name: let
+      v = set.${name};
+    in
+      if pred name v
+      then [(nameValuePair name v)]
+      else []) (builtins.attrNames set));
+  # }}}
+
+  # Module files for a single shard
+  # Type: String -> String -> ListOf Path
+  namesForShard = shard: type:
+    if type != "directory"
+    then warn "Ignored non-directory, whilst importing by-name directory (${fileName}): '${shard}'" {}
+    else let
+      mkPath = name: _type: let
+        path = baseDirectory + "/${shard}/${name}" + "/${fileName}";
+      in
+        if builtins.pathExists path
+        then path
+        else warn "'${builtins.toString path}' does not exist. Skipped" null;
+    in
+      filterAttrs (name: value: value != null)
+      (builtins.mapAttrs
+        mkPath
+        (builtins.readDir (baseDirectory + "/${shard}")));
+
+  # A list of all module paths.
+  # These can the be simply injected into `import`
+  files = flattenAttrs (mapAttrsToList namesForShard (builtins.readDir baseDirectory));
+  output =
+    builtins.mapAttrs
+    finalizeFunction
+    files;
+in
+  output
diff --git a/lib/default.nix b/lib/default.nix
new file mode 100644
index 00000000..7c86aba1
--- /dev/null
+++ b/lib/default.nix
@@ -0,0 +1,37 @@
+{}: let
+  # Taken from `nixpkgs/lib`.
+  # Is here to avoid a dependency on `lib` (making this file easy to test with `nix eval`)
+  warn =
+    # Since Nix 2.23, https://github.com/NixOS/nix/pull/10592
+    builtins.warn
+    or (
+      # Do not eta reduce v, so that we have the same strictness as `builtins.warn`.
+      msg: v:
+      # `builtins.warn` requires a string message, so we enforce that in our implementation, so that callers aren't accidentally incompatible with newer Nix versions.
+        assert builtins.isString msg;
+          builtins.trace "evaluation warning: ${msg}" v
+    );
+in {
+  mkByName = import ./by-name-overlay.nix {inherit warn;};
+
+  # Only merge two attrsets if the RHS does not share names with the LHS.
+  # Example:
+  # ```nix
+  # let
+  # attr1 = {atuin = "first"; default = "second";};
+  # attr2 = {atuin = "hi"; other = "hi2";};
+  # in
+  # maybeMerge attr1 attr2 "Failed to merge"
+  # ```
+  maybeMerge = attr1: attr2: message: let
+    list1 = builtins.attrNames attr1;
+
+    check = name2: value:
+      if builtins.elem name2 list1
+      then warn "Overriding ${name2} while merging two attrs sets: ${message}" value
+      else value;
+
+    checkedAttr2 = builtins.mapAttrs check attr2;
+  in
+    attr1 // checkedAttr2;
+}
diff --git a/modules/by-name-overlay.nix b/modules/by-name-overlay.nix
deleted file mode 100644
index 4349e112..00000000
--- a/modules/by-name-overlay.nix
+++ /dev/null
@@ -1,48 +0,0 @@
-# Adapted from this: https://github.com/NixOS/nixpkgs/blob/1814b56453c91192f6d5a6276079948f9fe96c18/pkgs/top-level/by-name-overlay.nix
-{baseDirectory}: let
-  # Taken straight out of the `nixpkgs/lib/lists.nix` file.
-  # We can't depended on `pkgs` (and thus on `lib`), because the `pkgs` module argument
-  # is only defined in modules we import.
-  flatten = x:
-    if builtins.isList x
-    then builtins.concatMap flatten x
-    else [x];
-  # Same thing as flatten.
-  warn =
-    # Since Nix 2.23, https://github.com/NixOS/nix/pull/10592
-    builtins.warn
-    or (
-      # Do not eta reduce v, so that we have the same strictness as `builtins.warn`.
-      msg: v:
-      # `builtins.warn` requires a string message, so we enforce that in our implementation, so that callers aren't accidentally incompatible with newer Nix versions.
-        assert builtins.isString msg;
-          builtins.trace "evaluation warning: ${msg}" v
-    );
-  # Same thing as flatten.
-  mapAttrsToList = f: attrs:
-    builtins.map (name: f name attrs.${name}) (builtins.attrNames attrs);
-
-  # Module files for a single shard
-  # Type: String -> String -> ListOf Path
-  namesForShard = shard: type:
-    if type != "directory"
-    then warn "Ignored non-directory, whilst importing by-name modules: '${shard}'" []
-    else let
-      mkPath = name: _type: let
-        path = baseDirectory + "/${shard}/${name}/module.nix";
-      in
-        if builtins.pathExists path
-        then path
-        else warn "'${builtins.toString path}' does not exist. Skipped" null;
-    in
-      flatten
-      (builtins.filter (it: it != null)
-        (mapAttrsToList
-          mkPath
-          (builtins.readDir (baseDirectory + "/${shard}"))));
-
-  # A list of all module paths.
-  # These can the be simply injected into `import`
-  moduleFiles = flatten (mapAttrsToList namesForShard (builtins.readDir baseDirectory));
-in
-  moduleFiles
diff --git a/modules/default.nix b/modules/default.nix
index a3bc1735..3364c3db 100644
--- a/modules/default.nix
+++ b/modules/default.nix
@@ -1,10 +1,15 @@
 # NOTE: This file **must** not depend on `pkgs`. This is because `pkgs` is defined in a
 # module imported by it, and thus would require infinite recursion.  <2024-10-18>
 {...}: let
-  files = import ./by-name-overlay.nix {
-    baseDirectory =
-      ./by-name;
-  };
+  nixLib = import ../lib {};
+
+  files =
+    builtins.attrValues
+    (nixLib.mkByName {
+      baseDirectory = ./by-name;
+      fileName = "module.nix";
+      finalizeFunction = name: value: value;
+    });
 in {
   imports = files;
 }
diff --git a/pkgs/by-name-overlay.nix b/pkgs/by-name-overlay.nix
deleted file mode 100644
index 7eb3d239..00000000
--- a/pkgs/by-name-overlay.nix
+++ /dev/null
@@ -1,70 +0,0 @@
-# Adapted from this: https://github.com/NixOS/nixpkgs/blob/1814b56453c91192f6d5a6276079948f9fe96c18/pkgs/top-level/by-name-overlay.nix
-{
-  lib,
-  sysLib,
-  pkgs,
-  baseDirectory,
-}:
-# This file turns the pkgs/by-name directory (see its README.md for more info) into an overlay that adds all the defined packages.
-# No validity checks are done here,
-# instead this file is optimised for performance,
-# and validity checks are done by CI on PRs.
-let
-  # FIXME: Check if we override something in the set <2024-05-22>
-  callPackage = lib.callPackageWith (pkgs // myPkgs // {inherit sysLib;});
-  inherit lib;
-
-  inherit
-    (builtins)
-    readDir
-    ;
-
-  inherit
-    (lib.attrsets)
-    mapAttrs
-    mapAttrsToList
-    mergeAttrsList
-    ;
-
-  # Package files for a single shard
-  # Type: String -> String -> AttrsOf Path
-  namesForShard = shard: type:
-    if type != "directory"
-    then
-      # Ignore all non-directories. Technically only README.md is allowed as a file in the base directory, so we could alternatively:
-      # - Assume that README.md is the only file and change the condition to `shard == "README.md"` for a minor performance improvement.
-      #   This would however cause very poor error messages if there's other files.
-      # - Ensure that README.md is the only file, throwing a better error message if that's not the case.
-      #   However this would make for a poor code architecture, because one type of error would have to be duplicated in the validity checks and here.
-      # Additionally in either of those alternatives, we would have to duplicate the hardcoding of "README.md"
-      {}
-    else
-      mapAttrs
-      (name: _: baseDirectory + "/${shard}/${name}/package.nix")
-      (readDir (baseDirectory + "/${shard}"));
-
-  # The attribute set mapping names to the package files defining them
-  # This is defined up here in order to allow reuse of the value (it's kind of expensive to compute)
-  # if the overlay has to be applied multiple times
-  packageFiles = mergeAttrsList (mapAttrsToList namesForShard (readDir baseDirectory));
-  myPkgs =
-    mapAttrs
-    (_: path: callPackage path {})
-    packageFiles;
-in
-  # # TODO: Consider optimising this using `builtins.deepSeq packageFiles`,
-  # # which could free up the above thunks and reduce GC times.
-  # # Currently this would be hard to measure until we have more packages
-  # # and ideally https://github.com/NixOS/nix/pull/8895
-  # self: super:
-  #   {
-  #     # This attribute is necessary to allow CI to ensure that all packages defined in `pkgs/by-name`
-  #     # don't have an overriding definition in `all-packages.nix` with an empty (`{ }`) second `callPackage` argument.
-  #     # It achieves that with an overlay that modifies both `callPackage` and this attribute to signal whether `callPackage` is used
-  #     # and whether it's defined by this file here or `all-packages.nix`.
-  #     # TODO: This can be removed once `pkgs/by-name` can handle custom `callPackage` arguments without `all-packages.nix` (or any other way of achieving the same result).
-  #     # Because at that point the code in ./stage.nix can be changed to not allow definitions in `all-packages.nix` to override ones from `pkgs/by-name` anymore and throw an error if that happens instead.
-  #     _internalCallByNamePackageFile = file: self.callPackage file {};
-  #   }
-  #   //
-  myPkgs
diff --git a/pkgs/default.nix b/pkgs/default.nix
index 66803694..39d225a9 100644
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -1,13 +1,22 @@
 {
-  pkgs,
-  sysLib,
+  pkgs ? (builtins.getFlake "nixpkgs").legacyPackages."x86_64-linux",
+  sysLib ? builtins.trace "Moking `sysLib`" {},
+  nixLib ? import ../lib {},
 }: let
-  # TODO: Filter the sources of every package in the shards <2024-05-25>
-  files = import ./by-name-overlay.nix {
-    inherit pkgs sysLib;
-    inherit (pkgs) lib;
-    baseDirectory =
-      ./by-name;
+  inherit (pkgs) lib;
+
+  # FIXME: Make this override check actually work.
+  # I think that some parts of the lazy eval are causing that to not actually evaluate the
+  # error message. <2024-10-23>
+  maybeMergeMessage = "While merging the pkgs in ./pkgs/by-name to the nixpkgs set.";
+  mMM = maybeMergeMessage;
+  callPackage = lib.callPackageWith (nixLib.maybeMerge (nixLib.maybeMerge pkgs myPkgs mMM) {inherit sysLib;} mMM);
+
+  myPkgs = nixLib.mkByName {
+    baseDirectory = ./by-name;
+    fileName = "package.nix";
+    finalizeFunction = name: value: callPackage value {};
   };
 in
-  files
+  # TODO: Filter the sources of every package in the shards <2024-05-25>
+  myPkgs