tests(tests): Initialize infrastructure and documentation for it

6 files changed, 363 insertions, 0 deletions

diff --git a/tests/README.md b/tests/README.md
new file mode 100644
index 00000000..fe93dc96
--- /dev/null
+++ b/tests/README.md
@@ -0,0 +1,75 @@
+# How to write a test
+
+Test are simple derivations, put into the `by-name` directory.  As you
+often want to test configuration, the 'standard' nixos-vm-test function
+starts a vm with the specified configuration and lets you run a python
+test script.
+
+But, I've noticed, that most of my tests in this nixos-configuration are
+actually testing the home-manager config of a program. As such I've built
+a test infrastructure on top of the pre-existing nixos-vm-tests. It is
+further described in the `mkTest` section.
+
+## `mkTest`
+A standard application test would look somewhat like this:
+
+```nix
+{mkTest}:
+mkTest {
+  name = "my-tests";
+
+  # The configuration to add.
+  # In this case, the less config is being tested.
+  configuration = {
+    imports = [
+      ../../../../modules/by-name/le/less/module.nix
+    ];
+    config.soispha.programs.less.enable = true;
+  };
+
+  # Files that are provided at the test directory.
+  # In this case, the test dir would have a `./data/test.file`
+  # path pre-populated with the
+  # contents of the file at the lhs.
+  testData = {
+    "data/test.file" = ./data/test.file;
+  };
+
+  # The description of what to do in the test.
+  description = ./test.desc;
+
+  # A sha256 hashsum of the concatenated output of the program.
+  # This can be `null` to disable the checksum mechanism.
+  hash = "87901231393b51cdd45bbb4339a32db2894a3a5ab164cb5c7a8fa14721fdcba7";
+}
+```
+
+### The test description file
+All line starting with `#` or only containing white space are
+ignored. Additionally, all leading and trailing white space is stripped
+before evaluating.
+
+The general syntax is:
+```
+COMMAND ARGS
+```
+where `COMMAND` is one of the commands listed in [Commands](#### Commands)
+
+The `ARGS` are the verbatim content from the space after the `COMMAND`
+to the end of line character.
+
+#### Commands
+##### `Type`
+Send the `ARGS` to the application. This interprets `ARGS` as specified
+by the tmux (1) man page on `send-keys`.
+
+#### `Sleep`
+Sleep for `ARGS` seconds.
+
+#### `Expect`
+Grep the currently visible screen for the regex `ARGS`.  This regex
+must match.
+#### `ExpectNot`
+Does the same thing as [Expect](##### Expect), but enforces, that the
+regex does not match.
+
diff --git a/tests/default.nix b/tests/default.nix
new file mode 100644
index 00000000..9803c99c
--- /dev/null
+++ b/tests/default.nix
@@ -0,0 +1,33 @@
+{
+  lib,
+  pkgs,
+  myPkgs,
+  nixpkgs_as_input,
+  extraModules,
+}: let
+  nixLib = import ../lib {};
+
+  mkTest = import ./infrastructure {
+    inherit
+      pkgs
+      lib
+      nixos-lib
+      myPkgs
+      extraModules
+      ;
+  };
+
+  nixos-lib = import (nixpkgs_as_input + "/nixos/lib") {};
+
+  tests = nixLib.mkByName {
+    baseDirectory = ./by-name;
+    fileName = "test.nix";
+    finalizeFunction = name: value:
+      import value {
+        inherit
+          mkTest
+          ;
+      };
+  };
+in
+  tests
diff --git a/tests/infrastructure/clean.awk b/tests/infrastructure/clean.awk
new file mode 100644
index 00000000..1208b1ef
--- /dev/null
+++ b/tests/infrastructure/clean.awk
@@ -0,0 +1,13 @@
+{
+    # Shell like comments
+    gsub(/^#.*$/, "", $0)
+
+    # Strip leading and trailing white space
+    gsub(/^[[:blank:]]*/, "", $0)
+    gsub(/[[:blank:]]*$/, "", $0)
+
+    # Only accept the line, if it contains something
+    if (NF) {
+        print $0
+    }
+}
diff --git a/tests/infrastructure/default.nix b/tests/infrastructure/default.nix
new file mode 100644
index 00000000..4f1ec7a2
--- /dev/null
+++ b/tests/infrastructure/default.nix
@@ -0,0 +1,136 @@
+{
+  pkgs,
+  myPkgs,
+  lib,
+  nixos-lib,
+  extraModules,
+  ...
+}: {
+  name,
+  configuration,
+  description,
+  hash,
+  testData,
+}:
+nixos-lib.runTest {
+  hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs
+
+  inherit name;
+
+  node = {
+    specialArgs = {
+      inherit myPkgs;
+    };
+    # Use the nixpkgs as constructed by the `nixpkgs.*` options
+    pkgs = null;
+  };
+
+  nodes = {
+    machine = {config, ...}: {
+      imports =
+        [
+          extraModules.home-manager
+          ../../modules/by-name/us/users/module.nix
+        ]
+        ++ configuration.imports;
+
+      config = lib.modules.mkMerge [
+        {
+          soispha = {
+            users = {
+              enable = true;
+
+              # The password is 'test'.
+              hashedPassword = "$y$j9T$yYYUpE9OaxmEO8MaPI2jr0$WGpYYWaLySakI.Mwqz4sljGSOetAp4s5CIUa1VUU1l2";
+            };
+          };
+          home-manager.users.soispha.home.stateVersion = "24.11";
+        }
+        configuration.config
+      ];
+    };
+  };
+
+  testScript = {nodes, ...}: let
+    testDir = "${nodes.machine.home-manager.users.soispha.home.homeDirectory}/test";
+    goldenFile = "${testDir}/__test_golden";
+    logFile = "${testDir}/__test_log";
+  in
+    /*
+    python
+    */
+    ''
+      start_all()
+
+      machine.succeed("sudo -u soispha ${pkgs.writeShellScript "mkTestEnvironment" ''
+        set -e
+
+        mkdir --parents "${testDir}"
+        cd "${testDir}"
+
+        ${
+          lib.strings.concatStringsSep "\n\n" (
+            builtins.attrValues
+            (builtins.mapAttrs
+              (name: value: ''
+                mkdir --parents "$(dirname '${name}')"
+                cp --recursive '${value}' '${name}'
+              '')
+              testData)
+          )
+        }
+      ''}")
+
+      machine.succeed("sudo -u soispha ${pkgs.writeShellScript "mkGoldenRecord" ''
+        set -e
+
+        # HACK: Prevent zsh from complaining about missing configuration.
+        # TODO: These tests should probably just run under bash. <2024-11-22>
+        touch ~soispha/.zshrc
+
+        cd "${testDir}"
+
+        __TEST_TMUX="${lib.getExe pkgs.tmux}"
+        __TEST_TMUX_PANE="__TEST_TMUX_PANE"
+
+        __TEST_EVAL_AWK_CLEAN_FILE="${./clean.awk}"
+        __TEST_EVAL_LOG_FILE="${logFile}"
+
+        . ${./driver.sh}
+
+        "$__TEST_TMUX" new-session -d -s "$__TEST_TMUX_PANE"
+        "$__TEST_TMUX" pipe-pane -t "$__TEST_TMUX_PANE" -o 'cat >>${goldenFile}'
+
+        __test_eval "${description}"
+
+        # Clear the pipe again
+        "$__TEST_TMUX" pipe-pane -t "$__TEST_TMUX_PANE"
+      ''}")
+
+      machine.succeed("sudo -u soispha ${pkgs.writeShellScript "testHashOfGolden" ''
+          set -e
+
+          cd "${testDir}"
+        ''
+        + lib.optionalString (hash != null)
+        /*
+        bash
+        */
+        ''
+          golden_hash="$(sha256sum ${goldenFile} | awk '{print $1}')"
+
+          if [ "$golden_hash" != "${hash}" ]; then
+            echo "Hash mismatch."
+            echo "Expected '${hash}',"
+            echo "but got '$golden_hash'"
+            exit 1
+          else
+            echo "Hash was successfully checked."
+          fi
+        ''}")
+
+
+      machine.copy_from_vm("${goldenFile}", "golden")
+      machine.copy_from_vm("${logFile}", "log")
+    '';
+}
diff --git a/tests/infrastructure/driver.sh b/tests/infrastructure/driver.sh
new file mode 100644
index 00000000..52f7d3ad
--- /dev/null
+++ b/tests/infrastructure/driver.sh
@@ -0,0 +1,72 @@
+#! /usr/bin/env sh
+set -e
+
+msg() {
+    if [ "$#" -ne 0 ]; then
+        echo "$@" | tee --append "$__TEST_EVAL_LOG_FILE" >&2
+    else
+        cat | tee --append "$__TEST_EVAL_LOG_FILE" >&2
+    fi
+}
+
+__test_eval() {
+    tmux="$__TEST_TMUX"
+    tpane="$__TEST_TMUX_PANE"
+    file="$1"
+
+    awk --file "$__TEST_EVAL_AWK_CLEAN_FILE" "$file" | while read -r cmd args; do
+        case "$cmd" in
+        "Type")
+            msg "Sending keys to application '$args'.."
+            "$tmux" send-keys -t "$tpane": "$args"
+            ;;
+        "Sleep")
+            msg "Sleeping for '$args' seconds.."
+            sleep "$args"
+            ;;
+        "Expect" | "ExpectNot")
+            msg "Trying to match regex ('$args') for currently visible content.."
+
+            matched=""
+            if "$tmux" capture-pane -t "$tpane" -p -S 0 -E - | grep "$args"; then
+                matched=true
+            else
+                matched=false
+            fi
+
+            case "$cmd" in
+            "Expect")
+                if [ "$matched" = true ]; then
+                    msg "Regex matched."
+                else
+                    msg "Failed to find string, matched by regex '$args' on the screen"
+                    msg current screen:
+                    "$tmux" capture-pane -t "$tpane" -p -S 0 -E - | msg
+
+                    exit 1
+                fi
+                ;;
+            "ExpectNot")
+                if [ "$matched" = false ]; then
+                    msg "Regex successfully not matched."
+                else
+                    msg "Found to find string, matched by regex '$args' on the screen. But expected none"
+                    msg current screen:
+                    "$tmux" capture-pane -t "$tpane" -p -S 0 -E - | msg
+
+                    exit 1
+                fi
+                ;;
+            *)
+                msg "Entered unrechable code. This is a bug."
+                exit 1
+                ;;
+            esac
+            ;;
+        *)
+            msg "Unrecognized command: '$cmd'"
+            exit 1
+            ;;
+        esac
+    done
+}
diff --git a/tests/infrastructure/run.nix b/tests/infrastructure/run.nix
new file mode 100644
index 00000000..91120ef4
--- /dev/null
+++ b/tests/infrastructure/run.nix
@@ -0,0 +1,34 @@
+{
+  pkgs,
+  lib,
+}:
+pkgs.writeShellScript "run_test_description" ''
+  set -e
+
+  [ "$#" -ne 1 ] && {
+    echo "Usage: $0 <Test description file>";
+    exit 2
+  }
+  description="$1"
+
+  __TEST_TMUX="${lib.getExe pkgs.tmux}"
+  __TEST_TMUX_PANE="__TEST_TMUX_PANE"
+  __TEST_AWK_CLEAN_FILE="${./clean.awk}"
+
+  . ${./driver.sh}
+
+  echo "Setting up a session.."
+  if "$__TEST_TMUX" has-session -t "$__TEST_TMUX_PANE"; then
+    echo "Killing old '$__TEST_TMUX_PANE'"
+    "$__TEST_TMUX" kill-session -t "$__TEST_TMUX_PANE"
+  fi
+  "$__TEST_TMUX" new-session -d -s "$__TEST_TMUX_PANE"
+
+  echo "Initializing pipe.."
+  "$__TEST_TMUX" pipe-pane -t "$__TEST_TMUX_PANE" -o 'cat >>./test.golden'
+
+  echo "Evaluating description.."
+  __test_eval "$description"
+
+  "$__TEST_TMUX" pipe-pane -t "$__TEST_TMUX_PANE"
+''