flake.nix: run tests against pinned nixpkgs

and migrate to the new runTest, which evaluates much faster.

flake.nix

@@ -21,27 +21,38 @@
     releases = [
       {
         name = "unstable";
+        nixpkgs = nixpkgs;
         pkgs = nixpkgs.legacyPackages.${system};
       }
       {
         name = "24.11";
+        nixpkgs = nixpkgs-24_11;
         pkgs = nixpkgs-24_11.legacyPackages.${system};
       }
     ];
     testNames = [
-      "internal"
-      "external"
       "clamav"
-      "multiple"
+      "external"
+      "internal"
       "ldap"
+      "multiple"
     ];
-    genTest = testName: release: {
+
-      "name"= "${testName}-${builtins.replaceStrings ["."] ["_"] release.name}";
+    genTest = testName: release: let
-      "value"= import (./tests/. + "/${testName}.nix") {
       pkgs = release.pkgs;
-        inherit blobs;
+      nixos-lib = import (release.nixpkgs + "/nixos/lib") {
+        inherit (pkgs) lib;
+      };
+    in {
+      name = "${testName}-${builtins.replaceStrings ["."] ["_"] release.name}";
+      value = nixos-lib.runTest {
+        hostPkgs = pkgs;
+        imports = [ ./tests/${testName}.nix ];
+        _module.args = { inherit blobs; };
+        extraBaseModules.imports = [ ./default.nix ];
       };
     };
+
     # Generate an attribute set such as
     # {
     #   external-unstable = <derivation>;

tests/clamav.nix

@@ -14,12 +14,17 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>
 
-{ pkgs ? import <nixpkgs> {}, blobs}:
+{
+  lib,
+  blobs,
+  ...
+}:
 
-pkgs.nixosTest {
+{
   name = "clamav";
+
   nodes = {
-    server = { config, pkgs, lib, ... }:
+    server = { pkgs, ... }:
         {
             imports = [
                 ../default.nix
@@ -28,6 +33,8 @@ pkgs.nixosTest {
 
             virtualisation.memorySize = 1500;
 
+            environment.systemPackages = with pkgs; [ netcat ];
+
             services.rsyslogd = {
               enable = true;
               defaultConfig = ''
@@ -83,7 +90,7 @@ pkgs.nixosTest {
               "root/eicar.com.txt".text = "X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*";
             };
         };
-      client = { nodes, config, pkgs, ... }: let
+      client = { nodes, pkgs, ... }: let
         serverIP = nodes.server.networking.primaryIPAddress;
         clientIP = nodes.client.networking.primaryIPAddress;
         grep-ip = pkgs.writeScriptBin "grep-ip" ''
@@ -180,8 +187,7 @@ pkgs.nixosTest {
       };
     };
 
-  testScript = { nodes, ... }:
+  testScript = ''
-      ''
       start_all()
 
       server.wait_for_unit("multi-user.target")
@@ -189,10 +195,10 @@ pkgs.nixosTest {
 
       # TODO put this blocking into the systemd units? I am not sure if rspamd already waits for the clamd socket.
       server.wait_until_succeeds(
-          "set +e; timeout 1 ${nodes.server.nixpkgs.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
+          "set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
       )
       server.wait_until_succeeds(
-          "set +e; timeout 1 ${nodes.server.nixpkgs.pkgs.netcat}/bin/nc -U /run/clamav/clamd.ctl < /dev/null; [ $? -eq 124 ]"
+          "set +e; timeout 1 nc -U /run/clamav/clamd.ctl < /dev/null; [ $? -eq 124 ]"
       )
 
       client.execute("cp -p /etc/root/.* ~/")

tests/external.nix

@@ -14,18 +14,19 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>
 
-{ pkgs ? import <nixpkgs> {}, ...}:
+{
-
-pkgs.nixosTest {
   name = "external";
+
   nodes = {
-    server = { config, pkgs, ... }:
+    server = { pkgs, ... }:
         {
             imports = [
                 ../default.nix
                 ./lib/config.nix
             ];
 
+            environment.systemPackages = with pkgs; [ netcat ];
+
             virtualisation.memorySize = 1024;
 
             services.rsyslogd = {
@@ -86,7 +87,7 @@ pkgs.nixosTest {
               };
             };
         };
-      client = { nodes, config, pkgs, ... }: let
+      client = { nodes, pkgs, ... }: let
         serverIP = nodes.server.networking.primaryIPAddress;
         clientIP = nodes.client.networking.primaryIPAddress;
         grep-ip = pkgs.writeScriptBin "grep-ip" ''
@@ -341,8 +342,7 @@ pkgs.nixosTest {
       };
     };
 
-  testScript = { nodes, ... }:
+  testScript = ''
-      ''
       start_all()
 
       server.wait_for_unit("multi-user.target")
@@ -350,7 +350,7 @@ pkgs.nixosTest {
 
       # TODO put this blocking into the systemd units?
       server.wait_until_succeeds(
-          "set +e; timeout 1 ${nodes.server.nixpkgs.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
+          "set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
       )
 
       client.execute("cp -p /etc/root/.* ~/")

tests/internal.nix

@@ -14,7 +14,10 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>
 
-{ pkgs ? import <nixpkgs> {}, ...}:
+{
+  pkgs,
+  ...
+}:
 
 let
   sendMail = pkgs.writeTextFile {
@@ -36,10 +39,11 @@ let
   hashedPasswordFile = hashPassword "my-password";
   passwordFile = pkgs.writeText "password" "my-password";
 in
-pkgs.nixosTest {
+{
   name = "internal";
+
   nodes = {
-    machine = { config, pkgs, ... }: {
+    machine = { pkgs, ... }: {
       imports = [
         ./../default.nix
         ./lib/config.nix
@@ -50,7 +54,12 @@ pkgs.nixosTest {
       environment.systemPackages = [
         (pkgs.writeScriptBin "mail-check" ''
           ${pkgs.python3}/bin/python ${../scripts/mail-check.py} $@
-        '')];
+        '')
+      ] ++ (with pkgs; [
+        curl
+        openssl
+        netcat
+      ]);
 
       mailserver = {
         enable = true;
@@ -174,22 +183,22 @@ pkgs.nixosTest {
         machine.wait_for_open_port(25)
         # TODO put this blocking into the systemd units
         machine.wait_until_succeeds(
-            "set +e; timeout 1 ${pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
+            "set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
         )
         machine.succeed(
-            "cat ${sendMail} | ${pkgs.netcat-gnu}/bin/nc localhost 25 | grep -q '554 5.5.0 Error'"
+            "cat ${sendMail} | nc localhost 25 | grep -q '554 5.5.0 Error'"
         )
 
     with subtest("rspamd controller serves web ui"):
         machine.succeed(
-            "set +o pipefail; ${pkgs.curl}/bin/curl --unix-socket /run/rspamd/worker-controller.sock http://localhost/ | grep -q '<body>'"
+            "set +o pipefail; curl --unix-socket /run/rspamd/worker-controller.sock http://localhost/ | grep -q '<body>'"
         )
 
     with subtest("imap port 143 is closed and imaps is serving SSL"):
         machine.wait_for_closed_port(143)
         machine.wait_for_open_port(993)
         machine.succeed(
-            "echo | ${pkgs.openssl}/bin/openssl s_client -connect localhost:993 | grep 'New, TLS'"
+            "echo | openssl s_client -connect localhost:993 | grep 'New, TLS'"
         )
   '';
 }

tests/ldap.nix

@@ -1,16 +1,13 @@
-{ pkgs ? import <nixpkgs> {}
-, ...
-}:
-
 let
   bindPassword = "unsafegibberish";
   alicePassword = "testalice";
   bobPassword = "testbob";
 in
-pkgs.nixosTest {
+{
   name = "ldap";
+
   nodes = {
-    machine = { config, pkgs, ... }: {
+    machine = { pkgs, ... }: {
       imports = [
         ./../default.nix
         ./lib/config.nix

tests/minimal.nix

@@ -14,18 +14,14 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>
 
-import <nixpkgs/nixos/tests/make-test-python.nix> {
-
-  nodes.machine =
-    { config, pkgs, ... }:
 {
-        imports = [
+  name = "minimal";
-            ./../default.nix
+
-        ];
+  nodes.machine = {
+    imports = [  ./../default.nix ];
   };
 
-  testScript =
+  testScript = ''
-    ''
     machine.wait_for_unit("multi-user.target");
   '';
 }

tests/multiple.nix

@@ -1,6 +1,9 @@
 # This tests is used to test features requiring several mail domains.
 
-{ pkgs ? import <nixpkgs> {}, ...}:
+{
+  pkgs,
+  ...
+}:
 
 let
     hashPassword = password: pkgs.runCommand
@@ -12,8 +15,9 @@ let
 
     password = pkgs.writeText "password" "password";
 
-    domainGenerator = domain: { config, pkgs, ... }: {
+    domainGenerator = domain: { pkgs, ... }: {
       imports = [../default.nix];
+      environment.systemPackages = with pkgs; [ netcat ];
       virtualisation.memorySize = 1024;
       mailserver = {
         enable = true;
@@ -36,8 +40,9 @@ let
 
 in
 
-pkgs.nixosTest {
+{
   name = "multiple";
+
   nodes = {
     domain1 = {...}: {
       imports = [
@@ -50,7 +55,7 @@ pkgs.nixosTest {
       };
     };
     domain2 = domainGenerator "domain2.com";
-    client = { config, pkgs, ... }: {
+    client = { pkgs, ... }: {
       environment.systemPackages = [
         (pkgs.writeScriptBin "mail-check" ''
           ${pkgs.python3}/bin/python ${../scripts/mail-check.py} $@
@@ -65,10 +70,10 @@ pkgs.nixosTest {
 
     # TODO put this blocking into the systemd units?
     domain1.wait_until_succeeds(
-        "set +e; timeout 1 ${pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
+        "set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
     )
     domain2.wait_until_succeeds(
-        "set +e; timeout 1 ${pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
+        "set +e; timeout 1 nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]"
     )
 
     # user@domain1.com sends a mail to user@domain2.com