From 4008d0cb53b9abd00d2dc568814566227ddd3825 Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Mon, 27 Jul 2020 22:50:27 +0200 Subject: [PATCH] Move tests to the Python framework --- nix/sources.json | 12 +-- tests/clamav.nix | 75 +++++++++--------- tests/default.nix | 2 +- tests/extern.nix | 191 ++++++++++++++++++++++++---------------------- tests/intern.nix | 87 ++++++++++----------- 5 files changed, 189 insertions(+), 178 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index c6f1041..92e90a3 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -5,10 +5,10 @@ "homepage": "https://github.com/NixOS/nixpkgs", "owner": "NixOS", "repo": "nixpkgs-channels", - "rev": "29eddfc36d720dcc4822581175217543b387b1e8", - "sha256": "1gqv2m7plkladd3va664xyqb962pqs4pizzibvkm1nh0f4rfpxvy", + "rev": "274831839c4f2622019a35d773d4ee588a08c311", + "sha256": "1dn0cv8ciq83c75hjqlqljjmsns60wqyijx9hzm3j8jvfxm4gzgk", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs-channels/archive/29eddfc36d720dcc4822581175217543b387b1e8.tar.gz", + "url": "https://github.com/NixOS/nixpkgs-channels/archive/274831839c4f2622019a35d773d4ee588a08c311.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-unstable": { @@ -17,10 +17,10 @@ "homepage": "https://github.com/NixOS/nixpkgs", "owner": "NixOS", "repo": "nixpkgs-channels", - "rev": "fce7562cf46727fdaf801b232116bc9ce0512049", - "sha256": "14rvi69ji61x3z88vbn17rg5vxrnw2wbnanxb7y0qzyqrj7spapx", + "rev": "5717d9d2f7ca0662291910c52f1d7b95b568fec2", + "sha256": "17gxd2f622pyss3r6cjngdav6wzdbr31d7bqx9z2lawxg47mmk1l", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs-channels/archive/fce7562cf46727fdaf801b232116bc9ce0512049.tar.gz", + "url": "https://github.com/NixOS/nixpkgs-channels/archive/5717d9d2f7ca0662291910c52f1d7b95b568fec2.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/tests/clamav.nix b/tests/clamav.nix index 05cb30c..c641622 100644 --- a/tests/clamav.nix +++ b/tests/clamav.nix @@ -16,8 +16,8 @@ { pkgs ? import {}}: -import (pkgs.path + "/nixos/tests/make-test.nix") { - +pkgs.nixosTest { + name = "clamav"; nodes = { server = { config, pkgs, lib, ... }: let @@ -73,7 +73,6 @@ import (pkgs.path + "/nixos/tests/make-test.nix") { mailserver = { enable = true; - debug = true; fqdn = "mail.example.com"; domains = [ "example.com" "example2.com" ]; virusScanning = true; @@ -194,52 +193,56 @@ import (pkgs.path + "/nixos/tests/make-test.nix") { testScript = { nodes, ... }: '' - startAll; + start_all() - $server->waitForUnit("multi-user.target"); - $client->waitForUnit("multi-user.target"); + server.wait_for_unit("multi-user.target") + client.wait_for_unit("multi-user.target") # TODO put this blocking into the systemd units? I am not sure if rspamd already waits for the clamd socket. - $server->waitUntilSucceeds("timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ \$? -eq 124 ]"); - $server->waitUntilSucceeds("timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/clamav/clamd.ctl < /dev/null; [ \$? -eq 124 ]"); + server.wait_until_succeeds( + "timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]" + ) + server.wait_until_succeeds( + "timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/clamav/clamd.ctl < /dev/null; [ $? -eq 124 ]" + ) - $client->execute("cp -p /etc/root/.* ~/"); - $client->succeed("mkdir -p ~/mail"); - $client->succeed("ls -la ~/ >&2"); - $client->succeed("cat ~/.fetchmailrc >&2"); - $client->succeed("cat ~/.procmailrc >&2"); - $client->succeed("cat ~/.msmtprc >&2"); + client.execute("cp -p /etc/root/.* ~/") + client.succeed("mkdir -p ~/mail") + client.succeed("ls -la ~/ >&2") + client.succeed("cat ~/.fetchmailrc >&2") + client.succeed("cat ~/.procmailrc >&2") + client.succeed("cat ~/.msmtprc >&2") # fetchmail returns EXIT_CODE 1 when no new mail - $client->succeed("fetchmail --nosslcertck -v || [ \$? -eq 1 ] >&2"); + client.succeed("fetchmail --nosslcertck -v || [ $? -eq 1 ] >&2") # Verify that mail can be sent and received before testing virus scanner - $client->execute("rm ~/mail/*"); - $client->succeed("msmtp -a user2 user1\@example.com < /etc/root/safe-email >&2"); + client.execute("rm ~/mail/*") + client.succeed("msmtp -a user2 user1@example.com < /etc/root/safe-email >&2") # give the mail server some time to process the mail - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); - $client->execute("rm ~/mail/*"); + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') + client.execute("rm ~/mail/*") # fetchmail returns EXIT_CODE 0 when it retrieves mail - $client->succeed("fetchmail --nosslcertck -v >&2"); - $client->execute("rm ~/mail/*"); + client.succeed("fetchmail --nosslcertck -v >&2") + client.execute("rm ~/mail/*") - subtest "virus scan file", sub { - $server->succeed("clamdscan \$(readlink -f /etc/root/eicar.com.txt) | grep \"Txt\\.Malware\\.Agent-1787597 FOUND\" >&2"); - }; + with subtest("virus scan file"): + server.succeed( + 'clamdscan $(readlink -f /etc/root/eicar.com.txt) | grep "Txt\\.Malware\\.Agent-1787597 FOUND" >&2' + ) - subtest "virus scan email", sub { - $client->succeed("msmtp -a user2 user1\@example.com < /etc/root/virus-email 2>&1 | tee /dev/stderr | grep \"server message: 554 5\\.7\\.1\" >&2"); - $server->succeed("journalctl -u rspamd | grep -i eicar"); + with subtest("virus scan email"): + client.succeed( + 'msmtp -a user2 user1\@example.com < /etc/root/virus-email 2>&1 | tee /dev/stderr | grep "server message: 554 5\\.7\\.1" >&2' + ) + server.succeed("journalctl -u rspamd | grep -i eicar") # give the mail server some time to process the mail - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); - }; - - subtest "no warnings or errors", sub { - $server->fail("journalctl -u postfix | grep -i error >&2"); - $server->fail("journalctl -u postfix | grep -i warning >&2"); - $server->fail("journalctl -u dovecot2 | grep -i error >&2"); - $server->fail("journalctl -u dovecot2 | grep -i warning >&2"); - }; + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') + with subtest("no warnings or errors"): + server.fail("journalctl -u postfix | grep -i error >&2") + server.fail("journalctl -u postfix | grep -i warning >&2") + server.fail("journalctl -u dovecot2 | grep -i error >&2") + server.fail("journalctl -u dovecot2 | grep -i warning >&2") ''; } diff --git a/tests/default.nix b/tests/default.nix index 11b3843..1e1b67b 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -23,7 +23,7 @@ let test = pkgs.callPackage (./. + "/${testName}.nix") { }; in { "name"= builtins.replaceStrings ["." "-"] ["_" "_"] release; - "value"= test { inherit pkgs; }; + "value"= test; }; releaseNames = [ diff --git a/tests/extern.nix b/tests/extern.nix index 343c458..f619c00 100644 --- a/tests/extern.nix +++ b/tests/extern.nix @@ -16,8 +16,8 @@ { pkgs ? import {}}: -import (pkgs.path + "/nixos/tests/make-test.nix") { - +pkgs.nixosTest { + name = "extern"; nodes = { server = { config, pkgs, ... }: { @@ -282,137 +282,144 @@ import (pkgs.path + "/nixos/tests/make-test.nix") { testScript = { nodes, ... }: '' - startAll; + start_all() - $server->waitForUnit("multi-user.target"); - $client->waitForUnit("multi-user.target"); + server.wait_for_unit("multi-user.target") + client.wait_for_unit("multi-user.target") # TODO put this blocking into the systemd units? - $server->waitUntilSucceeds("timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ \$? -eq 124 ]"); + server.wait_until_succeeds( + "timeout 1 ${nodes.server.pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]" + ) - $client->execute("cp -p /etc/root/.* ~/"); - $client->succeed("mkdir -p ~/mail"); - $client->succeed("ls -la ~/ >&2"); - $client->succeed("cat ~/.fetchmailrc >&2"); - $client->succeed("cat ~/.procmailrc >&2"); - $client->succeed("cat ~/.msmtprc >&2"); + client.execute("cp -p /etc/root/.* ~/") + client.succeed("mkdir -p ~/mail") + client.succeed("ls -la ~/ >&2") + client.succeed("cat ~/.fetchmailrc >&2") + client.succeed("cat ~/.procmailrc >&2") + client.succeed("cat ~/.msmtprc >&2") - subtest "imap retrieving mail", sub { + with subtest("imap retrieving mail"): # fetchmail returns EXIT_CODE 1 when no new mail - $client->succeed("fetchmail --nosslcertck -v || [ \$? -eq 1 ] >&2"); - }; + client.succeed("fetchmail --nosslcertck -v || [ $? -eq 1 ] >&2") - subtest "submission port send mail", sub { + with subtest("submission port send mail"): # send email from user2 to user1 - $client->succeed("msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2"); + client.succeed( + "msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2" + ) # give the mail server some time to process the mail - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); - }; + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') - subtest "imap retrieving mail 2", sub { - $client->execute("rm ~/mail/*"); + with subtest("imap retrieving mail 2"): + client.execute("rm ~/mail/*") # fetchmail returns EXIT_CODE 0 when it retrieves mail - $client->succeed("fetchmail --nosslcertck -v >&2"); - }; + client.succeed("fetchmail --nosslcertck -v >&2") - subtest "remove sensitive information on submission port", sub { - $client->succeed("cat ~/mail/* >&2"); - ## make sure our IP is _not_ in the email header - $client->fail("grep-ip ~/mail/*"); - $client->succeed("check-mail-id ~/mail/*"); - }; + with subtest("remove sensitive information on submission port"): + client.succeed("cat ~/mail/* >&2") + ## make sure our IP is _not_ in the email header + client.fail("grep-ip ~/mail/*") + client.succeed("check-mail-id ~/mail/*") - subtest "have correct fqdn as sender", sub { - $client->succeed("grep 'Received: from mail.example.com' ~/mail/*"); - }; + with subtest("have correct fqdn as sender"): + client.succeed("grep 'Received: from mail.example.com' ~/mail/*") - subtest "dkim has user-specified size", sub { - $server->succeed("openssl rsa -in /var/dkim/example.com.mail.key -text -noout | grep 'Private-Key: (1535 bit'"); - }; + with subtest("dkim has user-specified size"): + server.succeed( + "openssl rsa -in /var/dkim/example.com.mail.key -text -noout | grep 'Private-Key: (1535 bit'" + ) - subtest "dkim singing, multiple domains", sub { - $client->execute("rm ~/mail/*"); + with subtest("dkim singing, multiple domains"): + client.execute("rm ~/mail/*") # send email from user2 to user1 - $client->succeed("msmtp -a test2 --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email2 >&2"); - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + client.succeed( + "msmtp -a test2 --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email2 >&2" + ) + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') # fetchmail returns EXIT_CODE 0 when it retrieves mail - $client->succeed("fetchmail --nosslcertck -v"); - $client->succeed("cat ~/mail/* >&2"); + client.succeed("fetchmail --nosslcertck -v") + client.succeed("cat ~/mail/* >&2") # make sure it is dkim signed - $client->succeed("grep DKIM ~/mail/*"); - }; + client.succeed("grep DKIM ~/mail/*") - subtest "aliases", sub { - $client->execute("rm ~/mail/*"); + with subtest("aliases"): + client.execute("rm ~/mail/*") # send email from chuck to postmaster - $client->succeed("msmtp -a test3 --tls=on --tls-certcheck=off --auth=on postmaster\@example.com < /etc/root/email2 >&2"); - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + client.succeed( + "msmtp -a test3 --tls=on --tls-certcheck=off --auth=on postmaster\@example.com < /etc/root/email2 >&2" + ) + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') # fetchmail returns EXIT_CODE 0 when it retrieves mail - $client->succeed("fetchmail --nosslcertck -v"); - }; + client.succeed("fetchmail --nosslcertck -v") - subtest "catchAlls", sub { - $client->execute("rm ~/mail/*"); + with subtest("catchAlls"): + client.execute("rm ~/mail/*") # send email from chuck to non exsitent account - $client->succeed("msmtp -a test3 --tls=on --tls-certcheck=off --auth=on lol\@example.com < /etc/root/email2 >&2"); - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + client.succeed( + "msmtp -a test3 --tls=on --tls-certcheck=off --auth=on lol\@example.com < /etc/root/email2 >&2" + ) + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') # fetchmail returns EXIT_CODE 0 when it retrieves mail - $client->succeed("fetchmail --nosslcertck -v"); + client.succeed("fetchmail --nosslcertck -v") - $client->execute("rm ~/mail/*"); + client.execute("rm ~/mail/*") # send email from user1 to chuck - $client->succeed("msmtp -a test4 --tls=on --tls-certcheck=off --auth=on chuck\@example.com < /etc/root/email2 >&2"); - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + client.succeed( + "msmtp -a test4 --tls=on --tls-certcheck=off --auth=on chuck\@example.com < /etc/root/email2 >&2" + ) + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') # fetchmail returns EXIT_CODE 1 when no new mail # if this succeeds, it means that user1 recieved the mail that was intended for chuck. - $client->fail("fetchmail --nosslcertck -v"); - }; + client.fail("fetchmail --nosslcertck -v") - subtest "extraVirtualAliases", sub { - $client->execute("rm ~/mail/*"); + with subtest("extraVirtualAliases"): + client.execute("rm ~/mail/*") # send email from single-alias to user1 - $client->succeed("msmtp -a test5 --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email4 >&2"); - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + client.succeed( + "msmtp -a test5 --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email4 >&2" + ) + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') # fetchmail returns EXIT_CODE 0 when it retrieves mail - $client->succeed("fetchmail --nosslcertck -v"); + client.succeed("fetchmail --nosslcertck -v") - $client->execute("rm ~/mail/*"); + client.execute("rm ~/mail/*") # send email from user1 to multi-alias (user{1,2}@example.com) - $client->succeed("msmtp -a test --tls=on --tls-certcheck=off --auth=on multi-alias\@example.com < /etc/root/email5 >&2"); - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + client.succeed( + "msmtp -a test --tls=on --tls-certcheck=off --auth=on multi-alias\@example.com < /etc/root/email5 >&2" + ) + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') # fetchmail returns EXIT_CODE 0 when it retrieves mail - $client->succeed("fetchmail --nosslcertck -v"); - }; + client.succeed("fetchmail --nosslcertck -v") - subtest "quota", sub { - $client->execute("rm ~/mail/*"); - $client->execute("mv ~/.fetchmailRcLowQuota ~/.fetchmailrc"); + with subtest("quota"): + client.execute("rm ~/mail/*") + client.execute("mv ~/.fetchmailRcLowQuota ~/.fetchmailrc") - $client->succeed("msmtp -a test3 --tls=on --tls-certcheck=off --auth=on lowquota\@example.com < /etc/root/email2 >&2"); - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + client.succeed( + "msmtp -a test3 --tls=on --tls-certcheck=off --auth=on lowquota\@example.com < /etc/root/email2 >&2" + ) + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') # fetchmail returns EXIT_CODE 0 when it retrieves mail - $client->fail("fetchmail --nosslcertck -v"); + client.fail("fetchmail --nosslcertck -v") - }; - - subtest "imap sieve junk trainer", sub { + with subtest("imap sieve junk trainer"): # send email from user2 to user1 - $client->succeed("msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2"); + client.succeed( + "msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2" + ) # give the mail server some time to process the mail - $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + server.wait_until_fails('[ "$(postqueue -p)" != "Mail queue is empty" ]') - $client->succeed("imap-mark-spam >&2"); - $server->waitUntilSucceeds("journalctl -u dovecot2 | grep -i sa-learn-spam.sh >&2"); - $client->succeed("imap-mark-ham >&2"); - $server->waitUntilSucceeds("journalctl -u dovecot2 | grep -i sa-learn-ham.sh >&2"); - }; - - subtest "no warnings or errors", sub { - $server->fail("journalctl -u postfix | grep -i error >&2"); - $server->fail("journalctl -u postfix | grep -i warning >&2"); - $server->fail("journalctl -u dovecot2 | grep -i error >&2"); - $server->fail("journalctl -u dovecot2 | grep -i warning >&2"); - }; + client.succeed("imap-mark-spam >&2") + server.wait_until_succeeds("journalctl -u dovecot2 | grep -i sa-learn-spam.sh >&2") + client.succeed("imap-mark-ham >&2") + server.wait_until_succeeds("journalctl -u dovecot2 | grep -i sa-learn-ham.sh >&2") + with subtest("no warnings or errors"): + server.fail("journalctl -u postfix | grep -i error >&2") + server.fail("journalctl -u postfix | grep -i warning >&2") + server.fail("journalctl -u dovecot2 | grep -i error >&2") + server.fail("journalctl -u dovecot2 | grep -i warning >&2") ''; } diff --git a/tests/intern.nix b/tests/intern.nix index f96643c..b4f42db 100644 --- a/tests/intern.nix +++ b/tests/intern.nix @@ -34,56 +34,57 @@ let ''; in -import (pkgs.path + "/nixos/tests/make-test.nix") { +pkgs.nixosTest { + name = "intern"; + nodes = { + machine = { config, pkgs, ... }: { + imports = [ + ./../default.nix + ./lib/config.nix + ]; - machine = - { config, pkgs, ... }: - { - imports = [ - ./../default.nix - ./lib/config.nix - ]; + virtualisation.memorySize = 1024; - virtualisation.memorySize = 1024; + mailserver = { + enable = true; + fqdn = "mail.example.com"; + domains = [ "example.com" ]; - mailserver = { - enable = true; - fqdn = "mail.example.com"; - domains = [ "example.com" ]; - - loginAccounts = { - "user1@example.com" = { - hashedPassword = "$6$/z4n8AQl6K$kiOkBTWlZfBd7PvF5GsJ8PmPgdZsFGN1jPGZufxxr60PoR0oUsrvzm2oQiflyz5ir9fFJ.d/zKm/NgLXNUsNX/"; - }; - "send-only@example.com" = { - hashedPasswordFile = hashPassword "send-only"; - sendOnly = true; - }; + loginAccounts = { + "user1@example.com" = { + hashedPassword = "$6$/z4n8AQl6K$kiOkBTWlZfBd7PvF5GsJ8PmPgdZsFGN1jPGZufxxr60PoR0oUsrvzm2oQiflyz5ir9fFJ.d/zKm/NgLXNUsNX/"; + }; + "send-only@example.com" = { + hashedPasswordFile = hashPassword "send-only"; + sendOnly = true; }; - - vmailGroupName = "vmail"; - vmailUID = 5000; }; + + vmailGroupName = "vmail"; + vmailUID = 5000; + }; }; + }; + testScript = '' + machine.start() + machine.wait_for_unit("multi-user.target") - testScript = - '' - $machine->start; - $machine->waitForUnit("multi-user.target"); + with subtest("vmail gid is set correctly"): + machine.succeed("getent group vmail | grep 5000") - subtest "vmail gid is set correctly", sub { - $machine->succeed("getent group vmail | grep 5000"); - }; + with subtest("mail to send only accounts is rejected"): + machine.wait_for_open_port(25) + # TODO put this blocking into the systemd units + machine.wait_until_succeeds( + "timeout 1 ${pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ $? -eq 124 ]" + ) + machine.succeed( + "cat ${sendMail} | ${pkgs.netcat-gnu}/bin/nc localhost 25 | grep -q 'This account cannot receive emails'" + ) - subtest "mail to send only accounts is rejected", sub { - $machine->waitForOpenPort(25); - # TODO put this blocking into the systemd units? - $machine->waitUntilSucceeds("timeout 1 ${pkgs.netcat}/bin/nc -U /run/rspamd/rspamd-milter.sock < /dev/null; [ \$? -eq 124 ]"); - $machine->succeed("cat ${sendMail} | ${pkgs.netcat-gnu}/bin/nc localhost 25 | grep -q 'This account cannot receive emails'" ); - }; - - subtest "rspamd controller serves web ui", sub { - $machine->succeed("${pkgs.curl}/bin/curl --unix-socket /run/rspamd/worker-controller.sock http://localhost/ | grep -q ''" ); - }; - ''; + with subtest("rspamd controller serves web ui"): + machine.succeed( + "${pkgs.curl}/bin/curl --unix-socket /run/rspamd/worker-controller.sock http://localhost/ | grep -q ''" + ) + ''; }