dovecot: Add spam filter traning using imapsieve

mail-server/dovecot.nix

@@ -27,6 +27,26 @@ let
   dovecotMaildir = "maildir:${cfg.mailDirectory}/%d/%n${maildirLayoutAppendix}";
 
   postfixCfg = config.services.postfix;
+  dovecot2Cfg = config.services.dovecot2;
+
+  stateDir = "/var/lib/dovecot";
+
+  pipeBin = pkgs.stdenv.mkDerivation {
+    name = "pipe_bin";
+    src = ./dovecot/pipe_bin;
+    buildInputs = with pkgs; [ makeWrapper coreutils bash rspamd ];
+    buildCommand = ''
+      mkdir -p $out/pipe/bin
+      cp $src/* $out/pipe/bin/
+      chmod a+x $out/pipe/bin/*
+      patchShebangs $out/pipe/bin
+
+      for file in $out/pipe/bin/*; do
+        wrapProgram $file \
+          --set PATH "${pkgs.coreutils}/bin:${pkgs.rspamd}/bin"
+      done
+    '';
+  };
 in
 {
   config = with cfg; lib.mkIf enable {
@@ -68,6 +88,7 @@ in
 
         protocol imap {
           mail_max_userip_connections = ${toString cfg.maxConnectionsPerUser}
+          mail_plugins = $mail_plugins imap_sieve
         }
 
         protocol pop3 {
@@ -118,14 +139,40 @@ in
         }
 
         plugin {
+          sieve_plugins = sieve_imapsieve sieve_extprograms
           sieve = file:/var/sieve/%u/scripts;active=/var/sieve/%u/active.sieve
           sieve_default = file:/var/sieve/%u/default.sieve
           sieve_default_name = default
+
+          # From elsewhere to Spam folder
+          imapsieve_mailbox1_name = Junk
+          imapsieve_mailbox1_causes = COPY
+          imapsieve_mailbox1_before = file:${stateDir}/imap_sieve/report-spam.sieve
+
+          # From Spam folder to elsewhere
+          imapsieve_mailbox2_name = *
+          imapsieve_mailbox2_from = Junk
+          imapsieve_mailbox2_causes = COPY
+          imapsieve_mailbox2_before = file:${stateDir}/imap_sieve/report-ham.sieve
+
+          sieve_pipe_bin_dir = ${pipeBin}/pipe/bin
+
+          sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.environment
         }
 
         lda_mailbox_autosubscribe = yes
         lda_mailbox_autocreate = yes
       '';
     };
+
+    systemd.services.dovecot2.preStart = ''
+      rm -rf '${stateDir}/imap_sieve'
+      mkdir '${stateDir}/imap_sieve'
+      cp -p "${./dovecot/imap_sieve}"/*.sieve '${stateDir}/imap_sieve/'
+      for k in "${stateDir}/imap_sieve"/*.sieve ; do
+        ${pkgs.dovecot_pigeonhole}/bin/sievec "$k"
+      done
+      chown -R '${dovecot2Cfg.mailUser}:${dovecot2Cfg.mailGroup}' '${stateDir}/imap_sieve'
+    '';
   };
 }

mail-server/dovecot/imap_sieve/report-ham.sieve

@@ -0,0 +1,15 @@
+require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"];
+
+if environment :matches "imap.mailbox" "*" {
+  set "mailbox" "${1}";
+}
+
+if string "${mailbox}" "Trash" {
+  stop;
+}
+
+if environment :matches "imap.user" "*" {
+  set "username" "${1}";
+}
+
+pipe :copy "sa-learn-ham.sh" [ "${username}" ];

mail-server/dovecot/imap_sieve/report-spam.sieve

@@ -0,0 +1,7 @@
+require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"];
+
+if environment :matches "imap.user" "*" {
+  set "username" "${1}";
+}
+
+pipe :copy "sa-learn-spam.sh" [ "${username}" ];

mail-server/dovecot/pipe_bin/sa-learn-ham.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -o errexit
+exec rspamc -h /run/rspamd/worker-controller.sock learn_ham

mail-server/dovecot/pipe_bin/sa-learn-spam.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -o errexit
+exec rspamc -h /run/rspamd/worker-controller.sock learn_spam

mail-server/rspamd.nix

@@ -61,6 +61,16 @@ in
           }
         '';
       };
+      workers.controller = {
+        type = "controller";
+        count = 1;
+        bindSockets = [{
+          socket = "/run/rspamd/worker-controller.sock";
+          mode = "0666";
+        }];
+        includes = [];
+      };
+
     };
     systemd.services.rspamd = {
       requires = (lib.optional cfg.virusScanning "clamav-daemon.service");

tests/extern.nix

@@ -64,6 +64,7 @@ import <nixpkgs/nixos/tests/make-test.nix> {
               };
 
               enableImap = true;
+              enableImapSsl = true;
             };
         };
       client = { nodes, config, pkgs, ... }: let
@@ -79,9 +80,63 @@ import <nixpkgs/nixos/tests/make-test.nix> {
           echo grep '^Message-ID:.*@mail.example.com>$' "$@" >&2
           exec grep '^Message-ID:.*@mail.example.com>$' "$@"
         '';
+        test-imap-spam = pkgs.writeScriptBin "imap-mark-spam" ''
+          #!${pkgs.python3.interpreter}
+          import imaplib
+
+          with imaplib.IMAP4_SSL('${serverIP}') as imap:
+            imap.login('user1@example.com', 'user1')
+            imap.select()
+            status, [response] = imap.search(None, 'ALL')
+            msg_ids = response.decode("utf-8").split(' ')
+            print(msg_ids)
+            assert status == 'OK'
+            assert len(msg_ids) == 1
+
+            imap.copy(','.join(msg_ids), 'Junk')
+            for num in msg_ids:
+              imap.store(num, '+FLAGS', '\\Deleted')
+            imap.expunge()
+
+            imap.select('Junk')
+            status, [response] = imap.search(None, 'ALL')
+            msg_ids = response.decode("utf-8").split(' ')
+            print(msg_ids)
+            assert status == 'OK'
+            assert len(msg_ids) == 1
+
+            imap.close()
+        '';
+        test-imap-ham = pkgs.writeScriptBin "imap-mark-ham" ''
+          #!${pkgs.python3.interpreter}
+          import imaplib
+
+          with imaplib.IMAP4_SSL('${serverIP}') as imap:
+            imap.login('user1@example.com', 'user1')
+            imap.select('Junk')
+            status, [response] = imap.search(None, 'ALL')
+            msg_ids = response.decode("utf-8").split(' ')
+            print(msg_ids)
+            assert status == 'OK'
+            assert len(msg_ids) == 1
+
+            imap.copy(','.join(msg_ids), 'INBOX')
+            for num in msg_ids:
+              imap.store(num, '+FLAGS', '\\Deleted')
+            imap.expunge()
+
+            imap.select('INBOX')
+            status, [response] = imap.search(None, 'ALL')
+            msg_ids = response.decode("utf-8").split(' ')
+            print(msg_ids)
+            assert status == 'OK'
+            assert len(msg_ids) == 1
+
+            imap.close()
+        '';
       in {
         environment.systemPackages = with pkgs; [
-          fetchmail msmtp procmail findutils grep-ip check-mail-id
+          fetchmail msmtp procmail findutils grep-ip check-mail-id test-imap-spam test-imap-ham
         ];
         environment.etc = {
           "root/.fetchmailrc" = {
@@ -325,6 +380,18 @@ import <nixpkgs/nixos/tests/make-test.nix> {
 
       };
 
+      subtest "imap sieve junk trainer", sub {
+          # send email from user2 to user1
+          $client->succeed("msmtp -a test --tls=on --tls-certcheck=off --auth=on user1\@example.com < /etc/root/email1 >&2");
+          # give the mail server some time to process the mail
+          $server->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]');
+
+          $client->succeed("imap-mark-spam >&2");
+          $server->waitUntilSucceeds("journalctl -u dovecot2 | grep -i sa-learn-spam.sh >&2");
+          $client->succeed("imap-mark-ham >&2");
+          $server->waitUntilSucceeds("journalctl -u dovecot2 | grep -i sa-learn-ham.sh >&2");
+      };
+
       subtest "no warnings or errors", sub {
           $server->fail("journalctl -u postfix | grep -i error >&2");
           $server->fail("journalctl -u postfix | grep -i warning >&2");