fts xapian: adapt to newer versions

fts xapian does not publish configuration changes in a changelog. As a
result, some options that nixos mailserver was setting for it have been
ignored for several years. New options (process_limit) are now
recommended. This adapts the module to these changes.

The default value of partial= is 2, but fts_xapian 1.8.3 now requires it
to be at least 3, and fails loudly in case it is 2. As a result, this
change is required to support fts_xapian 1.8.3 and later.

default.nix

@@ -395,12 +395,6 @@ in
         '';
       };
 
-      indexAttachments = mkOption {
-        type = types.bool;
-        default = false;
-        description = "Also index text-only attachements. Binary attachements are never indexed.";
-      };
-
       enforced = mkOption {
         type = types.enum [ "yes" "no" "body" ];
         default = "no";
@@ -413,14 +407,9 @@ in
       };
 
       minSize = mkOption {
-        type = types.int;
+        type = types.ints.between 3 1000;
-        default = 2;
+        default = 3;
-        description = "Size of the smallest n-gram to index.";
+        description = "Minimum size of search terms";
-      };
-      maxSize = mkOption {
-        type = types.int;
-        default = 20;
-        description = "Size of the largest n-gram to index.";
       };
       memoryLimit = mkOption {
         type = types.nullOr types.int;
@@ -1321,6 +1310,12 @@ in
   };
 
   imports = [
+    (lib.mkRemovedOptionModule [ "mailserver" "fullTextSearch" "maxSize" ] ''
+    This option is not needed since fts-xapian 1.8.3
+    '')
+    (lib.mkRemovedOptionModule [ "mailserver" "fullTextSearch" "indexAttachments" ] ''
+    Text attachments are always indexed since fts-xapian 1.4.8
+    '')
     ./mail-server/assertions.nix
     ./mail-server/borgbackup.nix
     ./mail-server/debug.nix

docs/fts.rst

@@ -20,8 +20,6 @@ To enable indexing for full text search here is an example configuration.
         enable = true;
         # index new email as they arrive
         autoIndex = true;
-        # this only applies to plain text attachments, binary attachments are never indexed
-        indexAttachments = true;
         enforced = "body";
       };
     };
@@ -61,8 +59,7 @@ Mitigating resources requirements
 
 You can:
 
-* disable indexation of attachements ``mailserver.fullTextSearch.indexAttachments = false``
+* increase the minimum search term size ``mailserver.fullTextSearch.minSize``
-* reduce the size of ngrams to be indexed ``mailserver.fullTextSearch.minSize`` and ``maxSize``
 * disable automatic indexation for some folders with
   ``mailserver.fullTextSearch.autoIndexExclude``.  Folders can be specified by
   name (``"Trash"``), by special use (``"\\Junk"``) or with a wildcard.

mail-server/dovecot.nix

@@ -352,7 +352,7 @@ in
         plugin {
           plugin = fts fts_xapian
           fts = xapian
-          fts_xapian = partial=${toString cfg.fullTextSearch.minSize} full=${toString cfg.fullTextSearch.maxSize} attachments=${bool2int cfg.fullTextSearch.indexAttachments} verbose=${bool2int cfg.debug}
+          fts_xapian = partial=${toString cfg.fullTextSearch.minSize} verbose=${bool2int cfg.debug}
 
           fts_autoindex = ${if cfg.fullTextSearch.autoIndex then "yes" else "no"}
 
@@ -361,11 +361,12 @@ in
           fts_enforced = ${cfg.fullTextSearch.enforced}
         }
 
-        ${lib.optionalString (cfg.fullTextSearch.memoryLimit != null) ''
         service indexer-worker {
+        ${lib.optionalString (cfg.fullTextSearch.memoryLimit != null) ''
           vsz_limit = ${toString (cfg.fullTextSearch.memoryLimit*1024*1024)}
-        }
         ''}
+          process_limit = 0
+        }
         ''}
 
         lda_mailbox_autosubscribe = yes

tests/external.nix

@@ -322,7 +322,7 @@ pkgs.nixosTest {
             Hello User1,
 
             this email contains the needle:
-            576a4565b70f5a4c1a0925cabdb587a6 
+            576a4565b70f5a4c1a0925cabdb587a6
           '';
           "root/email7".text = ''
             Message-ID: <1234578qwerty@host.local.network>
@@ -508,7 +508,7 @@ pkgs.nixosTest {
           server.fail("journalctl -u dovecot2 | grep -v 'imap-login: Debug: SSL error: Connection closed' | grep -i error >&2")
           # harmless ? https://dovecot.org/pipermail/dovecot/2020-August/119575.html
           server.fail(
-              "journalctl -u dovecot2 |grep -v 'Expunged message reappeared, giving a new UID'| grep -v 'FTS Xapian: Box is empty' | grep -i warning >&2"
+              "journalctl -u dovecot2 |grep -v 'Expunged message reappeared, giving a new UID'| grep -v 'FTS Xapian: Box is empty' | grep -vE 'FTS Xapian:.*does not exist. Creating it' | grep -i warning >&2"
           )
     '';
 }