diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..069abc3 --- /dev/null +++ b/.envrc @@ -0,0 +1,3 @@ +# shellcheck shell=bash + +use flake diff --git a/.gitignore b/.gitignore index b2be92b..0d3fe25 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ result +.direnv +.pre-commit-config.yaml diff --git a/README.md b/README.md index d351c1a..11ccf3c 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,82 @@ # ![Simple Nixos MailServer][logo] + ![license](https://img.shields.io/badge/license-GPL3-brightgreen.svg) [![pipeline status](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/badges/master/pipeline.svg)](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/commits/master) - ## Release branches For each NixOS release, we publish a branch. You then have to use the SNM branch corresponding to your NixOS version. * For NixOS 24.11 - - Use the [SNM branch `nixos-24.11`](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/tree/nixos-24.11) - - [Documentation](https://nixos-mailserver.readthedocs.io/en/nixos-24.11/) - - [Release notes](https://nixos-mailserver.readthedocs.io/en/nixos-24.11/release-notes.html#nixos-24-11) + * Use the [SNM branch `nixos-24.11`](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/tree/nixos-24.11) + * [Documentation](https://nixos-mailserver.readthedocs.io/en/nixos-24.11/) + * [Release notes](https://nixos-mailserver.readthedocs.io/en/nixos-24.11/release-notes.html#nixos-24-11) * For NixOS 24.05 - - Use the [SNM branch `nixos-24.05`](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/tree/nixos-24.05) - - [Documentation](https://nixos-mailserver.readthedocs.io/en/nixos-24.05/) - - [Release notes](https://nixos-mailserver.readthedocs.io/en/nixos-24.05/release-notes.html#nixos-24-05) + * Use the [SNM branch `nixos-24.05`](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/tree/nixos-24.05) + * [Documentation](https://nixos-mailserver.readthedocs.io/en/nixos-24.05/) + * [Release notes](https://nixos-mailserver.readthedocs.io/en/nixos-24.05/release-notes.html#nixos-24-05) * For NixOS unstable - - Use the [SNM branch `master`](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/tree/master) - - [Documentation](https://nixos-mailserver.readthedocs.io/en/latest/) + * Use the [SNM branch `master`](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/tree/master) + * [Documentation](https://nixos-mailserver.readthedocs.io/en/latest/) ## Features - * [x] Continous Integration Testing - * [x] Multiple Domains - * Postfix - - [x] SMTP on port 25 - - [x] Submission TLS on port 465 - - [x] Submission StartTLS on port 587 - - [x] LMTP with Dovecot - * Dovecot - - [x] Maildir folders - - [x] IMAP with TLS on port 993 - - [x] POP3 with TLS on port 995 - - [x] IMAP with StartTLS on port 143 - - [x] POP3 with StartTLS on port 110 - * Certificates - - [x] ACME - - [x] Custom certificates - * Spam Filtering - - [x] Via Rspamd - * Virus Scanning - - [x] Via ClamAV - * DKIM Signing - - [x] Via Rspamd - * User Management - - [x] Declarative user management - - [x] Declarative password management - - [x] LDAP users - * Sieve - - [x] Allow user defined sieve scripts - - [x] Moving mails from/to junk trains the Bayes filter - - [x] ManageSieve support - * User Aliases - - [x] Regular aliases - - [x] Catch all aliases +* [x] Continous Integration Testing +* [x] Multiple Domains +* Postfix + * [x] SMTP on port 25 + * [x] Submission TLS on port 465 + * [x] Submission StartTLS on port 587 + * [x] LMTP with Dovecot +* Dovecot + * [x] Maildir folders + * [x] IMAP with TLS on port 993 + * [x] POP3 with TLS on port 995 + * [x] IMAP with StartTLS on port 143 + * [x] POP3 with StartTLS on port 110 +* Certificates + * [x] ACME + * [x] Custom certificates +* Spam Filtering + * [x] Via Rspamd +* Virus Scanning + * [x] Via ClamAV +* DKIM Signing + * [x] Via Rspamd +* User Management + * [x] Declarative user management + * [x] Declarative password management + * [x] LDAP users +* Sieve + * [x] Allow user defined sieve scripts + * [x] Moving mails from/to junk trains the Bayes filter + * [x] ManageSieve support +* User Aliases + * [x] Regular aliases + * [x] Catch all aliases ### In the future - * Automatic client configuration - - [ ] [Autoconfig](https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) - - [ ] [Autodiscovery](https://learn.microsoft.com/en-us/exchange/architecture/client-access/autodiscover?view=exchserver-2019) - - [ ] [Mobileconfig](https://support.apple.com/guide/profile-manager/distribute-profiles-manually-pmdbd71ebc9/mac) - * DKIM Signing - - [ ] Allow per domain selectors - - [ ] Allow passing DKIM signing keys - * Improve the Forwarding Experience - - [ ] Support [ARC](https://en.wikipedia.org/wiki/Authenticated_Received_Chain) signing with [Rspamd](https://rspamd.com/doc/modules/arc.html) - - [ ] Support [SRS](https://en.wikipedia.org/wiki/Sender_Rewriting_Scheme) with [postsrsd](https://github.com/roehling/postsrsd) - * User management - - [ ] Allow local and LDAP user to coexist - * OpenID Connect - - Depends on relevant clients adding support, e.g. [Thunderbird](https://bugzilla.mozilla.org/show_bug.cgi?id=1602166) +* Automatic client configuration + * [ ] [Autoconfig](https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) + * [ ] [Autodiscovery](https://learn.microsoft.com/en-us/exchange/architecture/client-access/autodiscover?view=exchserver-2019) + * [ ] [Mobileconfig](https://support.apple.com/guide/profile-manager/distribute-profiles-manually-pmdbd71ebc9/mac) +* DKIM Signing + * [ ] Allow per domain selectors + * [ ] Allow passing DKIM signing keys +* Improve the Forwarding Experience + * [ ] Support [ARC](https://en.wikipedia.org/wiki/Authenticated_Received_Chain) signing with [Rspamd](https://rspamd.com/doc/modules/arc.html) + * [ ] Support [SRS](https://en.wikipedia.org/wiki/Sender_Rewriting_Scheme) with [postsrsd](https://github.com/roehling/postsrsd) +* User management + * [ ] Allow local and LDAP user to coexist +* OpenID Connect + * Depends on relevant clients adding support, e.g. [Thunderbird](https://bugzilla.mozilla.org/show_bug.cgi?id=1602166) ### Get in touch -- Matrix: [#nixos-mailserver:nixos.org](https://matrix.to/#/#nixos-mailserver:nixos.org) -- IRC: `#nixos-mailserver` on [Libera Chat](https://libera.chat/guides/connect) +* Matrix: [#nixos-mailserver:nixos.org](https://matrix.to/#/#nixos-mailserver:nixos.org) +* IRC: `#nixos-mailserver` on [Libera Chat](https://libera.chat/guides/connect) ## How to Set Up a 10/10 Mail Server Guide @@ -89,16 +89,18 @@ For a complete list of options, [see in readthedocs](https://nixos-mailserver.re See the [How to Develop SNM](https://nixos-mailserver.readthedocs.io/en/latest/howto-develop.html) documentation page. ## Contributors + See the [contributor tab](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/graphs/master) ### Alternative Implementations - * [NixCloud Webservices](https://github.com/nixcloud/nixcloud-webservices) + +* [NixCloud Webservices](https://github.com/nixcloud/nixcloud-webservices) ### Credits - * send mail graphic by [tnp_dreamingmao](https://thenounproject.com/dreamingmao) + +* send mail graphic by [tnp_dreamingmao](https://thenounproject.com/dreamingmao) from [TheNounProject](https://thenounproject.com/) is licensed under [CC BY 3.0](http://creativecommons.org/~/3.0/) - * Logo made with [Logomakr.com](https://logomakr.com) - +* Logo made with [Logomakr.com](https://logomakr.com) [logo]: docs/logo.png diff --git a/docs/conf.py b/docs/conf.py index 1845917..7bc771b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,9 +17,9 @@ # -- Project information ----------------------------------------------------- -project = 'NixOS Mailserver' -copyright = '2022, NixOS Mailserver Contributors' -author = 'NixOS Mailserver Contributors' +project = "NixOS Mailserver" +copyright = "2022, NixOS Mailserver Contributors" +author = "NixOS Mailserver Contributors" # -- General configuration --------------------------------------------------- @@ -27,33 +27,31 @@ author = 'NixOS Mailserver Contributors' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ - 'myst_parser' -] +extensions = ["myst_parser"] myst_enable_extensions = [ - 'colon_fence', - 'linkify', + "colon_fence", + "linkify", ] smartquotes = False # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -master_doc = 'index' +master_doc = "index" # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/flakes.rst b/docs/flakes.rst index 254a02a..f56ec96 100644 --- a/docs/flakes.rst +++ b/docs/flakes.rst @@ -1,7 +1,7 @@ Nix Flakes ========== -If you're using `flakes `__, you can use +If you're using `flakes `__, you can use the following minimal ``flake.nix`` as an example: .. code:: nix diff --git a/docs/howto-develop.rst b/docs/howto-develop.rst index acdc7bd..40527f9 100644 --- a/docs/howto-develop.rst +++ b/docs/howto-develop.rst @@ -4,13 +4,33 @@ Contribute or troubleshoot To report an issue, please go to ``_. -You can also chat with us on the Libera IRC channel ``#nixos-mailserver``. +If you have questions, feel free to reach out: + +* Matrix: `#nixos-mailserver:nixos.org `__ +* IRC: `#nixos-mailserver `__ on `Libera Chat `__ + +All our workflows rely on Nix being configured with `Flakes `__. + +Development Shell +----------------- + +We provide a `flake.nix` devshell that automatically sets up pre-commit hooks, +which allows for fast feedback cycles when making changes to the repository. + + +:: + + $ nix develop + + +We recommend setting up `direnv `__ to automatically +attach to the development environment when entering the project directories. Run NixOS tests --------------- To run the test suite, you need to enable `Nix Flakes -`_. +`__. You can then run the testsuite via @@ -37,7 +57,7 @@ For the syntax, see the `RST/Sphinx primer `_. To build the documentation, you need to enable `Nix Flakes -`_. +`__. :: diff --git a/flake.lock b/flake.lock index d79f675..a712d89 100644 --- a/flake.lock +++ b/flake.lock @@ -32,6 +32,51 @@ "type": "github" } }, + "git-hooks": { + "inputs": { + "flake-compat": [ + "flake-compat" + ], + "gitignore": "gitignore", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1742649964, + "narHash": "sha256-DwOTp7nvfi8mRfuL1escHDXabVXFGT1VlPD1JHrtrco=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "dcf5072734cb576d2b0c59b2ac44f5050b5eac82", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1747179050, @@ -68,6 +113,7 @@ "inputs": { "blobs": "blobs", "flake-compat": "flake-compat", + "git-hooks": "git-hooks", "nixpkgs": "nixpkgs", "nixpkgs-24_11": "nixpkgs-24_11" } diff --git a/flake.nix b/flake.nix index dac72f8..96ffb21 100644 --- a/flake.nix +++ b/flake.nix @@ -3,9 +3,15 @@ inputs = { flake-compat = { + # for shell.nix compat url = "github:edolstra/flake-compat"; flake = false; }; + git-hooks = { + url = "github:cachix/git-hooks.nix"; + inputs.flake-compat.follows = "flake-compat"; + inputs.nixpkgs.follows = "nixpkgs"; + }; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; nixpkgs-24_11.url = "github:NixOS/nixpkgs/nixos-24.11"; blobs = { @@ -14,7 +20,7 @@ }; }; - outputs = { self, blobs, nixpkgs, nixpkgs-24_11, ... }: let + outputs = { self, blobs, git-hooks, nixpkgs, nixpkgs-24_11, ... }: let lib = nixpkgs.lib; system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; @@ -90,6 +96,7 @@ in pkgs.runCommand "options.md" { buildInputs = [pkgs.python3Minimal]; } '' echo "Generating options.md from ${options}" python ${./scripts/generate-options.py} ${options} > $out + echo $out ''; documentation = pkgs.stdenv.mkDerivation { @@ -122,16 +129,62 @@ nixosModule = self.nixosModules.default; # compatibility hydraJobs.${system} = allTests // { inherit documentation; + inherit (self.checks.${system}) pre-commit; + }; + checks.${system} = allTests // { + pre-commit = git-hooks.lib.${system}.run { + src = ./.; + hooks = { + # docs + markdownlint = { + enable = true; + settings.configuration = { + # Max line length, doesn't seem to correclty account for lines containing links + # https://github.com/DavidAnson/markdownlint/blob/main/doc/md013.md + MD013 = false; + }; + }; + rstcheck = { + enable = true; + entry = lib.getExe pkgs.rstcheckWithSphinx; + files = "\\.rst$"; + }; + + # nix + deadnix.enable = true; + + # python + pyright.enable = true; + ruff = { + enable = true; + args = [ + "--extend-select" + "I" + ]; + }; + ruff-format.enable = true; + + # scripts + shellcheck.enable = true; + + # sieve + check-sieve = { + enable = true; + entry = lib.getExe pkgs.check-sieve; + files = "\\.sieve$"; + }; + }; + }; }; - checks.${system} = allTests; packages.${system} = { inherit optionsDoc documentation; }; - devShells.${system}.default = pkgs.mkShell { + devShells.${system}.default = pkgs.mkShellNoCC { inputsFrom = [ documentation ]; packages = with pkgs; [ - clamav - ]; + glab + ] ++ self.checks.${system}.pre-commit.enabledPackages; + shellHook = self.checks.${system}.pre-commit.shellHook; }; devShell.${system} = self.devShells.${system}.default; # compatibility }; diff --git a/mail-server/clamav.nix b/mail-server/clamav.nix index 25418f0..0dafd4f 100644 --- a/mail-server/clamav.nix +++ b/mail-server/clamav.nix @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see -{ config, pkgs, lib, options, ... }: +{ config, lib, ... }: let cfg = config.mailserver; diff --git a/mail-server/dovecot.nix b/mail-server/dovecot.nix index 27af741..6704426 100644 --- a/mail-server/dovecot.nix +++ b/mail-server/dovecot.nix @@ -39,27 +39,6 @@ let ); postfixCfg = config.services.postfix; - dovecot2Cfg = config.services.dovecot2; - - stateDir = "/var/lib/dovecot"; - - pipeBin = pkgs.stdenv.mkDerivation { - name = "pipe_bin"; - src = ./dovecot/pipe_bin; - buildInputs = with pkgs; [ makeWrapper coreutils bash rspamd ]; - buildCommand = '' - mkdir -p $out/pipe/bin - cp $src/* $out/pipe/bin/ - chmod a+x $out/pipe/bin/* - patchShebangs $out/pipe/bin - - for file in $out/pipe/bin/*; do - wrapProgram $file \ - --set PATH "${pkgs.coreutils}/bin:${pkgs.rspamd}/bin" - done - ''; - }; - ldapConfig = pkgs.writeTextFile { name = "dovecot-ldap.conf.ext.template"; @@ -109,7 +88,7 @@ let # Prevent world-readable password files, even temporarily. umask 077 - for f in ${builtins.toString (lib.mapAttrsToList (name: value: passwordFiles."${name}") cfg.loginAccounts)}; do + for f in ${builtins.toString (lib.mapAttrsToList (name: _: passwordFiles."${name}") cfg.loginAccounts)}; do if [ ! -f "$f" ]; then echo "Expected password hash file $f does not exist!" exit 1 @@ -117,7 +96,7 @@ let done cat < ${passwdFile} - ${lib.concatStringsSep "\n" (lib.mapAttrsToList (name: value: + ${lib.concatStringsSep "\n" (lib.mapAttrsToList (name: _: "${name}:${"$(head -n 1 ${passwordFiles."${name}"})"}::::::" ) cfg.loginAccounts)} EOF @@ -130,7 +109,7 @@ let EOF ''; - junkMailboxes = builtins.attrNames (lib.filterAttrs (n: v: v ? "specialUse" && v.specialUse == "Junk") cfg.mailboxes); + junkMailboxes = builtins.attrNames (lib.filterAttrs (_: v: v ? "specialUse" && v.specialUse == "Junk") cfg.mailboxes); junkMailboxNumber = builtins.length junkMailboxes; # The assertion garantees there is exactly one Junk mailbox. junkMailboxName = if junkMailboxNumber == 1 then builtins.elemAt junkMailboxes 0 else ""; diff --git a/mail-server/dovecot/pipe_bin/sa-learn-ham.sh b/mail-server/dovecot/pipe_bin/sa-learn-ham.sh deleted file mode 100755 index 76fc4ed..0000000 --- a/mail-server/dovecot/pipe_bin/sa-learn-ham.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -o errexit -exec rspamc -h /run/rspamd/worker-controller.sock learn_ham \ No newline at end of file diff --git a/mail-server/dovecot/pipe_bin/sa-learn-spam.sh b/mail-server/dovecot/pipe_bin/sa-learn-spam.sh deleted file mode 100755 index 2a2f766..0000000 --- a/mail-server/dovecot/pipe_bin/sa-learn-spam.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -o errexit -exec rspamc -h /run/rspamd/worker-controller.sock learn_spam \ No newline at end of file diff --git a/mail-server/kresd.nix b/mail-server/kresd.nix index e3baa07..230bdea 100644 --- a/mail-server/kresd.nix +++ b/mail-server/kresd.nix @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see -{ config, pkgs, lib, ... }: +{ config, lib, ... }: let cfg = config.mailserver; diff --git a/mail-server/monit.nix b/mail-server/monit.nix index c69b19e..c3f8760 100644 --- a/mail-server/monit.nix +++ b/mail-server/monit.nix @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see -{ config, pkgs, lib, ... }: +{ config, lib, ... }: let cfg = config.mailserver; diff --git a/mail-server/postfix.nix b/mail-server/postfix.nix index db3e581..d1c59b2 100644 --- a/mail-server/postfix.nix +++ b/mail-server/postfix.nix @@ -25,7 +25,7 @@ let # Merge several lookup tables. A lookup table is a attribute set where # - the key is an address (user@example.com) or a domain (@example.com) # - the value is a list of addresses - mergeLookupTables = tables: lib.zipAttrsWith (n: v: lib.flatten v) tables; + mergeLookupTables = tables: lib.zipAttrsWith (_: v: lib.flatten v) tables; # valiases_postfix :: Map String [String] valiases_postfix = mergeLookupTables (lib.flatten (lib.mapAttrsToList @@ -123,13 +123,8 @@ let /^Message-ID:\s+<(.*?)@.*?>/ REPLACE Message-ID: <$1@${cfg.fqdn}> ''); - inetSocket = addr: port: "inet:[${toString port}@${addr}]"; - unixSocket = sock: "unix:${sock}"; - smtpdMilters = [ "unix:/run/rspamd/rspamd-milter.sock" ]; - policyd-spf = pkgs.writeText "policyd-spf.conf" cfg.policydSPFExtraConfig; - mappedFile = name: "hash:/var/lib/postfix/conf/${name}"; mappedRegexFile = name: "pcre:/var/lib/postfix/conf/${name}"; diff --git a/mail-server/rspamd.nix b/mail-server/rspamd.nix index fd94c84..0e37c20 100644 --- a/mail-server/rspamd.nix +++ b/mail-server/rspamd.nix @@ -50,7 +50,7 @@ in nativeBuildInputs = with pkgs; [ makeWrapper ]; }'' makeWrapper ${pkgs.rspamd}/bin/rspamc $out/bin/rspamc \ - --add-flags "-h /var/run/rspamd/worker-controller.sock" + --add-flags "-h /run/rspamd/worker-controller.sock" '') ]; diff --git a/scripts/generate-options.py b/scripts/generate-options.py index 75a25ae..e78e262 100644 --- a/scripts/generate-options.py +++ b/scripts/generate-options.py @@ -1,5 +1,7 @@ import json import sys +from textwrap import indent +from typing import Any, Mapping header = """ # Mailserver options @@ -21,62 +23,87 @@ template = """ f = open(sys.argv[1]) options = json.load(f) -groups = ["mailserver.loginAccounts", - "mailserver.certificate", - "mailserver.dkim", - "mailserver.dmarcReporting", - "mailserver.fullTextSearch", - "mailserver.redis", - "mailserver.ldap", - "mailserver.monitoring", - "mailserver.backup", - "mailserver.borgbackup"] +groups = [ + "mailserver.loginAccounts", + "mailserver.certificate", + "mailserver.dkim", + "mailserver.dmarcReporting", + "mailserver.fullTextSearch", + "mailserver.redis", + "mailserver.ldap", + "mailserver.monitoring", + "mailserver.backup", + "mailserver.borgbackup", +] -def render_option_value(opt, attr): - if attr in opt: - if isinstance(opt[attr], dict) and '_type' in opt[attr]: - if opt[attr]['_type'] == 'literalExpression': - if '\n' in opt[attr]['text']: - res = '\n```nix\n' + opt[attr]['text'].rstrip('\n') + '\n```' - else: - res = '```{}```'.format(opt[attr]['text']) - elif opt[attr]['_type'] == 'literalMD': - res = opt[attr]['text'] - else: - s = str(opt[attr]) - if s == "": - res = '`""`' - elif '\n' in s: - res = '\n```\n' + s.rstrip('\n') + '\n```' - else: - res = '```{}```'.format(s) - res = '- ' + attr + ': ' + res - else: - res = "" - return res -def print_option(opt): - if isinstance(opt['description'], dict) and '_type' in opt['description']: # mdDoc - description = opt['description']['text'] +def md_literal(value: str) -> str: + return f"`{value}`" + + +def md_codefence(value: str, language: str = "nix") -> str: + return indent( + f"\n```{language}\n{value}\n```", + prefix=2 * " ", + ) + + +def render_option_value(option: Mapping[str, Any], key: str) -> str: + if key not in option: + return "" + + if isinstance(option[key], dict) and "_type" in option[key]: + if option[key]["_type"] == "literalExpression": + # multi-line codeblock + if "\n" in option[key]["text"]: + text = option[key]["text"].rstrip("\n") + value = md_codefence(text) + # inline codeblock + else: + value = md_literal(option[key]["text"]) + # literal markdown + elif option[key]["_type"] == "literalMD": + value = option[key]["text"] + else: + assert RuntimeError(f"Unhandled option type {option[key]['_type']}") else: - description = opt['description'] - print(template.format( - key=opt['name'], - description=description or "", - type="- type: ```{}```".format(opt['type']), - default=render_option_value(opt, 'default'), - example=render_option_value(opt, 'example'))) + text = str(option[key]) + if text == "": + value = md_literal('""') + elif "\n" in text: + value = md_codefence(text.rstrip("\n")) + else: + value = md_literal(text) + + return f"- {key}: {value}" # type: ignore + + +def print_option(option): + if ( + isinstance(option["description"], dict) and "_type" in option["description"] + ): # mdDoc + description = option["description"]["text"] + else: + description = option["description"] + print( + template.format( + key=option["name"], + description=description or "", + type=f"- type: {md_literal(option['type'])}", + default=render_option_value(option, "default"), + example=render_option_value(option, "example"), + ) + ) print(header) for opt in options: - if any([opt['name'].startswith(c) for c in groups]): + if any([opt["name"].startswith(c) for c in groups]): continue print_option(opt) for c in groups: - print('## `{}`'.format(c)) - print() + print(f"## `{c}`\n") for opt in options: - if opt['name'].startswith(c): + if opt["name"].startswith(c): print_option(opt) diff --git a/scripts/mail-check.py b/scripts/mail-check.py index 7d3935c..39b2688 100644 --- a/scripts/mail-check.py +++ b/scripts/mail-check.py @@ -1,32 +1,31 @@ -import smtplib, sys import argparse -import os -import uuid -import imaplib -from datetime import datetime, timedelta import email import email.utils +import imaplib +import smtplib import time +import uuid +from datetime import datetime, timedelta +from typing import cast RETRY = 100 -def _send_mail(smtp_host, smtp_port, smtp_username, from_addr, from_pwd, to_addr, subject, starttls): - print("Sending mail with subject '{}'".format(subject)) - message = "\n".join([ - "From: {from_addr}", - "To: {to_addr}", - "Subject: {subject}", - "Message-ID: {random}@mail-check.py", - "Date: {date}", - "", - "This validates our mail server can send to Gmail :/"]).format( - from_addr=from_addr, - to_addr=to_addr, - subject=subject, - random=str(uuid.uuid4()), - date=email.utils.formatdate(), - ) +def _send_mail( + smtp_host, smtp_port, smtp_username, from_addr, from_pwd, to_addr, subject, starttls +): + print(f"Sending mail with subject '{subject}'") + message = "\n".join( + [ + f"From: {from_addr}", + f"To: {to_addr}", + f"Subject: {subject}", + f"Message-ID: {uuid.uuid4()}@mail-check.py", + f"Date: {email.utils.formatdate()}", + "", + "This validates our mail server can send to Gmail :/", + ] + ) retry = RETRY while True: @@ -43,7 +42,9 @@ def _send_mail(smtp_host, smtp_port, smtp_username, from_addr, from_pwd, to_addr except smtplib.SMTPResponseException as e: if e.smtp_code == 451: # service unavailable error print(e) - elif e.smtp_code == 454: # smtplib.SMTPResponseException: (454, b'4.3.0 Try again later') + elif ( + e.smtp_code == 454 + ): # smtplib.SMTPResponseException: (454, b'4.3.0 Try again later') print(e) else: raise @@ -61,16 +62,18 @@ def _send_mail(smtp_host, smtp_port, smtp_username, from_addr, from_pwd, to_addr print("Retry attempts exhausted") exit(5) + def _read_mail( - imap_host, - imap_port, - imap_username, - to_pwd, - subject, - ignore_dkim_spf, - show_body=False, - delete=True): - print("Reading mail from %s" % imap_username) + imap_host, + imap_port, + imap_username, + to_pwd, + subject, + ignore_dkim_spf, + show_body=False, + delete=True, +): + print("Reading mail from {imap_username}") message = None @@ -80,49 +83,62 @@ def _read_mail( today = datetime.today() cutoff = today - timedelta(days=1) - dt = cutoff.strftime('%d-%b-%Y') + dt = cutoff.strftime("%d-%b-%Y") for _ in range(0, RETRY): print("Retrying") obj.select() - typ, data = obj.search(None, '(SINCE %s) (SUBJECT "%s")'%(dt, subject)) - if data == [b'']: + _, data = obj.search(None, f'(SINCE {dt}) (SUBJECT "{subject}")') + if data == [b""]: time.sleep(1) continue uids = data[0].decode("utf-8").split(" ") if len(uids) != 1: - print("Warning: %d messages have been found with subject containing %s " % (len(uids), subject)) + print( + f"Warning: {len(uids)} messages have been found with subject containing {subject}" + ) # FIXME: we only consider the first matching message... uid = uids[0] - _, raw = obj.fetch(uid, '(RFC822)') + _, raw = obj.fetch(uid, "(RFC822)") if delete: - obj.store(uid, '+FLAGS', '\\Deleted') + obj.store(uid, "+FLAGS", "\\Deleted") obj.expunge() - message = email.message_from_bytes(raw[0][1]) - print("Message with subject '%s' has been found" % message['subject']) + assert raw[0] and raw[0][1] + message = email.message_from_bytes(cast(bytes, raw[0][1])) + print(f"Message with subject '{message['subject']}' has been found") if show_body: - for m in message.get_payload(): - if m.get_content_type() == 'text/plain': - print("Body:\n%s" % m.get_payload(decode=True).decode('utf-8')) + if message.is_multipart(): + for part in message.walk(): + ctype = part.get_content_type() + if ctype == "text/plain": + body = cast(bytes, part.get_payload(decode=True)).decode() + print(f"Body:\n{body}") + else: + print(f"Body with content type {ctype} not printed") + else: + body = cast(bytes, message.get_payload(decode=True)).decode() + print(f"Body:\n{body}") break if message is None: - print("Error: no message with subject '%s' has been found in INBOX of %s" % (subject, imap_username)) + print( + f"Error: no message with subject '{subject}' has been found in INBOX of {imap_username}" + ) exit(1) if ignore_dkim_spf: return # gmail set this standardized header - if 'ARC-Authentication-Results' in message: - if "dkim=pass" in message['ARC-Authentication-Results']: + if "ARC-Authentication-Results" in message: + if "dkim=pass" in message["ARC-Authentication-Results"]: print("DKIM ok") else: print("Error: no DKIM validation found in message:") print(message.as_string()) exit(2) - if "spf=pass" in message['ARC-Authentication-Results']: + if "spf=pass" in message["ARC-Authentication-Results"]: print("SPF ok") else: print("Error: no SPF validation found in message:") @@ -132,71 +148,108 @@ def _read_mail( print("DKIM and SPF verification failed") exit(4) + def send_and_read(args): src_pwd = None if args.src_password_file is not None: src_pwd = args.src_password_file.readline().rstrip() dst_pwd = args.dst_password_file.readline().rstrip() - if args.imap_username != '': + if args.imap_username != "": imap_username = args.imap_username else: imap_username = args.to_addr - subject = "{}".format(uuid.uuid4()) + subject = f"{uuid.uuid4()}" - _send_mail(smtp_host=args.smtp_host, - smtp_port=args.smtp_port, - smtp_username=args.smtp_username, - from_addr=args.from_addr, - from_pwd=src_pwd, - to_addr=args.to_addr, - subject=subject, - starttls=args.smtp_starttls) + _send_mail( + smtp_host=args.smtp_host, + smtp_port=args.smtp_port, + smtp_username=args.smtp_username, + from_addr=args.from_addr, + from_pwd=src_pwd, + to_addr=args.to_addr, + subject=subject, + starttls=args.smtp_starttls, + ) + + _read_mail( + imap_host=args.imap_host, + imap_port=args.imap_port, + imap_username=imap_username, + to_pwd=dst_pwd, + subject=subject, + ignore_dkim_spf=args.ignore_dkim_spf, + ) - _read_mail(imap_host=args.imap_host, - imap_port=args.imap_port, - imap_username=imap_username, - to_pwd=dst_pwd, - subject=subject, - ignore_dkim_spf=args.ignore_dkim_spf) def read(args): - _read_mail(imap_host=args.imap_host, - imap_port=args.imap_port, - to_addr=args.imap_username, - to_pwd=args.imap_password, - subject=args.subject, - ignore_dkim_spf=args.ignore_dkim_spf, - show_body=args.show_body, - delete=False) + _read_mail( + imap_host=args.imap_host, + imap_port=args.imap_port, + imap_username=args.imap_username, + to_pwd=args.imap_password, + subject=args.subject, + ignore_dkim_spf=args.ignore_dkim_spf, + show_body=args.show_body, + delete=False, + ) + parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() -parser_send_and_read = subparsers.add_parser('send-and-read', description="Send a email with a subject containing a random UUID and then try to read this email from the recipient INBOX.") -parser_send_and_read.add_argument('--smtp-host', type=str) -parser_send_and_read.add_argument('--smtp-port', type=str, default=25) -parser_send_and_read.add_argument('--smtp-starttls', action='store_true') -parser_send_and_read.add_argument('--smtp-username', type=str, default='', help="username used for smtp login. If not specified, the from-addr value is used") -parser_send_and_read.add_argument('--from-addr', type=str) -parser_send_and_read.add_argument('--imap-host', required=True, type=str) -parser_send_and_read.add_argument('--imap-port', type=str, default=993) -parser_send_and_read.add_argument('--to-addr', type=str, required=True) -parser_send_and_read.add_argument('--imap-username', type=str, default='', help="username used for imap login. If not specified, the to-addr value is used") -parser_send_and_read.add_argument('--src-password-file', type=argparse.FileType('r')) -parser_send_and_read.add_argument('--dst-password-file', required=True, type=argparse.FileType('r')) -parser_send_and_read.add_argument('--ignore-dkim-spf', action='store_true', help="to ignore the dkim and spf verification on the read mail") +parser_send_and_read = subparsers.add_parser( + "send-and-read", + description="Send a email with a subject containing a random UUID and then try to read this email from the recipient INBOX.", +) +parser_send_and_read.add_argument("--smtp-host", type=str) +parser_send_and_read.add_argument("--smtp-port", type=str, default=25) +parser_send_and_read.add_argument("--smtp-starttls", action="store_true") +parser_send_and_read.add_argument( + "--smtp-username", + type=str, + default="", + help="username used for smtp login. If not specified, the from-addr value is used", +) +parser_send_and_read.add_argument("--from-addr", type=str) +parser_send_and_read.add_argument("--imap-host", required=True, type=str) +parser_send_and_read.add_argument("--imap-port", type=str, default=993) +parser_send_and_read.add_argument("--to-addr", type=str, required=True) +parser_send_and_read.add_argument( + "--imap-username", + type=str, + default="", + help="username used for imap login. If not specified, the to-addr value is used", +) +parser_send_and_read.add_argument("--src-password-file", type=argparse.FileType("r")) +parser_send_and_read.add_argument( + "--dst-password-file", required=True, type=argparse.FileType("r") +) +parser_send_and_read.add_argument( + "--ignore-dkim-spf", + action="store_true", + help="to ignore the dkim and spf verification on the read mail", +) parser_send_and_read.set_defaults(func=send_and_read) -parser_read = subparsers.add_parser('read', description="Search for an email with a subject containing 'subject' in the INBOX.") -parser_read.add_argument('--imap-host', type=str, default="localhost") -parser_read.add_argument('--imap-port', type=str, default=993) -parser_read.add_argument('--imap-username', required=True, type=str) -parser_read.add_argument('--imap-password', required=True, type=str) -parser_read.add_argument('--ignore-dkim-spf', action='store_true', help="to ignore the dkim and spf verification on the read mail") -parser_read.add_argument('--show-body', action='store_true', help="print mail text/plain payload") -parser_read.add_argument('subject', type=str) +parser_read = subparsers.add_parser( + "read", + description="Search for an email with a subject containing 'subject' in the INBOX.", +) +parser_read.add_argument("--imap-host", type=str, default="localhost") +parser_read.add_argument("--imap-port", type=str, default=993) +parser_read.add_argument("--imap-username", required=True, type=str) +parser_read.add_argument("--imap-password", required=True, type=str) +parser_read.add_argument( + "--ignore-dkim-spf", + action="store_true", + help="to ignore the dkim and spf verification on the read mail", +) +parser_read.add_argument( + "--show-body", action="store_true", help="print mail text/plain payload" +) +parser_read.add_argument("subject", type=str) parser_read.set_defaults(func=read) args = parser.parse_args()