{ system ? builtins.currentSystem , config ? {} , pkgs ? import ../.. { inherit system config; } }: let inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest; inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge removeSuffix replaceChars singleton splitString; /* * The attrset `exporterTests` contains one attribute * for each exporter test. Each of these attributes * is expected to be an attrset containing: * * `exporterConfig`: * this attribute set contains config for the exporter itself * * `exporterTest` * this attribute set contains test instructions * * `metricProvider` (optional) * this attribute contains additional machine config * * `nodeName` (optional) * override an incompatible testnode name * * Example: * exporterTests. = { * exporterConfig = { * enable = true; * }; * metricProvider = { * services..enable = true; * }; * exporterTest = '' * wait_for_unit("prometheus--exporter.service") * wait_for_open_port("1234") * succeed("curl -sSf 'localhost:1234/metrics'") * ''; * }; * * # this would generate the following test config: * * nodes. = { * services.prometheus. = { * enable = true; * }; * services..enable = true; * }; * * testScript = '' * .start() * .wait_for_unit("prometheus--exporter.service") * .wait_for_open_port("1234") * .succeed("curl -sSf 'localhost:1234/metrics'") * .shutdown() * ''; */ exporterTests = { apcupsd = { exporterConfig = { enable = true; }; metricProvider = { services.apcupsd.enable = true; }; exporterTest = '' wait_for_unit("apcupsd.service") wait_for_open_port(3551) wait_for_unit("prometheus-apcupsd-exporter.service") wait_for_open_port(9162) succeed("curl -sSf http://localhost:9162/metrics | grep -q 'apcupsd_info'") ''; }; bind = { exporterConfig = { enable = true; }; metricProvider = { services.bind.enable = true; services.bind.extraConfig = '' statistics-channels { inet 127.0.0.1 port 8053 allow { localhost; }; }; ''; }; exporterTest = '' wait_for_unit("prometheus-bind-exporter.service") wait_for_open_port(9119) succeed( "curl -sSf http://localhost:9119/metrics | grep -q 'bind_query_recursions_total 0'" ) ''; }; blackbox = { exporterConfig = { enable = true; configFile = pkgs.writeText "config.yml" (builtins.toJSON { modules.icmp_v6 = { prober = "icmp"; icmp.preferred_ip_protocol = "ip6"; }; }); }; exporterTest = '' wait_for_unit("prometheus-blackbox-exporter.service") wait_for_open_port(9115) succeed( "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep -q 'probe_success 1'" ) ''; }; collectd = { exporterConfig = { enable = true; extraFlags = [ "--web.collectd-push-path /collectd" ]; }; exporterTest = let postData = replaceChars [ "\n" ] [ "" ] '' [{ "values":[23], "dstypes":["gauge"], "type":"gauge", "interval":1000, "host":"testhost", "plugin":"testplugin", "time":DATE }] ''; in '' wait_for_unit("prometheus-collectd-exporter.service") wait_for_open_port(9103) succeed( 'echo \'${postData}\'> /tmp/data.json' ) succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json') succeed( "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd" ) succeed( "curl -sSf localhost:9103/metrics | grep -q 'collectd_testplugin_gauge{instance=\"testhost\"} 23'" ) ''; }; dnsmasq = { exporterConfig = { enable = true; leasesPath = "/var/lib/dnsmasq/dnsmasq.leases"; }; metricProvider = { services.dnsmasq.enable = true; }; exporterTest = '' wait_for_unit("prometheus-dnsmasq-exporter.service") wait_for_open_port(9153) succeed("curl -sSf http://localhost:9153/metrics | grep -q 'dnsmasq_leases 0'") ''; }; dovecot = { exporterConfig = { enable = true; scopes = [ "global" ]; socketPath = "/var/run/dovecot2/old-stats"; user = "root"; # <- don't use user root in production }; metricProvider = { services.dovecot2.enable = true; }; exporterTest = '' wait_for_unit("prometheus-dovecot-exporter.service") wait_for_open_port(9166) succeed( "curl -sSf http://localhost:9166/metrics | grep -q 'dovecot_up{scope=\"global\"} 1'" ) ''; }; fritzbox = { # TODO add proper test case exporterConfig = { enable = true; }; exporterTest = '' wait_for_unit("prometheus-fritzbox-exporter.service") wait_for_open_port(9133) succeed( "curl -sSf http://localhost:9133/metrics | grep -q 'fritzbox_exporter_collect_errors 0'" ) ''; }; json = { exporterConfig = { enable = true; url = "http://localhost"; configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON [{ name = "json_test_metric"; path = "$.test"; }]); }; metricProvider = { systemd.services.prometheus-json-exporter.after = [ "nginx.service" ]; services.nginx = { enable = true; virtualHosts.localhost.locations."/".extraConfig = '' return 200 "{\"test\":1}"; ''; }; }; exporterTest = '' wait_for_unit("nginx.service") wait_for_open_port(80) wait_for_unit("prometheus-json-exporter.service") wait_for_open_port(7979) succeed("curl -sSf localhost:7979/metrics | grep -q 'json_test_metric 1'") ''; }; keylight = { # A hardware device is required to properly test this exporter, so just # perform a couple of basic sanity checks that the exporter is running # and requires a target, but cannot reach a specified target. exporterConfig = { enable = true; }; exporterTest = '' wait_for_unit("prometheus-keylight-exporter.service") wait_for_open_port(9288) succeed( "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep -q '400'" ) succeed( "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep -q '500'" ) ''; }; lnd = { exporterConfig = { enable = true; lndTlsPath = "/var/lib/lnd/tls.cert"; lndMacaroonDir = "/var/lib/lnd"; }; metricProvider = { systemd.services.prometheus-lnd-exporter.serviceConfig.DynamicUser = false; services.bitcoind.enable = true; services.bitcoind.extraConfig = '' rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7 bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 ''; systemd.services.lnd = { serviceConfig.ExecStart = '' ${pkgs.lnd}/bin/lnd \ --datadir=/var/lib/lnd \ --tlscertpath=/var/lib/lnd/tls.cert \ --tlskeypath=/var/lib/lnd/tls.key \ --logdir=/var/log/lnd \ --bitcoin.active \ --bitcoin.mainnet \ --bitcoin.node=bitcoind \ --bitcoind.rpcuser=bitcoinrpc \ --bitcoind.rpcpass=hunter2 \ --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \ --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \ --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon ''; serviceConfig.StateDirectory = "lnd"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; }; }; exporterTest = '' wait_for_unit("lnd.service") wait_for_open_port(10009) wait_for_unit("prometheus-lnd-exporter.service") wait_for_open_port(9092) succeed("curl -sSf localhost:9092/metrics | grep -q '^promhttp_metric_handler'") ''; }; mail = { exporterConfig = { enable = true; configuration = { monitoringInterval = "2s"; mailCheckTimeout = "10s"; servers = [ { name = "testserver"; server = "localhost"; port = 25; from = "mail-exporter@localhost"; to = "mail-exporter@localhost"; detectionDir = "/var/spool/mail/mail-exporter/new"; } ]; }; }; metricProvider = { services.postfix.enable = true; systemd.services.prometheus-mail-exporter = { after = [ "postfix.service" ]; requires = [ "postfix.service" ]; preStart = '' mkdir -p -m 0700 mail-exporter/new ''; serviceConfig = { ProtectHome = true; ReadOnlyPaths = "/"; ReadWritePaths = "/var/spool/mail"; WorkingDirectory = "/var/spool/mail"; }; }; users.users.mailexporter.isSystemUser = true; }; exporterTest = '' wait_for_unit("postfix.service") wait_for_unit("prometheus-mail-exporter.service") wait_for_open_port(9225) wait_until_succeeds( "curl -sSf http://localhost:9225/metrics | grep -q 'mail_deliver_success{configname=\"testserver\"} 1'" ) ''; }; mikrotik = { exporterConfig = { enable = true; extraFlags = [ "-timeout=1s" ]; configuration = { devices = [ { name = "router"; address = "192.168.42.48"; user = "prometheus"; password = "shh"; } ]; features = { bgp = true; dhcp = true; dhcpl = true; dhcpv6 = true; health = true; routes = true; poe = true; pools = true; optics = true; w60g = true; wlansta = true; wlanif = true; monitor = true; ipsec = true; }; }; }; exporterTest = '' wait_for_unit("prometheus-mikrotik-exporter.service") wait_for_open_port(9436) succeed( "curl -sSf http://localhost:9436/metrics | grep -q 'mikrotik_scrape_collector_success{device=\"router\"} 0'" ) ''; }; modemmanager = { exporterConfig = { enable = true; refreshRate = "10s"; }; metricProvider = { # ModemManager is installed when NetworkManager is enabled. Ensure it is # started and is wanted by NM and the exporter to start everything up # in the right order. networking.networkmanager.enable = true; systemd.services.ModemManager = { enable = true; wantedBy = [ "NetworkManager.service" "prometheus-modemmanager-exporter.service" ]; }; }; exporterTest = '' wait_for_unit("ModemManager.service") wait_for_unit("prometheus-modemmanager-exporter.service") wait_for_open_port(9539) succeed( "curl -sSf http://localhost:9539/metrics | grep -q 'modemmanager_info'" ) ''; }; nextcloud = { exporterConfig = { enable = true; passwordFile = "/var/nextcloud-pwfile"; url = "http://localhost/negative-space.xml"; }; metricProvider = { systemd.services.nc-pwfile = let passfile = (pkgs.writeText "pwfile" "snakeoilpw"); in { requiredBy = [ "prometheus-nextcloud-exporter.service" ]; before = [ "prometheus-nextcloud-exporter.service" ]; serviceConfig.ExecStart = '' ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile ''; }; services.nginx = { enable = true; virtualHosts."localhost" = { basicAuth.nextcloud-exporter = "snakeoilpw"; locations."/" = { root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata"; }; }; }; }; exporterTest = '' wait_for_unit("nginx.service") wait_for_unit("prometheus-nextcloud-exporter.service") wait_for_open_port(9205) succeed("curl -sSf http://localhost:9205/metrics | grep -q 'nextcloud_up 1'") ''; }; nginx = { exporterConfig = { enable = true; }; metricProvider = { services.nginx = { enable = true; statusPage = true; virtualHosts."test".extraConfig = "return 204;"; }; }; exporterTest = '' wait_for_unit("nginx.service") wait_for_unit("prometheus-nginx-exporter.service") wait_for_open_port(9113) succeed("curl -sSf http://localhost:9113/metrics | grep -q 'nginx_up 1'") ''; }; node = { exporterConfig = { enable = true; }; exporterTest = '' wait_for_unit("prometheus-node-exporter.service") wait_for_open_port(9100) succeed( "curl -sSf http://localhost:9100/metrics | grep -q 'node_exporter_build_info{.\\+} 1'" ) ''; }; openvpn = { exporterConfig = { enable = true; group = "openvpn"; statusPaths = ["/run/openvpn-test"]; }; metricProvider = { users.groups.openvpn = {}; services.openvpn.servers.test = { config = '' dev tun status /run/openvpn-test status-version 3 ''; up = "chmod g+r /run/openvpn-test"; }; systemd.services."openvpn-test".serviceConfig.Group = "openvpn"; }; exporterTest = '' wait_for_unit("openvpn-test.service") wait_for_unit("prometheus-openvpn-exporter.service") succeed("curl -sSf http://localhost:9176/metrics | grep -q 'openvpn_up{.*} 1'") ''; }; postfix = { exporterConfig = { enable = true; }; metricProvider = { services.postfix.enable = true; }; exporterTest = '' wait_for_unit("prometheus-postfix-exporter.service") wait_for_file("/var/lib/postfix/queue/public/showq") wait_for_open_port(9154) succeed( "curl -sSf http://localhost:9154/metrics | grep -q 'postfix_smtpd_connects_total 0'" ) succeed("curl -sSf http://localhost:9154/metrics | grep -q 'postfix_up{.*} 1'") ''; }; postgres = { exporterConfig = { enable = true; runAsLocalSuperUser = true; }; metricProvider = { services.postgresql.enable = true; }; exporterTest = '' wait_for_unit("prometheus-postgres-exporter.service") wait_for_open_port(9187) wait_for_unit("postgresql.service") succeed( "curl -sSf http://localhost:9187/metrics | grep -q 'pg_exporter_last_scrape_error 0'" ) succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 1'") systemctl("stop postgresql.service") succeed( "curl -sSf http://localhost:9187/metrics | grep -qv 'pg_exporter_last_scrape_error 0'" ) succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 0'") systemctl("start postgresql.service") wait_for_unit("postgresql.service") succeed( "curl -sSf http://localhost:9187/metrics | grep -q 'pg_exporter_last_scrape_error 0'" ) succeed("curl -sSf http://localhost:9187/metrics | grep -q 'pg_up 1'") ''; }; redis = { exporterConfig = { enable = true; }; metricProvider.services.redis.enable = true; exporterTest = '' wait_for_unit("redis.service") wait_for_unit("prometheus-redis-exporter.service") wait_for_open_port(6379) wait_for_open_port(9121) wait_until_succeeds("curl -sSf localhost:9121/metrics | grep -q 'redis_up 1'") ''; }; rspamd = { exporterConfig = { enable = true; }; metricProvider = { services.rspamd.enable = true; virtualisation.memorySize = 1024; }; exporterTest = '' wait_for_unit("rspamd.service") wait_for_unit("prometheus-rspamd-exporter.service") wait_for_open_port(11334) wait_for_open_port(7980) wait_until_succeeds( "curl -sSf localhost:7980/metrics | grep -q 'rspamd_scanned{host=\"rspamd\"} 0'" ) ''; }; rtl_433 = { exporterConfig = { enable = true; }; metricProvider = { # Mock rtl_433 binary to return a dummy metric stream. nixpkgs.overlays = [ (self: super: { rtl_433 = self.runCommand "rtl_433" {} '' mkdir -p "$out/bin" cat < "$out/bin/rtl_433" #!/bin/sh while true; do printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n' sleep 4 done EOF chmod +x "$out/bin/rtl_433" ''; }) ]; }; exporterTest = '' wait_for_unit("prometheus-rtl_433-exporter.service") wait_for_open_port(9550) wait_until_succeeds( "curl -sSf localhost:9550/metrics | grep -q '{}'".format( 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18' ) ) ''; }; snmp = { exporterConfig = { enable = true; configuration.default = { version = 2; auth.community = "public"; }; }; exporterTest = '' wait_for_unit("prometheus-snmp-exporter.service") wait_for_open_port(9116) succeed("curl -sSf localhost:9116/metrics | grep -q 'snmp_request_errors_total 0'") ''; }; sql = { exporterConfig = { configuration.jobs.points = { interval = "1m"; connections = [ "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable" ]; queries = { points = { labels = [ "name" ]; help = "Amount of points accumulated per person"; values = [ "amount" ]; query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name"; }; }; }; enable = true; user = "prometheus-sql-exporter"; }; metricProvider = { services.postgresql = { enable = true; initialScript = builtins.toFile "init.sql" '' CREATE DATABASE data; \c data; CREATE TABLE points (amount INT, name TEXT); INSERT INTO points(amount, name) VALUES (1, 'jack'); INSERT INTO points(amount, name) VALUES (2, 'jill'); INSERT INTO points(amount, name) VALUES (3, 'jack'); CREATE USER "prometheus-sql-exporter"; GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter"; GRANT SELECT ON points TO "prometheus-sql-exporter"; ''; }; systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ]; }; exporterTest = '' wait_for_unit("prometheus-sql-exporter.service") wait_for_open_port(9237) succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep -q 2") ''; }; surfboard = { exporterConfig = { enable = true; modemAddress = "localhost"; }; metricProvider = { systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ]; services.nginx = { enable = true; virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = '' return 204; ''; }; }; exporterTest = '' wait_for_unit("nginx.service") wait_for_open_port(80) wait_for_unit("prometheus-surfboard-exporter.service") wait_for_open_port(9239) succeed("curl -sSf localhost:9239/metrics | grep -q 'surfboard_up 1'") ''; }; tor = { exporterConfig = { enable = true; }; metricProvider = { # Note: this does not connect the test environment to the Tor network. # Client, relay, bridge or exit connectivity are disabled by default. services.tor.enable = true; services.tor.controlPort = 9051; }; exporterTest = '' wait_for_unit("tor.service") wait_for_open_port(9051) wait_for_unit("prometheus-tor-exporter.service") wait_for_open_port(9130) succeed("curl -sSf localhost:9130/metrics | grep -q 'tor_version{.\\+} 1'") ''; }; unifi-poller = { nodeName = "unifi_poller"; exporterConfig.enable = true; exporterConfig.controllers = [ { } ]; exporterTest = '' wait_for_unit("prometheus-unifi-poller-exporter.service") wait_for_open_port(9130) succeed( "curl -sSf localhost:9130/metrics | grep -q 'unifipoller_build_info{.\\+} 1'" ) ''; }; varnish = { exporterConfig = { enable = true; instance = "/var/spool/varnish/varnish"; group = "varnish"; }; metricProvider = { systemd.services.prometheus-varnish-exporter.after = [ "varnish.service" ]; services.varnish = { enable = true; config = '' vcl 4.0; backend default { .host = "127.0.0.1"; .port = "80"; } ''; }; }; exporterTest = '' wait_for_unit("prometheus-varnish-exporter.service") wait_for_open_port(6081) wait_for_open_port(9131) succeed("curl -sSf http://localhost:9131/metrics | grep -q 'varnish_up 1'") ''; }; wireguard = let snakeoil = import ./wireguard/snakeoil-keys.nix; in { exporterConfig.enable = true; metricProvider = { networking.wireguard.interfaces.wg0 = { ips = [ "10.23.42.1/32" "fc00::1/128" ]; listenPort = 23542; inherit (snakeoil.peer0) privateKey; peers = singleton { allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ]; inherit (snakeoil.peer1) publicKey; }; }; systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ]; }; exporterTest = '' wait_for_unit("prometheus-wireguard-exporter.service") wait_for_open_port(9586) wait_until_succeeds( "curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'" ) ''; }; }; in mapAttrs (exporter: testConfig: (makeTest (let nodeName = testConfig.nodeName or exporter; in { name = "prometheus-${exporter}-exporter"; nodes.${nodeName} = mkMerge [{ services.prometheus.exporters.${exporter} = testConfig.exporterConfig; } testConfig.metricProvider or {}]; testScript = '' ${nodeName}.start() ${concatStringsSep "\n" (map (line: if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")") then line else "${nodeName}.${line}" ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))} ${nodeName}.shutdown() ''; meta = with maintainers; { maintainers = [ willibutz elseym ]; }; }))) exporterTests