nixfiles/hosts/kaon/monitoring.nix

{ config, pkgs, lib, ... }:

with lib;

let
  hosts = (import ../../lib/hosts.nix { inherit pkgs; }).hosts;

  monitoringHosts = filterAttrs (name: host:
    attrByPath ["clerie" "monitoring" "enable"] false host.config)
    hosts;

  monitoringHostsNames = mapAttrs' (name: host:
    nameValuePair "${host.config.clerie.monitoring.networkBase}${host.config.clerie.monitoring.id}"  ["${host.config.networking.hostName}.mon.entr0py.de"])
    monitoringHosts;

  monitoringPeers = mapAttrsToList (name: host: {
      allowedIPs = [ "${host.config.clerie.monitoring.networkBase}${host.config.clerie.monitoring.id}/128" ];
      publicKey = host.config.clerie.monitoring.pubkey;
    })
    monitoringHosts;

  monitoringTargets = mapAttrsToList (name: host:
    "${host.config.networking.hostName}.mon.entr0py.de:9100")
    monitoringHosts;

  auxHosts = (import ./auxMonHosts.nix {});

in {

  networking.hosts = {
    "fd00:23:23:23::1" = [ "kaon.mon.entr0py.de" ];
  }
  // monitoringHostsNames;

  networking.wireguard.enable = true;
  networking.wireguard.interfaces = {
    wg-monitoring = {
      ips = [ "fd00:23:23:23::1/64" ];
      listenPort = 51820;
      peers = monitoringPeers;
      privateKeyFile = "/run/keys/wg-mon_priv";
    };
  };

  networking.firewall.allowedUDPPorts = [ config.networking.wireguard.interfaces.wg-monitoring.listenPort ];

  services.prometheus.exporters.node.enable = true;

  services.prometheus = {
    enable = true;
    listenAddress = "[::1]";
    scrapeConfigs = [
      {
        job_name = "prometheus";
        scrape_interval = "20s";
        scheme = "http";
        static_configs = [
          {
            targets = [
              "[::1]:9090"
            ];
          }
        ];
      }
      {
        job_name = "node-exporter";
        scrape_interval = "20s";
        static_configs = [
          {
            targets = [
              "kaon.mon.entr0py.de:9100"
            ]
            ++ monitoringTargets;
          }
        ];
      }
      {
        job_name = "postgres";
        scrape_interval = "5s";
        static_configs = [
          {
            targets = auxHosts.postgresExporter;
          }
        ];
      }
      {
        job_name = "synapse";
        scrape_interval = "5s";
        metrics_path = "/_synapse/metrics";
        static_configs = [{
            targets = auxHosts.synapseExporter;
        }];
      }
    ];
  };

  services.postgresql = {
    enable = true;
    package = pkgs.postgresql_12;
    ensureDatabases = [ "grafana" ];
    ensureUsers = [
      {
        name = "grafana";
        ensurePermissions."DATABASE grafana" = "ALL PRIVILEGES";
      }
    ];
  };
  services.prometheus.exporters.postgres = {
    enable = true;
    runAsLocalSuperUser = true;
    port = 9101;
  };

  services.grafana = {
    enable  = true;
    domain  = "grafana.monitoring.entr0py.de";
    rootUrl = "https://grafana.monitoring.entr0py.de";
    port    = 3001;
    addr = "::1";

    database = {
      type = "postgres";
      name = "grafana";
      user = "grafana";
      host = "/run/postgresql";
    };

    auth.anonymous.enable = true;
    security = {
      adminUser = "garionion";
      adminPasswordFile = "/run/keys/grafana-admin";
    };

    provision = {
      enable = true;
      datasources = [
        {
          type = "prometheus";
          name = "Prometheus";
          url = "http://[::1]:9090";
          isDefault = true;
        }
      ];
      dashboards = [
        {
          options.path = ./dashboards;
        }
      ];
    };
  };
  systemd.services.grafana.after = ["postgresql.service"];


  services.nginx = {
    enable = true;

    virtualHosts = {
      "prometheus.monitoring.entr0py.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:9090/";
      };
      "grafana.monitoring.entr0py.de" = {
        enableACME = true;
        forceSSL   = true;
        locations."/".proxyPass = "http://[::1]:${toString config.services.grafana.port}/";
      };
    };
  };

  networking.firewall.allowedTCPPorts = [ 80 443 ];
}