add postgres monitoring

hosts/dyon/configuration.nix

@@ -2,9 +2,10 @@
   imports = [
     ./hardware-configuration.nix
     ../../common/common.nix
-    ../../common/nginx.nix
     ../../common/netdata.nix
+    ./nginx.nix
     ./wireguard.nix
+    ./coturn.nix
   ];
 
   

hosts/dyon/coturn.nix

@@ -0,0 +1,64 @@
+{config, pkgs, lib, ...}: {
+  # enable coturn
+  services.coturn = rec {
+    enable = true;
+    no-cli = true;
+    no-tcp-relay = true;
+    min-port = 49000;
+    max-port = 50000;
+    use-auth-secret = true;
+    static-auth-secret = (builtins.readFile ./turn.secret);
+    realm = "turn.entr0py.de";
+    cert = "${config.security.acme.certs.${realm}.directory}/full.pem";
+    pkey = "${config.security.acme.certs.${realm}.directory}/key.pem";
+    extraConfig = ''
+      # for debugging
+      verbose
+      # ban private IP ranges
+      no-multicast-peers
+      denied-peer-ip=0.0.0.0-0.255.255.255
+      denied-peer-ip=10.0.0.0-10.255.255.255
+      denied-peer-ip=100.64.0.0-100.127.255.255
+      denied-peer-ip=127.0.0.0-127.255.255.255
+      denied-peer-ip=169.254.0.0-169.254.255.255
+      denied-peer-ip=172.16.0.0-172.31.255.255
+      denied-peer-ip=192.0.0.0-192.0.0.255
+      denied-peer-ip=192.0.2.0-192.0.2.255
+      denied-peer-ip=192.88.99.0-192.88.99.255
+      denied-peer-ip=192.168.0.0-192.168.255.255
+      denied-peer-ip=198.18.0.0-198.19.255.255
+      denied-peer-ip=198.51.100.0-198.51.100.255
+      denied-peer-ip=203.0.113.0-203.0.113.255
+      denied-peer-ip=240.0.0.0-255.255.255.255
+      denied-peer-ip=::1
+      denied-peer-ip=64:ff9b::-64:ff9b::ffff:ffff
+      denied-peer-ip=::ffff:0.0.0.0-::ffff:255.255.255.255
+      denied-peer-ip=100::-100::ffff:ffff:ffff:ffff
+      denied-peer-ip=2001::-2001:1ff:ffff:ffff:ffff:ffff:ffff:ffff
+      denied-peer-ip=2002::-2002:ffff:ffff:ffff:ffff:ffff:ffff:ffff
+      denied-peer-ip=fc00::-fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff
+      denied-peer-ip=fe80::-febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff
+    '';
+  };
+  # open the firewall
+  networking.firewall = {
+    interfaces.ens3 = let
+      range = with config.services.coturn; [ {
+      from = min-port;
+      to = max-port;
+    } ];
+    in
+    {
+      allowedUDPPortRanges = range;
+      allowedUDPPorts = [ 3478 ];
+      allowedTCPPortRanges = range;
+      allowedTCPPorts = [ 3478 ];
+    };
+  };
+  # get a certificate
+  security.acme.certs.${config.services.coturn.realm} = {
+    webroot = "/var/www/letsencrypt";
+    postRun = "systemctl restart coturn.service";
+    group = "turnserver";
+  };
+}

hosts/dyon/nginx.nix

@@ -10,6 +10,12 @@
             addresses = [ "[2a02:2970:1002::18]" "46.182.19.48" "[2001:1608:10:25::9249:d69b]" "84.200.70.40"  ];
         };
         appendHttpConfig = ''
+            server {
+                listen turn.entr0py.de:80 default;
+                location ^~ /.well-known/acme-challenge/ {
+                    root /var/www/letsencrypt;
+                }
+            }
             server {
                 listen 0.0.0.0:80 default;
                 listen [::]:80 default;

hosts/geon/configuration.nix

@@ -21,6 +21,7 @@
   networking.hostName = "geon";
   networking.useDHCP = false;
   networking.interfaces.ens18.useDHCP = true;
+  networking.interfaces.ens18.mtu = 1420;
   networking.interfaces.ens18.ipv6.addresses = [{
     address = "2a01:4f8:c010:61dc:200::500:1";
     prefixLength = 72;

hosts/geon/gitea.nix

@@ -72,6 +72,13 @@
       }
     ];
   };
+  services.prometheus.exporters.postgres = {
+    enable = true;
+    port = 9101;
+    runAsLocalSuperUser = true;
+    openFirewall = true;
+    firewallFilter = "-i wg-monitoring -p tcp -m tcp --dport 9101";
+  };
 
   networking.firewall.allowedTCPPorts = [ 80 443 ];
   services.nginx = {

hosts/kaon/auxMonHosts.nix

@@ -0,0 +1,14 @@
+{ }:
+
+rec {
+
+  postgresExporter = [
+      "kaon.mon.entr0py.de:9101"
+      "luxon.mon.entr0py.de:9101"
+      "geon.mon.entr0py.de:9101"
+  ];
+
+  synapseExporter = [
+      "luxon.mon.entr0py.de:9123"
+  ];
+}

hosts/kaon/configuration.nix

@@ -21,6 +21,7 @@
   networking.hostName = "kaon";
   networking.useDHCP = false;
   networking.interfaces.ens18.useDHCP = true;
+  networking.interfaces.ens18.mtu = 1420;
   networking.interfaces.ens18.ipv6.addresses = [{
     address = "2a01:4f8:c010:61dc:200::42:1";
     prefixLength = 72;

hosts/kaon/monitoring.nix

@@ -23,10 +23,12 @@ let
     "${host.config.networking.hostName}.mon.entr0py.de:9100")
     monitoringHosts;
 
+  auxHosts = (import ./auxMonHosts.nix {});
+
 in {
 
   networking.hosts = {
-    "fd00:23:23:23::1" = [ "monitoring-1.mon.entr0py.de" ];
+    "fd00:23:23:23::1" = [ "kaon.mon.entr0py.de" ];
   }
   // monitoringHostsNames;
 
@@ -62,16 +64,33 @@ in {
       }
       {
         job_name = "node-exporter";
-        scrape_interval = "60s";
+        scrape_interval = "20s";
         static_configs = [
           {
             targets = [
-              "monitoring-1.mon.entr0py.de:9100"
+              "kaon.mon.entr0py.de:9100"
             ]
             ++ monitoringTargets;
           }
         ];
       }
+      {
+        job_name = "postgres";
+        scrape_interval = "5s";
+        static_configs = [
+          {
+            targets = auxHosts.postgresExporter;
+          }
+        ];
+      }
+      {
+        job_name = "synapse";
+        scrape_interval = "5s";
+        metrics_path = "/_synapse/metrics";
+        static_configs = [{
+            targets = auxHosts.synapseExporter;
+        }];
+      }
     ];
   };
 
@@ -86,6 +105,12 @@ in {
       }
     ];
   };
+  services.prometheus.exporters.postgres = {
+    enable = true;
+    runAsLocalSuperUser = true;
+    port = 9101;
+  };
+
   services.grafana = {
     enable  = true;
     domain  = "grafana.monitoring.entr0py.de";

hosts/luxon/configuration.nix

@@ -0,0 +1,49 @@
+{ config, pkgs, ... }:
+
+{
+  imports =
+    [ # Include the results of the hardware scan.
+        ./hardware-configuration.nix
+        ../../common/common.nix
+        ../../common/netdata.nix
+        ./matrix.nix
+    ];
+
+  # Use the systemd-boot EFI boot loader.
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+
+  networking.hostName = "luxon";
+  networking.useDHCP = false;
+  networking.interfaces.ens18.useDHCP = true;
+  networking.interfaces.ens18.mtu = 1420;
+  networking.interfaces.ens18.ipv6.addresses = [{
+    address = "2a01:4f8:c010:61dc:200::500:23";
+    prefixLength = 72;
+  }];
+
+  networking.defaultGateway6 = {
+    address = "2a01:4f8:c010:61dc:200::1";
+    interface = "ens18";
+  };
+
+  clerie.monitoring = {
+    enable = true;
+    id = "204";
+    privKeyFile = "/run/keys/wg-luxon_priv";
+    pubkey = "QAcy1H4ze8fkTbhkoMx58S48FhitQmqPClUfFduj6Vw=";
+  };
+
+
+  services.postgresql = {
+    enable = true;
+    package = pkgs.postgresql_12;
+  };
+  services.prometheus.exporters.postgres = {
+    enable = true;
+    port = 9101;
+    runAsLocalSuperUser = true;
+    openFirewall = true;
+    firewallFilter = "-i wg-monitoring -p tcp -m tcp --dport 9101";
+  };
+}

hosts/luxon/hardware-configuration.nix

@@ -0,0 +1,27 @@
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/profiles/qemu-guest.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/b24a0b24-ee5e-450c-930b-85ae5ac60275";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/DBBB-F848";
+      fsType = "vfat";
+    };
+
+  swapDevices =
+    [ { device = "/dev/disk/by-uuid/2698b80a-3b18-4252-beb2-6b2c3b5e4e76"; }
+    ];
+
+}

hosts/luxon/matrix.nix

@@ -0,0 +1,172 @@
+{config, pkgs, lib, ...}: {
+    networking.firewall.allowedTCPPorts = [ 80 443 8448 ];
+
+    services.nginx = {
+        enable = true;                                          # Enable Nginx
+        recommendedGzipSettings = true;
+        recommendedOptimisation = true;
+        recommendedProxySettings = true;
+        recommendedTlsSettings = true;
+        virtualHosts = {
+            "entr0py.de" = {                  
+                enableACME = true;                               
+                forceSSL = true;
+                locations."= /.well-known/matrix/server".extraConfig =
+                    let
+                        # use 443 instead of the default 8448 port to unite
+                        # the client-server and server-server port for simplicity
+                        server = { "m.server" = "matrix.entr0py.de:443"; };
+                    in ''
+                        add_header Content-Type application/json;
+                        return 200 '${builtins.toJSON server}';
+                    '';
+                locations."= /.well-known/matrix/client".extraConfig =
+                    let
+                        client = {
+                            "m.homeserver" =  { "base_url" = "https://matrix.entr0py.de"; };
+                            "m.identity_server" =  { "base_url" = "https://element.io"; };
+                        };
+                    # ACAO required to allow element-web on any URL to request this json file
+                    in ''
+                        add_header Content-Type application/json;
+                        add_header Access-Control-Allow-Origin *;
+                        return 200 '${builtins.toJSON client}';
+                    '';
+            };
+
+            # Reverse proxy for Matrix client-server and server-server communication
+            "matrix.entr0py.de" = {
+                enableACME = true;
+                forceSSL = true;
+
+                # Or do a redirect instead of the 404, or whatever is appropriate for you.
+                # But do not put a Matrix Web client here! See the Element web section below.
+                locations."/".extraConfig = ''
+                    return 404;
+                '';
+
+                # forward all Matrix API calls to the synapse Matrix homeserver
+                locations."/_matrix" = {
+                proxyPass = "http://[::1]:8008"; # without a trailing /
+                };
+            };
+        };
+    };
+
+    services.matrix-synapse = {
+        enable = true;
+        server_name = "entr0py.de";
+        enable_registration = true;
+        database_type = "psycopg2";
+        database_name = "matrix_synapse";
+        database_user = "matrix-synapse";
+        max_upload_size = "100M";
+        enable_metrics = true;
+        extraConfig = ''
+            admin_contact: "mailto:admin@entr0py.de"
+        '';
+        turn_uris = ["turn:turn.entr0py.de:3478?transport=udp" "turn:turn.entr0py.de:3478?transport=tcp"];
+        turn_user_lifetime = "1d";
+        turn_shared_secret = (builtins.readFile ../dyon/turn.secret);
+        app_service_config_files = [
+            "/var/lib/matrix-appservice-irc/registration.yml"
+        ];
+        listeners = [{
+            port = 8008;
+            bind_address = "::1";
+            type = "http";
+            tls = false;
+            x_forwarded = true;
+            resources = [{
+                names = [ "client" "federation" ];
+                compress = false;
+            }];
+        }{
+            port = 9123;
+            bind_address = "[::]"; #does not work right now
+            
+            type = "metrics";
+            tls = false;
+            resources = [{
+                names = [ "metric"  ];
+                compress = false;
+            }];
+        }];
+    };
+
+
+    services.postgresql = {
+
+        ensureDatabases = [ "matrix_irc" ];
+        ensureUsers = [
+            {
+                name = "matrix-synapse";
+            }
+            {
+                name = "matrix-appservice-irc";
+                ensurePermissions."DATABASE matrix_irc" = "ALL PRIVILEGES";
+            }
+        ];
+        initialScript = pkgs.writeText "synapse-init.sql" ''
+            CREATE DATABASE "matrix_synapse" WITH OWNER "matrix-synapse"
+            TEMPLATE template0
+            LC_COLLATE = "C"
+            LC_CTYPE = "C";
+        '';
+    };
+
+    services.matrix-appservice-irc = {
+        enable = true;
+        registrationUrl = "http://localhost:8009"; 
+
+        # Everything from here is passed to the appservice
+        settings = {
+            homeserver.url = "https://matrix.entr0py.de"; # Or localhost
+            homeserver.domain = "entr0py.de";
+            database = {
+                engine = "postgres";
+                connectionString = "postgres:///matrix_irc?host=/run/postgresql";
+            };
+
+            ircService.ircHandler.mapIrcMentionsToMatrix = "on";
+
+            ircService.servers."irc.hackint.org" = {
+                name = "HackInt";
+                port = 6697;
+                ssl = true;
+                dynamicChannels = {
+                    enabled = true;
+                    createAlias = true;
+                    aliasTemplate = "#hackint_$CHANNEL";
+                    groupId = "+hackint:entr0py.de";
+                    
+                };
+                matrixClients = {
+                    userTemplate = "@hackint_$NICK";
+                    displayName = "$NICK (HackInt)";
+                };
+                ircClients = {
+                    nickTemplate = "$DISPLAY[m]";
+                    allowNickChanges = true;
+
+                    lineLimit = 15;
+                    idleTimeout = 0;
+                };
+
+                membershipLists = {
+                    enabled = true;
+                    global = {
+                        ircToMatrix = {
+                            initial = true;
+                            incremental = true;
+                        };
+                        matrixToIrc = {
+                            initial = true;
+                            incremental = true;
+                        };
+                    };
+                };
+            };
+        };
+    };
+}

hosts/nebula/configuration.nix

@@ -18,6 +18,7 @@
 
   networking.useDHCP = false;
   networking.interfaces.ens18.useDHCP = true;
+  networking.interfaces.ens18.mtu = 1420;
   networking.interfaces.ens18.ipv6.addresses = [{
     address = "2a01:4f8:c010:61dc:200::500:2";
     prefixLength = 72;

hosts/nebula/drone.nix

@@ -32,7 +32,7 @@
         ];
     };
     users.users.drone = {
-      isNormalUser = false;
+      isSystemUser = true;
     };
     virtualisation.oci-containers.backend = "podman";
     virtualisation.oci-containers.containers = {
@@ -68,6 +68,7 @@
                 DRONE_RPC_HOST="drone.entr0py.de";
                 DRONE_RUNNER_CAPACITY="4";
                 DRONE_RUNNER_NAME="nebula";
+                DRONE_TMATE_ENABLED="true";
             };
             ports = [
                 "3000:3000"

hosts/pion/configuration.nix

@@ -45,4 +45,11 @@
       ens3.rules."2a03:4000:45:5f3:500::3/128".method = "static";
     };
   };
+
+  clerie.monitoring = {
+    enable = true;
+    id = "101";
+    privKeyFile = "/run/keys/wg-pion_priv";
+    pubkey = "X4TNT0aseK11D+jLKvNSl2qI9GU7HDTnKzVsgtiPpAw=";
+  };
 }

hosts/tailpipe/configuration.nix

@@ -22,4 +22,11 @@
   };
   networking.defaultGateway = { address = "172.31.1.1"; interface = "ens3"; };
 
+  clerie.monitoring = {
+    enable = true;
+    id = "7";
+    privKeyFile = "/run/keys/wg-tailpipe_priv";
+    pubkey = "M1ETBQW1tas8Vp2gr6zesTL+6Y9Q1pctZtdYS0nEJn4=";
+  };
+
 }

nixops/bulldog/deployment.nix

@@ -7,4 +7,5 @@
     dyon = import ../../hosts/dyon/configuration.nix;
     kaon = import ../../hosts/kaon/configuration.nix;
     nebula = import ../../hosts/nebula/configuration.nix;
+    luxon = import ../../hosts/luxon/configuration.nix;
 }

nixops/bulldog/deployment_ip.nix

@@ -4,23 +4,34 @@
     };
     geon = { config, pkgs, ... }:{
         deployment.targetHost = "geon.net.entr0py.de";
+        hasFastConnection = true;
     };
     pion = { config, pkgs, ... }:{
         deployment.targetHost = "pion.net.entr0py.de";
+        hasFastConnection = true;
     };
     scotty-router = { config, pkgs, ... }:{
         deployment.targetHost = "scotty-router";
+        hasFastConnection = true;
     };
     tailpipe = { config, pkgs, ... }:{ 
         deployment.targetHost = "tailpipe.net.entr0py.de";
+        hasFastConnection = true;
     };
     dyon = { config, pkgs, ... }:{
         deployment.targetHost = "dyon.net.entr0py.de";
+        hasFastConnection = true;
     };
     kaon = { config, pkgs, ... }:{ 
         deployment.targetHost = "kaon.net.entr0py.de";
+        hasFastConnection = true;
     };
     nebula = { config, pkgs, ... }:{ 
         deployment.targetHost = "nebula.net.entr0py.de";
+        hasFastConnection = true;
+    };
+    luxon = { config, pkgs, ... }:{ 
+        deployment.targetHost = "luxon.net.entr0py.de";
+        hasFastConnection = true;
     };
 }