nixos/services/self_host/monitor.nix

{
  config,
  pkgs,
  lib,
  ...
}:

let
  cfg = config.service.selfhost.monitor;
  dashboardsDir = ../../assets/grafana_dashboards;
  monitored = [
    "nginx"
    "grafana"
  ];
in
{
  config = lib.mkIf cfg {
    services.grafana = {
      enable = true;
      package = pkgs.grafana;
      dataDir = "/var/lib/grafana";
      provision = {
        dashboards.settings.providers = [
          {
            name = "nixos-dashboards";
            type = "file";
            updateIntervalSeconds = 30;
            editable = false;

            options = {
              path = "/etc/grafana/dashboards";
              foldersFromFilesStructure = false;
            };
          }
        ];
        datasources.settings.datasources = [
          {
            name = "Prometheus";
            type = "prometheus";
            uid = "prometheus";
            access = "proxy";
            url = "http://127.0.0.1:9090";
            isDefault = true;
            editable = false;
            jsonData = {
              httpMethod = "POST";
              timeInterval = "15s";
            };
          }
        ];
      };
      settings = {
        server = {
          root_url = "https://monitor.enium.eu";
          domain = "monitor.enium.eu";
          serve_from_sub_path = false;
        };

        "auth.generic_oauth" = {
          enabled = true;
          name = "Enium";
          allow_sign_up = true;
          client_id = "grafana";
          client_secret = "$__file{${config.age.secrets.grafana-oidc-secret.path}}";
          scopes = "openid profile email groups";
          auth_url = "https://auth.enium.eu/ui/oauth2";
          token_url = "https://auth.enium.eu/oauth2/token";
          api_url = "https://auth.enium.eu/oauth2/openid/grafana/userinfo";
          redirect_uri = "https://monitor.enium.eu/login/generic_oauth";
          use_pkce = true;
          use_refresh_token = true;
          login_attribute_path = "preferred_username";
          name_attribute_path = "name";
          email_attribute_path = "email";
          groups_attribute_path = "groups";
          role_attribute_path = "contains(groups, 'grafana_superadmins@enium.eu') && 'GrafanaAdmin' || contains(groups, 'grafana_admins@enium.eu') && 'Admin' || contains(groups, 'grafana_editors@enium.eu') && 'Editor' || 'Viewer'";
          allow_assign_grafana_admin = true;
          role_attribute_strict = false;
          skip_org_role_sync = false;
        };
        log.level = "debug";
        auth = {
          disable_login_form = true;
          disable_signout_menu = false;
        };
        security = {
          cookie_secure = true;
          cookie_samesite = "none";
          allow_embedding = true;
        };
      };
    };

    environment.etc."process-exporter.json".text = builtins.toJSON {
      procMatchers = lib.map (svc: {
        name = svc;
        cmdline = [
          "${svc}:"
        ];
      }) monitored;
    };

    systemd.services.process_exporter = {
      description = "Prometheus Process Exporter";
      after = [ "network.target" ];
      wantedBy = [ "multi-user.target" ];
      serviceConfig = {
        ExecStart = "${pkgs.prometheus-process-exporter}/bin/process-exporter --config.path /etc/process-exporter.json";
        Restart = "always";
      };
    };

    services.prometheus = {
      enable = true;
      checkConfig = false;
      exporters = {
        blackbox = {
          enable = true;
          configFile = pkgs.writeText "blackbox-exporter.yml" ''
            modules:
              http_2xx:
                prober: http
                timeout: 5s
                http:
                  valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
                  valid_status_codes: []
                  method: GET
                  no_follow_redirects: false
                  fail_if_not_ssl: false
          '';
        };
        node.enable = true;
        systemd.enable = true;
      };
      scrapeConfigs = [
        {
          job_name = "systemd_exporter";
          metrics_path = "/metrics";
          static_configs = [
            {
              targets = [
                "127.0.0.1:9558"
              ];
            }
          ];
        }
        {
          job_name = "node_exporter";
          static_configs = [
            {
              targets = [
                "127.0.0.1:9100"
              ];
            }
          ];
        }
        {
          job_name = "process_exporter";
          metrics_path = "/metrics";
          scheme = "http";
          static_configs = [
            {
              targets = [
                "127.0.0.1:9256"
              ];
            }
          ];
        }
        {
          job_name = "blackbox_http_probe";
          metrics_path = "/probe";
          params = {
            module = [
              "http_2xx"
            ];
          };
          static_configs = [
            {
              targets = [
                "https://raphael.parodi.pro"
                "https://nextcloud.enium.eu"
                "https://htop.enium.eu"
                "https://monitor.enium.eu"
                "https://ollama.enium.eu"
                "http://relance-pas-stp.me:4242"
              ];
            }
          ];
          relabel_configs = [
            {
              source_labels = [ "__address__" ];
              target_label = "__param_target";
            }
            {
              source_labels = [ "__param_target" ];
              target_label = "instance";
            }
            {
              target_label = "__address__";
              replacement = "127.0.0.1:9115";
            }
          ];
          proxy_url = "http://127.0.0.1:9115";
        }
      ];
      ruleFiles = lib.mkForce [ "/etc/prometheus/services.rules" ];
    };

    environment.etc."grafana/dashboards".source = dashboardsDir;

    environment.etc."prometheus/services.rules".text = ''
      groups:
      - name: services
        rules:
        - alert: nginxServiceDown
          expr: process_up{job="process_exporter",name="nginx"} == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "Processus nginx arrêté"
            description: "Le processus nginx ne tourne plus depuis >1m."

        - alert: nginxServiceUp
          expr: process_up{job="process_exporter",name="nginx"} == 1
          for: 1m
          labels:
            severity: info
          annotations:
            summary: "Processus nginx rétabli"
            description: "Le processus nginx tourne de nouveau."

        - alert: grafanaServiceDown
          expr: process_up{job="process_exporter",name="grafana"} == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "Processus grafana arrêté"
            description: "Le processus grafana ne tourne plus depuis >1m."

        - alert: grafanaServiceUp
          expr: process_up{job="process_exporter",name="grafana"} == 1
          for: 1m
          labels:
            severity: info
          annotations:
            summary: "Processus grafana rétabli"
            description: "Le processus grafana tourne de nouveau."
    '';

    services.nginx.virtualHosts."monitor.enium.eu" = {
      enableACME = true;
      forceSSL = true;
      locations."/" = {
        proxyPass = "http://127.0.0.1:3000";
        proxyWebsockets = true;
      };
    };
  };
}