feat: grafana + prometheus + loki on flora-6 #77

Merged
teutat3s merged 15 commits from feat/grafana into main 2023-12-13 23:21:32 +00:00
19 changed files with 24288 additions and 6 deletions

View file

@ -27,6 +27,25 @@
reverse_proxy :4000
'';
};
"flora-6.pub.solar" = {
logFormat = lib.mkForce ''
output discard
'';
extraConfig = ''
basicauth * {
hakkonaut $2a$14$mmIAy/Ezm6YGohUtXa2mWeW6Bcw1MQXPhrRbz14jAD2iUu3oob/t.
}
reverse_proxy :${toString config.services.loki.configuration.server.http_listen_port}
'';
};
"grafana.pub.solar" = {
logFormat = lib.mkForce ''
output discard
'';
extraConfig = ''
reverse_proxy :${toString config.services.grafana.settings.server.http_port}
'';
};
"obs-portal.pub.solar" = {
logFormat = lib.mkForce ''
output discard

View file

@ -10,6 +10,10 @@
mode = "644";
};
# Trust docker bridge interface traffic
# Needed for the docker runner to communicate with the act_runner cache
networking.firewall.trustedInterfaces = [ "br-+" ];
# forgejo actions runner
# https://forgejo.org/docs/latest/admin/actions/
# https://docs.gitea.com/usage/actions/quickstart

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,108 @@
{
config,
lib,
pkgs,
flake,
...
}: {
age.secrets.grafana-admin-password = {
file = "${flake.self}/secrets/grafana-admin-password.age";
mode = "644";
owner = "grafana";
};
age.secrets.grafana-smtp-password = {
file = "${flake.self}/secrets/grafana-smtp-password.age";
mode = "644";
owner = "grafana";
};
age.secrets.grafana-keycloak-client-secret = {
file = "${flake.self}/secrets/grafana-keycloak-client-secret.age";
mode = "644";
owner = "grafana";
};
environment.etc = {
"grafana-dashboards/node-exporter-full_rev33.json" = {
source = ./grafana-dashboards/node-exporter-full_rev33.json;
group = "grafana";
user = "grafana";
};
};
services.grafana = {
enable = true;
settings = {
server = {
# Listening Address
http_addr = "127.0.0.1";
# and Port
http_port = 3000;
# Grafana needs to know on which domain and URL it's running
domain = "grafana.pub.solar";
root_url = "https://grafana.pub.solar";
enable_gzip = true;
};
smtp = {
enabled = true;
host = "mail.greenbaum.zone:465";
user = "admins@pub.solar";
password = "\$__file{${config.age.secrets.grafana-smtp-password.path}}";
from_address = "no-reply@pub.solar";
from_name = "grafana.pub.solar";
ehlo_identity = "flora-6.pub.solar";
};
security = {
admin_email = "crew@pub.solar";
admin_password = "\$__file{${config.age.secrets.grafana-admin-password.path}}";
};
"auth.generic_oauth" = {
enabled = true;
name = "pub.solar ID";
allow_sign_up = true;
client_id = "grafana";
client_secret = "\$__file{${config.age.secrets.grafana-keycloak-client-secret.path}}";
scopes = "openid email profile offline_access roles";
email_attribute_path = "email";
login_attribute_path = "preferred_username";
name_attribute_path = "full_name";
auth_url = "https://auth.pub.solar/realms/pub.solar/protocol/openid-connect/auth";
token_url = "https://auth.pub.solar/realms/pub.solar/protocol/openid-connect/token";
api_url = "https://auth.pub.solar/realms/pub.solar/protocol/openid-connect/userinfo";
role_attribute_path = "contains(roles[*], 'admin') && 'GrafanaAdmin' || 'Viewer'";
allow_assign_grafana_admin = true;
};
};
provision = {
enable = true;
datasources = {
settings = {
datasources = [
{
name = "Prometheus";
type = "prometheus";
access = "proxy";
url = "http://127.0.0.1:${toString config.services.prometheus.port}";
isDefault = true;
}
{
name = "Loki";
type = "loki";
access = "proxy";
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
}
];
};
};
dashboards = {
settings = {
providers = [
{
name = "pub.solar Dashboards";
options.path = "/etc/grafana-dashboards";
}
];
};
};
};
};
}

View file

@ -0,0 +1,85 @@
{
config,
lib,
pkgs,
flake,
...
}: {
# source: https://gist.github.com/rickhull/895b0cb38fdd537c1078a858cf15d63e
# https://grafana.com/docs/loki/latest/configure/examples/#1-local-configuration-exampleyaml
services.loki = {
enable = true;
configuration = {
server.http_listen_port = 3100;
auth_enabled = false;
common = {
ring = {
instance_addr = "127.0.0.1";
kvstore = {
store = "inmemory";
};
};
replication_factor = 1;
path_prefix = "/var/lib/loki";
storage = {
filesystem = {
chunks_directory = "chunks/";
rules_directory = "rules/";
};
};
};
# Keep logs for 4 weeks
# https://grafana.com/docs/loki/latest/operations/storage/retention/
limits_config.retention_period = "4w";
compactor = {
shared_store = "filesystem";
compaction_interval = "10m";
retention_enabled = true;
retention_delete_delay = "2h";
retention_delete_worker_count = 150;
};
schema_config = {
configs = [{
from = "2020-05-15";
store = "boltdb-shipper";
object_store = "filesystem";
schema = "v11";
index = {
prefix = "index_";
period = "24h";
};
}];
};
};
};
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 9080;
grpc_listen_port = 0;
};
positions = {
filename = "/tmp/positions.yaml";
};
clients = [{
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
}];
scrape_configs = [{
job_name = "journal";
journal = {
max_age = "24h";
labels = {
job = "systemd-journal";
host = "flora-6";
};
};
relabel_configs = [{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}];
}];
};
};
}

View file

@ -0,0 +1,55 @@
{
config,
lib,
pkgs,
flake,
...
}: {
age.secrets.nachtigall-metrics-prometheus-basic-auth-password = {
file = "${flake.self}/secrets/nachtigall-metrics-prometheus-basic-auth-password.age";
mode = "600";
owner = "prometheus";
};
services.prometheus = {
enable = true;
port = 9001;
exporters = {
node = {
enable = true;
enabledCollectors = [ "systemd" ];
port = 9002;
};
};
globalConfig = {
scrape_interval = "10s";
scrape_timeout = "9s";
};
scrapeConfigs = [
{
job_name = "http-targets";
static_configs = [{
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
labels = {
instance = "flora-6";
};
}];
}
{
job_name = "https-targets";
scheme = "https";
metrics_path = "/metrics";
basic_auth = {
username = "hakkonaut";
password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
};
static_configs = [{
targets = [ "nachtigall.pub.solar" ];
labels = {
instance = "nachtigall";
};
}];
}
];
};
}

View file

@ -29,6 +29,13 @@ in {
# Force getting the hostname from cloud-init
networking.hostName = lib.mkDefault "";
# We use cloud-init to configure networking, this option should fix
# systemd-networkd-wait-online timeouts
#systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
systemd.network.wait-online.ignoredInterfaces = [
"docker0"
];
# List services that you want to enable:
services.cloud-init.enable = true;
services.cloud-init.ext4.enable = true;

View file

@ -11,5 +11,8 @@
./apps/drone.nix
./apps/forgejo-actions-runner.nix
./apps/grafana.nix
./apps/prometheus.nix
./apps/loki.nix
];
}

View file

@ -0,0 +1,19 @@
{ config, flake, ... }:
{
age.secrets.nachtigall-metrics-nginx-basic-auth = {
file = "${flake.self}/secrets/nachtigall-metrics-nginx-basic-auth.age";
mode = "600";
owner = "nginx";
};
services.nginx.virtualHosts = {
"nachtigall.pub.solar" = {
enableACME = true;
addSSL = true;
basicAuthFile = "${config.age.secrets.nachtigall-metrics-nginx-basic-auth.path}";
locations."/metrics" = {
proxyPass = "http://127.0.0.1:${toString(config.services.prometheus.exporters.node.port)}";
};
};
};
}

View file

@ -0,0 +1,14 @@
{
config,
...
}: {
services.prometheus = {
exporters = {
node = {
enable = true;
enabledCollectors = [ "systemd" ];
port = 9002;
};
};
};
}

View file

@ -0,0 +1,47 @@
{
config,
lib,
pkgs,
flake,
...
}: {
age.secrets.nachtigall-metrics-prometheus-basic-auth-password = {
file = "${flake.self}/secrets/nachtigall-metrics-prometheus-basic-auth-password.age";
mode = "600";
owner = "promtail";
};
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 9080;
grpc_listen_port = 0;
};
positions = {
filename = "/tmp/positions.yaml";
};
clients = [{
url = "https://flora-6.pub.solar/loki/api/v1/push";
basic_auth = {
username = "hakkonaut";
password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
};
}];
scrape_configs = [{
job_name = "journal";
journal = {
max_age = "24h";
labels = {
job = "systemd-journal";
host = "nachtigall";
};
};
relabel_configs = [{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}];
}];
};
};
}

View file

@ -18,17 +18,20 @@
./apps/mastodon.nix
./apps/mediawiki.nix
./apps/nextcloud.nix
./apps/owncast.nix
./apps/nginx-mastodon.nix
./apps/nginx-mastodon-files.nix
./apps/nginx-prometheus-exporters.nix
./apps/nginx-website.nix
./apps/opensearch.nix
./apps/owncast.nix
./apps/postgresql.nix
./apps/prometheus-exporters.nix
./apps/promtail.nix
./apps/searx.nix
./apps/matrix/irc.nix
./apps/matrix/mautrix-telegram.nix
./apps/matrix/synapse.nix
./apps/matrix/irc.nix
./apps/nginx-matrix.nix
];
}

Binary file not shown.

View file

@ -0,0 +1,28 @@
age-encryption.org/v1
-> ssh-ed25519 Y0ZZaw M6ha3gQ4Oq4PdymYZ5ZG0qGwFlpCYfJdhOBwH9n1gxg
zCtB0PJanufNdV0ShynDT0Z/2jxMFDRby8xsfv6YPaA
-> ssh-ed25519 uYcDNw V89Ll4HJ3ZkQegiCI6gswz736domVgDGSDCA8bZBwHs
W7IrEL+1xUXuVdy6A61z6P+pS/ajTGPL+qv+9Jh8UxI
-> ssh-rsa kFDS0A
SV6QVIW8MCQVB8ABiOGxLTXEMO6rfeG82CktBFtf76WeIYzlkho/IaGgWXoqoIQ0
KC/ev7vNGnB01AOWe/xkuMZDRvK+qGaOLB7wpZG1cJhqSon9oZtztoDjd/Crp5K0
nfeHjY9E/jgFr0KYeaLedw5OJuaOw4YiuKyTThVbpRZwbof30nvHXqrYKPZJi1gq
s5spoWYH2ijZi9mrJojP2ZqK5DJjCteXqP1YHdz3LjxomoDyl5cv/tLNsvrptfxD
FvZMcPrvrC/IWqJ8qGW+f8ENUGyjXxx6jFQ2WN9IMIdJYk5bz458ip3GKqnAlwi3
SZbaxRuEYEoy6ikKGRuXMAwpJd3YXcRcaRdetw0a4grdD6hF21bTl2+LnTb1ydnb
frzeoXaqbBdhEyLpZFAmGLydteIyA/Kl/D/PEJ0MHc0G0EGofMm6YsNJJrP3mQgi
mXC2Kto6WV/JLVEnURayf12rPR1T/VPIyYZ/Xi9HfPh0p3Y21nadPAcEq/PltWgR
AqELfBbVpNtcxTP2pjEJqGskJCYKAmMeM+yQ0moKVmuMWicahMqjQRJO1jnvTwwd
GhJlUO32EuI6Fn6sApthv2FfLrle+x0H4/v9xvHDJIVSmLYtzK+9ueUPn/A1x8X1
lGeJh+ecEV2r630insGAp8WQzyXhraHrn3lgyacwRmA
-> ssh-ed25519 YFSOsg KKhXh/XW7iF7wMA7JD9fbgmty5yVPaSS1vGdHz0Xh0M
eLJc+F/yIR1ckZX/npLI+l3I2iB+OrKBkJAQTkbWVF4
-> ssh-ed25519 iHV63A xoJ7Tr8mKgYVPPeJYBnOHLBY5E0i34vEQR3pMVKxbAc
TKqc9Y/RpnfTP3CNvCearB4FuvNmW0mcGVLh7Ebjzeo
-> ssh-ed25519 BVsyTA LaMK6X/MJyQTQ24p9uHXh75leMcp/akCA2YZACEG03M
psw6sVlNGT8WsG3L9kbXdrhqxp8hIdSF7s4o60jTYgY
-> vcxmk`-grease 8^p$~+LB -G)+N&$^ P)7#7[wX
8TyK2RrSHFuMyFy9YY7ZI6RSduF5hw6xZKhiysVkif4Husb1flN8QVmWtoW8laWz
n8772TmNTcfq5ebUp+UA+S6MVgf75D1GnDumEDH/LbM4LNjRZzyw3nBGu/Q
--- Ouu56e69gTpAY1ouLPlzI/n6geKz1CMmTl8wAVyIDPM
Ÿ·¢5¿ä7W>J@°óðjÁ–€l_ƒ¥«­Ï/œö÷ú=ßÕ»‰4(²<18>²K» µÅÑ¥„zSÌsæ

Binary file not shown.

View file

@ -0,0 +1,31 @@
age-encryption.org/v1
-> ssh-ed25519 Y0ZZaw FWuk2kYGB+GfoY3rWfeCosoBOLvUHrH7SR8Fv18o+XI
YyOTULtyOJ3vfAOnYSMzeCCyipJ4Fqrr3PJgRtbElJg
-> ssh-ed25519 iDKjwg Bq6lNuS5MOhsU/7ypHw/E70BktIA+SmN6e3pvrIqRBQ
Xo0OOUXfOkPQfArhqSJyiAkH5lxcJIAO7M5krkCZNfc
-> ssh-ed25519 uYcDNw EfB1B4CSNk8Oe5B7T+KSl9O5OsCrulaLOjR3PBtxpSk
xJxkmBSENc5JosdRiEAC3a41WI6TmTlTxm+lclup+g4
-> ssh-rsa kFDS0A
dYH3A43wClFnDQp8m3ZnhTK5d8LeG6ZkqDQ5dS1yB//4G5TaUnMqOp5Q2G1gbgXY
Zu9qYOHdUydn5HIRSwBXj/KbBm5xJ1zFImOszn7S5mk4iReHFyTnSzAi4utatQcY
DEjGnvKKRoc7ih08+F44kq6DYnhUBFqF8eigQZIsyeWpiW6C1FzasL0KnXoedPG2
AYJForNB8zKp7a2Evxi0MY7a+ldHAekktz1Fta2u9MvrWUtqP/yLqJhCwCNvos7J
kG+XO4j0kiOQCIO9TOeLAu59+VCVM64mY+dp+xc8tX0fWuu7ItSAh6jRHzfgSKjC
qDJc/1YpUG1EnYSH39mfVox3ndeMuVrG6Q1h509jZuxsw/zoDsbY3bbhTaUQ3X8Y
5ShCponnEGBLqeSm1gALCAnlgu8IS4gL6ePKuAhN0qMYj6iiXP/Ugp3lTcv1TvFD
KINnV/tas1CO3PApQm6JgijHEPT9zyUbqR/xN06+OCWbg4hHuEix+0OhM1T5w2xC
KvKF30iUK0tU2hZvKdku2MpbP4N0cQLqBEWiyrUKHRMCdXi3kyO5D84UdWXvETAt
BfEvZ8ZG5fiSXzbPLxVqObXFZUirLuWomWtstqkDuadL9xJkTcsbr8ZCCNpPhxdL
oOfao+tox3RBilAS3AfQVhrPvD2rVUptm+0nPtnO3rY
-> ssh-ed25519 YFSOsg T2OdtA0kY4DqDIxE1QxMV5aCygvKlI5LgXQ+QYYuOko
l0Kzo02jGISCT1zrGf5soXYj7FMVrN/9REF3Zscbmik
-> ssh-ed25519 iHV63A 75daRGD2TQ/mXRsckaH9sGGkHMkLxgHFhn0eDdkDsU8
TXeoLqfU0ywQucPayYoG43Gr56uZoYIWaK9F2YJJ0FM
-> ssh-ed25519 BVsyTA J/xNtG1CAzfoiKPsnWwDp4pId7d3MywXpfhKAmpze3I
8uMO07Se/6krP79flt+XZfjIsw12kWsoD6LqZyLG70M
-> B-grease y3$t@ ; Bs *w
dUrvWB09znCDyvO7RnduMguc9pWTn19q1fc0MHFUXk7WQWns+4kpJIX1qljB5hz/
NPAbNzwMDQKj6awHAth1iFLaEw
--- rI4jrrXCiUpV/EzGsla+lxONmL5/Eel/LODoIM80jcM
˜_°0àÆ7Jˆq•[÷ç<>è'/ù‘õŽi„Ü<E2809E>Òl°mÙ
ÌÂ!JPþ¼>œ… wk¡ž·³¤+ é™)ÚÈPhUÜóç²O=>k=?ÂTÐ

View file

@ -0,0 +1,30 @@
age-encryption.org/v1
-> ssh-ed25519 Y0ZZaw CxhF1nK1+6OmJb/68UQ4mBIqxGgr8ngkNsL9dfaPN2s
jZ/JBaTCjFcL0SAGVx5ECDanVn4TGt0g2yn2OQOP9iY
-> ssh-ed25519 iDKjwg D/xqqA53Lw2UQJesg27wmK/UNCV+s914mvMlbKN1rhg
AOg0SkPvSotuSHk33zVfRxB0wn67a29YWc/itDUZ/LQ
-> ssh-ed25519 uYcDNw /QdfQUJmBMQZ+KRCst1gA0LqFGvM1K91ZL/RIRP+qBc
Ttksa44OdwLuRmgYPC2rIn+wy/SooRPUq8gQTR+pF0k
-> ssh-rsa kFDS0A
L9MGJFRceqbge3EF/rqXdT13jt9faxP1NmfRB3i2mrTasvCaovc/62bA0UmlsB/9
Y3hIzo28d6pZRcMm91l6PhWV0M33YNwPQf87vd7klv++1aMIdZ6/jHsQiohIBkRd
4pBe6rrx/lUqEqfQVYUFPfRE50ufkw+hRw/NJCvcBgHgNhhDoeb8keWRPZhhuv0Z
f0eP9ORKjeKxjv3tsIPjiE7aqxE1zTdrnSr7FuqklJhMYRdwVv+2ofNEh05hU6pR
VL4AS7d6Di/0dWTWc/Je2ytsrdio2v0rPAUXN1fyTh4AtrAmGQzUXNWnr4sB5xH9
QlL0Ea3IwndJSDNkqc4qI3JL0vx6QMUbsuNcMmVWSMkODP+gNQYXQNbnwNfeMAnE
V++WBfyrA8+V+ES+usqeWoOXjApzShn+gnrV0DHHXDAzNR+M647rQcsLePSyNjf/
NKd7Z8VfEq7m65AxmSHPezSGdICMf63WLG/Bffj9rWiQxaoiayGF8jbALpXlu93X
txOw8pK7zA8xFEBujmkrDPH3sJFPLOgOMYa0uuCMbrCGxeJ34nuQMhSUTamESSXb
AD3AgUrRvte1iXwy2PoZGolRLZfdq9zcAfFyq9KvIhvz/8b2F+KbqHQlAiKVPw8p
XQo4sXcDAmF251WSCJGN1C6Doxj/6XLuWILbkobQqoI
-> ssh-ed25519 YFSOsg FtIvWeEXI9blJIFAWMacXgPym5ePGXsuiOR+Gh3b3R8
0rp/NIu4kCCt05Is2+eRdUmgNX8QPMsDPhZWIejnBDA
-> ssh-ed25519 iHV63A 85G1w54UHS/gFcLvsXyYLPXvLHkJl3YQCi8ehb+ZrU8
lXDaMXlPw5ohaaYpiEkCNAmE2tJ2824ydmp9EakPtD8
-> ssh-ed25519 BVsyTA XimcaonVCGGyyCfn3BSX/a7zjJkWeaVY/xAcdNDrl1U
RaqpXzUd54qrkYYRbRTUclTpZdZx2us42lkP6wBxjBM
-> CWM8^B-grease
HvBgzYx54YVP0M6pk1bp9qegLscQ4tHIV9DZhr7jnrW41adgY0D39wnE2IgIRc6g
keRHAr7QVqdPy/kr+u0GwQ1MGFKI8Jss8vRxKwv/UgQfmg
--- dJWXhQRYjxWchTW1u3TrF7KvQIOdrOvkEC7oUtFcGeE
l>qFÞ®/®â@tË\Å&Zò êÄ:„Þ@ ò ÚKÏx©ªr¾áHK ûĦb0ÊÖ—5Ëm¸/

View file

@ -62,4 +62,11 @@ in {
"mediawiki-secret-key.age".publicKeys = nachtigallKeys ++ baseKeys;
"coturn-static-auth-secret.age".publicKeys = nachtigallKeys ++ baseKeys;
"grafana-admin-password.age".publicKeys = flora6Keys ++ baseKeys;
"grafana-keycloak-client-secret.age".publicKeys = flora6Keys ++ baseKeys;
"grafana-smtp-password.age".publicKeys = flora6Keys ++ baseKeys;
"nachtigall-metrics-nginx-basic-auth.age".publicKeys = nachtigallKeys ++ baseKeys;
"nachtigall-metrics-prometheus-basic-auth-password.age".publicKeys = flora6Keys ++ nachtigallKeys ++ baseKeys;
}

View file

@ -38,13 +38,11 @@ resource "namecheap_domain_records" "pub-solar" {
hostname = "obs-portal"
type = "A"
address = "80.71.153.210"
ttl = 60
}
record {
hostname = "vpn"
type = "A"
address = "80.71.153.210"
ttl = 60
}
record {
hostname = "cache"
@ -81,13 +79,16 @@ resource "namecheap_domain_records" "pub-solar" {
hostname = "turn"
type = "A"
address = "138.201.80.102"
ttl = 300
}
record {
hostname = "grafana"
type = "A"
address = "80.71.153.210"
}
record {
hostname = "hpb"
type = "A"
address = "80.71.153.239"
ttl = 60
}
record {
hostname = "files"