Agent-Intervall 6h+Jitter, Debug-Modus, Health-Monitoring
Siehe neue ADR (platform-docs). Migration 0022 (devices.debug_mode_until +
6 Health-Spalten). AGENT_POLL_INTERVAL_SECONDS-Default 300 -> 21600 (6h,
Jitter kommt agentenseitig). agent_checkin() liefert dynamisch das kurze
Debug-Intervall statt des Standards, solange debug_mode_until in der
Zukunft liegt - reiner Zeitvergleich, kein Cron zum Zurücksetzen nötig.
Neue Endpunkte POST/DELETE /api/v1/devices/{id}/debug-mode. Check-in
nimmt optional health-Objekt entgegen (Disk/RAM/Uptime/Load), schreibt es
kombiniert mit agent_last_checkin in einem UPDATE. fetch_devices_for_
organization()/fetch_all_devices() liefern debug_mode_until + Health jetzt
mit aus (Kundenplattform braucht sie für Anzeige/Toggle-Zustand, kein
Zusatz-Request nötig).
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
eeb5879a46
commit
2f6ff909f9
151
app.py
151
app.py
@ -1,5 +1,5 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import json
|
import json
|
||||||
@ -22,7 +22,15 @@ from fastapi.responses import FileResponse
|
|||||||
|
|
||||||
LOG_PATH = Path("activation.log")
|
LOG_PATH = Path("activation.log")
|
||||||
DATABASE_URL = os.environ.get("TUXFLOTTE_DATABASE_URL")
|
DATABASE_URL = os.environ.get("TUXFLOTTE_DATABASE_URL")
|
||||||
AGENT_POLL_INTERVAL_SECONDS = int(os.environ.get("TUXFLOTTE_AGENT_POLL_INTERVAL", "300"))
|
# Standard 6h (26.08.2026-Planung) - Jitter kommt bewusst nicht von hier,
|
||||||
|
# sondern aus dem Agenten selbst (agent.py), der pro Geraet zufaellig
|
||||||
|
# streut, damit nicht alle Geraete exakt gleichzeitig einchecken.
|
||||||
|
AGENT_POLL_INTERVAL_SECONDS = int(os.environ.get("TUXFLOTTE_AGENT_POLL_INTERVAL", str(6 * 60 * 60)))
|
||||||
|
# Debug-Modus: kurzes Intervall zum Testen, faellt nach DEBUG_MODE_DURATION_HOURS
|
||||||
|
# automatisch auf AGENT_POLL_INTERVAL_SECONDS zurueck - rein durch Zeitvergleich
|
||||||
|
# bei jedem Checkin, kein Cron/Aufraeum-Job noetig (siehe agent_checkin()).
|
||||||
|
DEBUG_MODE_POLL_INTERVAL_SECONDS = 60
|
||||||
|
DEBUG_MODE_DURATION_HOURS = 6
|
||||||
ANSIBLE_CONTENT_REPO = os.environ.get("TUXFLOTTE_ANSIBLE_CONTENT_REPO", "")
|
ANSIBLE_CONTENT_REPO = os.environ.get("TUXFLOTTE_ANSIBLE_CONTENT_REPO", "")
|
||||||
ADMIN_TOKEN = os.environ.get("TUXFLOTTE_ADMIN_TOKEN", "")
|
ADMIN_TOKEN = os.environ.get("TUXFLOTTE_ADMIN_TOKEN", "")
|
||||||
KUNDENPLATTFORM_TOKEN = os.environ.get("TUXFLOTTE_KUNDENPLATTFORM_TOKEN", "")
|
KUNDENPLATTFORM_TOKEN = os.environ.get("TUXFLOTTE_KUNDENPLATTFORM_TOKEN", "")
|
||||||
@ -77,8 +85,25 @@ class AgentBootstrapRequest(BaseModel):
|
|||||||
device_id: str
|
device_id: str
|
||||||
|
|
||||||
|
|
||||||
|
class AgentHealthPayload(BaseModel):
|
||||||
|
"""
|
||||||
|
Health-Metriken (26.08.2026-Planung) - alle optional, damit ein
|
||||||
|
fehlgeschlagener Einzelwert (z.B. load average auf ungewoehnlichen
|
||||||
|
Systemen) nicht den ganzen Checkin blockiert. Werte kommen direkt aus
|
||||||
|
Python-Stdlib-Aufrufen im Agenten (kein neues Paket noetig).
|
||||||
|
"""
|
||||||
|
|
||||||
|
disk_used_percent: float | None = None
|
||||||
|
ram_used_percent: float | None = None
|
||||||
|
uptime_seconds: int | None = None
|
||||||
|
load_1m: float | None = None
|
||||||
|
load_5m: float | None = None
|
||||||
|
load_15m: float | None = None
|
||||||
|
|
||||||
|
|
||||||
class AgentCheckinRequest(BaseModel):
|
class AgentCheckinRequest(BaseModel):
|
||||||
device_id: str
|
device_id: str
|
||||||
|
health: AgentHealthPayload | None = None
|
||||||
|
|
||||||
|
|
||||||
class AgentReportEntry(BaseModel):
|
class AgentReportEntry(BaseModel):
|
||||||
@ -1360,6 +1385,29 @@ def fetch_device_gruppen(device_id: str):
|
|||||||
)
|
)
|
||||||
return [{"id": str(gruppe_id), "name": name} for gruppe_id, name in cur.fetchall()]
|
return [{"id": str(gruppe_id), "name": name} for gruppe_id, name in cur.fetchall()]
|
||||||
|
|
||||||
|
# Debug-Modus + Health-Spalten (26.08.2026-Planung) - gemeinsam von
|
||||||
|
# fetch_devices_for_organization() und fetch_all_devices() gebraucht, daher
|
||||||
|
# als Konstante statt zweimal ausgeschrieben.
|
||||||
|
GERAETE_HEALTH_SPALTEN_SQL = """d.debug_mode_until, d.health_disk_used_percent, d.health_ram_used_percent,
|
||||||
|
d.health_uptime_seconds, d.health_load_1m, d.health_load_5m, d.health_load_15m,
|
||||||
|
d.health_collected_at"""
|
||||||
|
|
||||||
|
|
||||||
|
def _health_felder_aus_row(debug_mode_until, disk, ram, uptime, load1, load5, load15, collected_at) -> dict:
|
||||||
|
return {
|
||||||
|
"debug_mode_until": debug_mode_until.isoformat() if debug_mode_until is not None else None,
|
||||||
|
"health": {
|
||||||
|
"disk_used_percent": disk,
|
||||||
|
"ram_used_percent": ram,
|
||||||
|
"uptime_seconds": uptime,
|
||||||
|
"load_1m": load1,
|
||||||
|
"load_5m": load5,
|
||||||
|
"load_15m": load15,
|
||||||
|
"collected_at": collected_at.isoformat() if collected_at is not None else None,
|
||||||
|
} if collected_at is not None else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def fetch_devices_for_organization(organization_id: str):
|
def fetch_devices_for_organization(organization_id: str):
|
||||||
"""
|
"""
|
||||||
Für Kundenplattforms Geräteliste + Besitz-Validierung (siehe ADR-0011):
|
Für Kundenplattforms Geräteliste + Besitz-Validierung (siehe ADR-0011):
|
||||||
@ -1370,9 +1418,9 @@ def fetch_devices_for_organization(organization_id: str):
|
|||||||
with get_database_connection() as conn:
|
with get_database_connection() as conn:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
f"""
|
||||||
SELECT d.id, d.hostname, d.device_fingerprint, d.agent_last_checkin,
|
SELECT d.id, d.hostname, d.device_fingerprint, d.agent_last_checkin,
|
||||||
d.deprovisioned_at, d.oe_id, oe.name
|
d.deprovisioned_at, d.oe_id, oe.name, {GERAETE_HEALTH_SPALTEN_SQL}
|
||||||
FROM devices d
|
FROM devices d
|
||||||
LEFT JOIN organisationseinheiten oe ON oe.id = d.oe_id
|
LEFT JOIN organisationseinheiten oe ON oe.id = d.oe_id
|
||||||
WHERE d.organization_id = %s AND d.archived_at IS NULL
|
WHERE d.organization_id = %s AND d.archived_at IS NULL
|
||||||
@ -1393,8 +1441,12 @@ def fetch_devices_for_organization(organization_id: str):
|
|||||||
),
|
),
|
||||||
"oe_id": str(oe_id) if oe_id is not None else None,
|
"oe_id": str(oe_id) if oe_id is not None else None,
|
||||||
"oe_name": oe_name,
|
"oe_name": oe_name,
|
||||||
|
**_health_felder_aus_row(*health_spalten),
|
||||||
}
|
}
|
||||||
for device_id, hostname, fingerprint, last_checkin, deprovisioned_at, oe_id, oe_name in cur.fetchall()
|
for (
|
||||||
|
device_id, hostname, fingerprint, last_checkin, deprovisioned_at, oe_id, oe_name,
|
||||||
|
*health_spalten,
|
||||||
|
) in cur.fetchall()
|
||||||
]
|
]
|
||||||
|
|
||||||
def fetch_all_devices():
|
def fetch_all_devices():
|
||||||
@ -1407,9 +1459,10 @@ def fetch_all_devices():
|
|||||||
with get_database_connection() as conn:
|
with get_database_connection() as conn:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
f"""
|
||||||
SELECT d.id, d.hostname, d.device_fingerprint, d.agent_last_checkin,
|
SELECT d.id, d.hostname, d.device_fingerprint, d.agent_last_checkin,
|
||||||
d.organization_id, o.name, d.deprovisioned_at, d.oe_id, oe.name
|
d.organization_id, o.name, d.deprovisioned_at, d.oe_id, oe.name,
|
||||||
|
{GERAETE_HEALTH_SPALTEN_SQL}
|
||||||
FROM devices d
|
FROM devices d
|
||||||
JOIN organizations o ON o.id = d.organization_id
|
JOIN organizations o ON o.id = d.organization_id
|
||||||
LEFT JOIN organisationseinheiten oe ON oe.id = d.oe_id
|
LEFT JOIN organisationseinheiten oe ON oe.id = d.oe_id
|
||||||
@ -1432,11 +1485,13 @@ def fetch_all_devices():
|
|||||||
),
|
),
|
||||||
"oe_id": str(oe_id) if oe_id is not None else None,
|
"oe_id": str(oe_id) if oe_id is not None else None,
|
||||||
"oe_name": oe_name,
|
"oe_name": oe_name,
|
||||||
|
**_health_felder_aus_row(*health_spalten),
|
||||||
}
|
}
|
||||||
for (
|
for (
|
||||||
device_id, hostname, fingerprint, last_checkin,
|
device_id, hostname, fingerprint, last_checkin,
|
||||||
organization_id, organization_name, deprovisioned_at,
|
organization_id, organization_name, deprovisioned_at,
|
||||||
oe_id, oe_name,
|
oe_id, oe_name,
|
||||||
|
*health_spalten,
|
||||||
) in cur.fetchall()
|
) in cur.fetchall()
|
||||||
]
|
]
|
||||||
|
|
||||||
@ -2577,19 +2632,57 @@ def agent_checkin(
|
|||||||
"message": "Ungültiges Agent-Secret.",
|
"message": "Ungültiges Agent-Secret.",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
health = payload.health
|
||||||
|
jetzt_ts = datetime.now(timezone.utc)
|
||||||
|
|
||||||
with get_database_connection() as conn:
|
with get_database_connection() as conn:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
|
# Ein kombiniertes UPDATE ... RETURNING statt zwei Roundtrips -
|
||||||
|
# debug_mode_until wird direkt mitgeliefert, um das effektive
|
||||||
|
# Intervall unten zu bestimmen (siehe ADR: Debug-Modus faellt
|
||||||
|
# rein durch Zeitvergleich automatisch zurueck, kein Cron noetig).
|
||||||
|
# Die health_*-Spalten werden nur ueberschrieben, wenn dieser
|
||||||
|
# Checkin ueberhaupt ein health-Objekt mitschickt - fehlt es
|
||||||
|
# (z.B. Sammel-Fehler im Agenten), bleibt der letzte bekannte
|
||||||
|
# Stand stehen statt auf NULL zurueckzufallen.
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"UPDATE devices SET agent_last_checkin = %s WHERE id = %s",
|
"""
|
||||||
(datetime.now(timezone.utc), payload.device_id),
|
UPDATE devices
|
||||||
|
SET agent_last_checkin = %(jetzt)s,
|
||||||
|
health_disk_used_percent = CASE WHEN %(hat_health)s THEN %(disk)s ELSE health_disk_used_percent END,
|
||||||
|
health_ram_used_percent = CASE WHEN %(hat_health)s THEN %(ram)s ELSE health_ram_used_percent END,
|
||||||
|
health_uptime_seconds = CASE WHEN %(hat_health)s THEN %(uptime)s ELSE health_uptime_seconds END,
|
||||||
|
health_load_1m = CASE WHEN %(hat_health)s THEN %(load1)s ELSE health_load_1m END,
|
||||||
|
health_load_5m = CASE WHEN %(hat_health)s THEN %(load5)s ELSE health_load_5m END,
|
||||||
|
health_load_15m = CASE WHEN %(hat_health)s THEN %(load15)s ELSE health_load_15m END,
|
||||||
|
health_collected_at = CASE WHEN %(hat_health)s THEN %(jetzt)s ELSE health_collected_at END
|
||||||
|
WHERE id = %(device_id)s
|
||||||
|
RETURNING debug_mode_until
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"jetzt": jetzt_ts,
|
||||||
|
"hat_health": health is not None,
|
||||||
|
"disk": health.disk_used_percent if health else None,
|
||||||
|
"ram": health.ram_used_percent if health else None,
|
||||||
|
"uptime": health.uptime_seconds if health else None,
|
||||||
|
"load1": health.load_1m if health else None,
|
||||||
|
"load5": health.load_5m if health else None,
|
||||||
|
"load15": health.load_15m if health else None,
|
||||||
|
"device_id": payload.device_id,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
row = cur.fetchone()
|
||||||
|
debug_mode_until = row[0] if row else None
|
||||||
|
|
||||||
|
im_debug_modus = debug_mode_until is not None and debug_mode_until > jetzt_ts
|
||||||
|
effektives_intervall = DEBUG_MODE_POLL_INTERVAL_SECONDS if im_debug_modus else AGENT_POLL_INTERVAL_SECONDS
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"success": True,
|
"success": True,
|
||||||
"blueprints": fetch_assigned_blueprints(payload.device_id),
|
"blueprints": fetch_assigned_blueprints(payload.device_id),
|
||||||
"auftraege": fetch_auftragskatalog_state(payload.device_id),
|
"auftraege": fetch_auftragskatalog_state(payload.device_id),
|
||||||
"ansible_repo": ANSIBLE_CONTENT_REPO,
|
"ansible_repo": ANSIBLE_CONTENT_REPO,
|
||||||
"poll_interval_seconds": AGENT_POLL_INTERVAL_SECONDS,
|
"poll_interval_seconds": effektives_intervall,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -3487,3 +3580,41 @@ def remove_device_gruppe_endpoint(device_id: str, gruppe_id: str, authorization:
|
|||||||
remove_device_from_gruppe(device_id, gruppe_id)
|
remove_device_from_gruppe(device_id, gruppe_id)
|
||||||
|
|
||||||
return {"success": True}
|
return {"success": True}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/devices/{device_id}/debug-mode")
|
||||||
|
def aktiviere_debug_modus(device_id: str, authorization: str | None = Header(default=None)):
|
||||||
|
"""
|
||||||
|
Debug-Modus (26.08.2026-Planung): setzt debug_mode_until auf jetzt+6h -
|
||||||
|
agent_checkin() vergleicht das bei jedem Checkin und liefert solange das
|
||||||
|
kurze Debug-Intervall statt des Standardintervalls, faellt danach von
|
||||||
|
selbst zurueck (kein Cron noetig). Org-Zugehoerigkeitspruefung ist wie
|
||||||
|
ueberall Aufgabe der aufrufenden Kundenplattform (find_device_in_
|
||||||
|
organization()), nicht dieses Endpunkts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not require_service_token(authorization):
|
||||||
|
return {"success": False, "error": "unauthorized", "message": "Fehlendes oder ungültiges Service-Token."}
|
||||||
|
|
||||||
|
debug_bis = datetime.now(timezone.utc) + timedelta(hours=DEBUG_MODE_DURATION_HOURS)
|
||||||
|
|
||||||
|
with get_database_connection() as conn:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"UPDATE devices SET debug_mode_until = %s WHERE id = %s",
|
||||||
|
(debug_bis, device_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"success": True, "debug_mode_until": debug_bis.isoformat()}
|
||||||
|
|
||||||
|
|
||||||
|
@app.delete("/api/v1/devices/{device_id}/debug-mode")
|
||||||
|
def deaktiviere_debug_modus(device_id: str, authorization: str | None = Header(default=None)):
|
||||||
|
if not require_service_token(authorization):
|
||||||
|
return {"success": False, "error": "unauthorized", "message": "Fehlendes oder ungültiges Service-Token."}
|
||||||
|
|
||||||
|
with get_database_connection() as conn:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("UPDATE devices SET debug_mode_until = NULL WHERE id = %s", (device_id,))
|
||||||
|
|
||||||
|
return {"success": True}
|
||||||
|
|||||||
17
migrations/0022_agent_debug_mode_und_health.sql
Normal file
17
migrations/0022_agent_debug_mode_und_health.sql
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
-- Agent-Intervall/Debug-Modus/Health-Monitoring (26.08.2026-Planung, siehe
|
||||||
|
-- neue ADR). debug_mode_until ist der einzige Zustand fuer den Debug-Modus -
|
||||||
|
-- der Checkin-Handler vergleicht ihn bei jeder Anfrage mit CURRENT_TIMESTAMP,
|
||||||
|
-- kein separater Cron/Aufraeum-Job noetig, das Zurueckfallen auf das
|
||||||
|
-- Standardintervall passiert von selbst.
|
||||||
|
--
|
||||||
|
-- Health-Spalten halten bewusst nur den jeweils letzten Stand (wie schon
|
||||||
|
-- agent_last_checkin), keine wachsende Historie - konsistent mit dem
|
||||||
|
-- Plattenplatz-Bewusstsein an anderer Stelle (siehe iso_builds-Aufraeumung).
|
||||||
|
ALTER TABLE devices ADD COLUMN debug_mode_until TIMESTAMPTZ;
|
||||||
|
ALTER TABLE devices ADD COLUMN health_disk_used_percent REAL;
|
||||||
|
ALTER TABLE devices ADD COLUMN health_ram_used_percent REAL;
|
||||||
|
ALTER TABLE devices ADD COLUMN health_uptime_seconds BIGINT;
|
||||||
|
ALTER TABLE devices ADD COLUMN health_load_1m REAL;
|
||||||
|
ALTER TABLE devices ADD COLUMN health_load_5m REAL;
|
||||||
|
ALTER TABLE devices ADD COLUMN health_load_15m REAL;
|
||||||
|
ALTER TABLE devices ADD COLUMN health_collected_at TIMESTAMPTZ;
|
||||||
Loading…
x
Reference in New Issue
Block a user