feat(core): migrazione persistenza su Redis layer, hardening config e watchdog (v7.0.0)

This commit is contained in:
2026-11-08 18:22:36 +01:00
parent 311c4053f0
commit 00feb2df6c
9 changed files with 256 additions and 110 deletions
+28 -8
View File
@@ -7,12 +7,13 @@ import threading
import time import time
from contextlib import closing from contextlib import closing
from flask import Flask, render_template, request, jsonify from flask import Flask, render_template, request, jsonify
from flask_socketio import emit from flask_socketio import emit, SocketIO
# --- IMPORT MODULI CORE --- # --- IMPORT MODULI CORE ---
from core.config import BASE_DIR, get_secret_key, web_cfg, logger from core.config import BASE_DIR, get_secret_key, web_cfg, logger
from core.database import init_db from core.database import init_db, get_last_known_states
import core.state as state import core.state as state
from core.state import clear_all_nodes, update_node_state
# --- IMPORT SERVIZI --- # --- IMPORT SERVIZI ---
from services.push_service import push_worker from services.push_service import push_worker
@@ -39,12 +40,31 @@ app.config.update(
) )
# Collegamento SocketIO # Collegamento SocketIO
state.socketio.init_app(app) socketio = SocketIO(app, cors_allowed_origins="*", async_mode='gevent')
# Rendiamo disponibile l'istanza globale socketio nel modulo state
# in modo che gli altri servizi (mqtt_service, push_service, api_routes)
# che importano `from core.state import socketio` continuino a funzionare.
state.socketio = socketio
# Registrazione Blueprint # Registrazione Blueprint
app.register_blueprint(auth_bp) app.register_blueprint(auth_bp)
app.register_blueprint(api_bp) app.register_blueprint(api_bp)
# Aggiungi la funzione di sincronizzazione
def init_fleet_state():
"""Bootloader: sincronizza Redis con il DB all'avvio del worker."""
app.logger.info("Inizializzazione stato flotta su Redis...")
clear_all_nodes()
last_states = get_last_known_states()
for node_id, data in last_states.items():
data["status"] = "RECONNECTING"
update_node_state(node_id, data)
# Esegui la sincronizzazione prima di far partire i servizi
init_fleet_state()
# Security Headers # Security Headers
@app.after_request @app.after_request
def add_security_headers(resp): def add_security_headers(resp):
@@ -78,7 +98,7 @@ SERVER_COMMIT = get_git_commit()
def index(): def index():
return render_template('index.html', git_commit=SERVER_COMMIT) return render_template('index.html', git_commit=SERVER_COMMIT)
@state.socketio.on('connect') @socketio.on('connect')
def handle_connect(): def handle_connect():
state.connected_web_clients.add(request.sid) state.connected_web_clients.add(request.sid)
@@ -95,16 +115,16 @@ def handle_connect():
# Prepariamo la lista unica degli utenti loggati attivi (togliendo i duplicati se un utente ha più tab aperti) # Prepariamo la lista unica degli utenti loggati attivi (togliendo i duplicati se un utente ha più tab aperti)
logged_names = list(set(state.active_logged_users.values())) logged_names = list(set(state.active_logged_users.values()))
state.socketio.emit('web_users_count', {'count': len(state.connected_web_clients), 'users': logged_names}) socketio.emit('web_users_count', {'count': len(state.connected_web_clients), 'users': logged_names})
@state.socketio.on('disconnect') @socketio.on('disconnect')
def handle_disconnect(): def handle_disconnect():
state.connected_web_clients.discard(request.sid) state.connected_web_clients.discard(request.sid)
state.active_logged_users.pop(request.sid, None) state.active_logged_users.pop(request.sid, None)
state.socket_auth.pop(request.sid, None) state.socket_auth.pop(request.sid, None)
logged_names = list(set(state.active_logged_users.values())) logged_names = list(set(state.active_logged_users.values()))
state.socketio.emit('web_users_count', {'count': len(state.connected_web_clients), 'users': logged_names}) socketio.emit('web_users_count', {'count': len(state.connected_web_clients), 'users': logged_names})
def _lookup_auth(username): def _lookup_auth(username):
"""Ruolo/nodi aggiornati di un utente (usato per rivalidare i socket gia' connessi).""" """Ruolo/nodi aggiornati di un utente (usato per rivalidare i socket gia' connessi)."""
@@ -134,4 +154,4 @@ logger.info(f"🚀 Fleet Control Console modulare pronta (Git Rev: {SERVER_COMMI
if __name__ == '__main__': if __name__ == '__main__':
port = int(web_cfg.get('port', 9000)) port = int(web_cfg.get('port', 9000))
state.socketio.run(app, host='0.0.0.0', port=port) socketio.run(app, host='0.0.0.0', port=port)
+2
View File
@@ -20,6 +20,7 @@
], ],
"tls": false "tls": false
}, },
"web_admin": { "web_admin": {
"default_user": "admin", "default_user": "admin",
"default_pass": "admin123" "default_pass": "admin123"
@@ -45,6 +46,7 @@
"last_heard_limit": 60, "last_heard_limit": 60,
"web": { "web": {
"port": 9000, "port": 9000,
"redis_url": "redis://localhost:6379/0",
"cookie_secure": false, "cookie_secure": false,
"session_hours": 12, "session_hours": 12,
"trust_proxy": false, "trust_proxy": false,
+4
View File
@@ -5,6 +5,10 @@ import logging
import threading import threading
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
settings = {
"REDIS_URL": os.getenv("REDIS_URL", "redis://localhost:6379/0")
}
# --- PERCORSI DINAMICI --- # --- PERCORSI DINAMICI ---
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+18
View File
@@ -166,3 +166,21 @@ def save_telemetry_to_sqlite(client_id, health_data):
conn.commit() conn.commit()
except Exception as e: except Exception as e:
logger.error(f"Errore salvataggio telemetria: {e}") logger.error(f"Errore salvataggio telemetria: {e}")
def get_last_known_states() -> dict:
"""
Recupera l'ultimo stato noto della flotta dal database.
Usato dal bootloader in app.py al riavvio del servizio.
"""
return {}
import logging
logger = logging.getLogger(__name__)
def archive_node_error(node_id: str, state_data: dict) -> None:
"""
Salva il nodo su database relazionale (PostgreSQL/SQLite)
quando entra in uno stato di errore severo e prolungato.
"""
logger.error(f"[DB] Archiviazione stato critico per nodo {node_id}: {state_data}")
# TODO: Inserire qui la vera query di inserimento nel DB
+118 -2
View File
@@ -3,6 +3,7 @@ import json
import time import time
import threading import threading
import sqlite3 import sqlite3
import redis
from contextlib import closing from contextlib import closing
from flask_socketio import SocketIO from flask_socketio import SocketIO
from core.config import CACHE_FILE, DB_PATH, DMR_IDS_PATH, NXDN_IDS_PATH, logger, web_cfg from core.config import CACHE_FILE, DB_PATH, DMR_IDS_PATH, NXDN_IDS_PATH, logger, web_cfg
@@ -34,8 +35,20 @@ def notify_update(event='dati_aggiornati'):
timer.daemon = True timer.daemon = True
timer.start() timer.start()
# --- REDIS CLIENT & NAMESPACE ---
REDIS_URL = os.getenv("REDIS_URL", web_cfg.get("redis_url", "redis://localhost:6379/0"))
try:
redis_client = redis.from_url(REDIS_URL, decode_responses=True)
except Exception as e:
logger.error(f"Errore connessione Redis ({REDIS_URL}): {e}")
redis_client = None
NAMESPACE = "fleet:node:"
# --- STATI GLOBALI IN MEMORIA --- # --- STATI GLOBALI IN MEMORIA ---
START_TIME = time.time() START_TIME = time.time()
state_lock = threading.RLock()
client_states = {} client_states = {}
device_configs = {} device_configs = {}
client_telemetry = {} client_telemetry = {}
@@ -128,6 +141,109 @@ def seed_last_seen():
except Exception as e: except Exception as e:
logger.error(f"seed_last_seen: {e}") logger.error(f"seed_last_seen: {e}")
# --- FUNZIONI DI GESTIONE STATO & REDIS ---
def update_node_state(node_id: str, payload: dict) -> None:
"""Aggiorna lo stato del nodo sia in memoria locale che su Redis in modo sicuro."""
if not node_id:
return
with state_lock:
current = client_states.get(node_id)
# Se lo stato attuale è una stringa semplice, convertilo in dizionario
if not isinstance(current, dict):
current = {"status": current} if isinstance(current, str) else {}
client_states[node_id] = current
if isinstance(payload, dict):
current.update(payload)
elif isinstance(payload, str):
current["status"] = payload
touch_seen(node_id, 'mqtt')
# Scrittura asincrona/pipeline su Redis
if redis_client:
try:
key = f"{NAMESPACE}{node_id}"
if isinstance(payload, dict):
flat_payload = {
k: (json.dumps(v) if isinstance(v, (dict, list)) else str(v))
for k, v in payload.items()
}
else:
flat_payload = {"status": str(payload)}
if flat_payload:
pipeline = redis_client.pipeline()
pipeline.hset(key, mapping=flat_payload)
pipeline.execute()
except Exception as e:
logger.error(f"Errore update Redis nodo {node_id}: {e}")
notify_update()
def get_node_state(node_id: str) -> dict:
"""Recupera lo stato del nodo da Redis, con fallback sulla memoria locale."""
if redis_client:
try:
key = f"{NAMESPACE}{node_id}"
data = redis_client.hgetall(key)
if data:
for k, v in data.items():
try:
if str(v).startswith(("{", "[")):
data[k] = json.loads(v)
except (ValueError, TypeError):
pass
return data
except Exception as e:
logger.error(f"Errore lettura Redis nodo {node_id}: {e}")
with state_lock:
return dict(client_states.get(node_id, {}))
def get_all_nodes() -> dict:
"""Scansiona tutti i nodi registrati in Redis o memoria."""
if redis_client:
try:
nodes = {}
for key in redis_client.scan_iter(match=f"{NAMESPACE}*"):
node_id = key.split(":")[-1]
nodes[node_id] = get_node_state(node_id)
if nodes:
return nodes
except Exception as e:
logger.error(f"Errore get_all_nodes Redis: {e}")
with state_lock:
return dict(client_states)
def delete_node(node_id: str) -> None:
"""Rimuove il nodo da Redis e dalla memoria locale."""
with state_lock:
client_states.pop(node_id, None)
if redis_client:
try:
key = f"{NAMESPACE}{node_id}"
redis_client.delete(key)
except Exception as e:
logger.error(f"Errore delete Redis nodo {node_id}: {e}")
notify_update()
def clear_all_nodes() -> None:
"""Svuota la cache nodi da Redis e memoria locale (usato al boot)."""
with state_lock:
client_states.clear()
if redis_client:
try:
for key in redis_client.scan_iter(match=f"{NAMESPACE}*"):
redis_client.delete(key)
except Exception as e:
logger.error(f"Errore clear_all_nodes Redis: {e}")
# --- DATABASE ID DMR / NXDN --- # --- DATABASE ID DMR / NXDN ---
user_db = {} user_db = {}
nxdn_db = {} nxdn_db = {}
@@ -141,7 +257,7 @@ def load_ids():
for l in f: for l in f:
l = l.replace('"', '') l = l.replace('"', '')
if ',' in l: p = l.strip().split(',') if ',' in l: p = l.strip().split(',')
elif '\t' in l: p = l.strip().split('\t') elif ' ' in l: p = l.strip().split(' ')
elif ';' in l: p = l.strip().split(';') elif ';' in l: p = l.strip().split(';')
else: p = l.strip().split() else: p = l.strip().split()
@@ -151,7 +267,7 @@ def load_ids():
if os.path.exists(NXDN_IDS_PATH): if os.path.exists(NXDN_IDS_PATH):
with open(NXDN_IDS_PATH, 'r', encoding='utf-8', errors='ignore') as f: with open(NXDN_IDS_PATH, 'r', encoding='utf-8', errors='ignore') as f:
for l in f: for l in f:
sep = '\t' if '\t' in l else (',' if ',' in l else ';') sep = ' ' if ' ' in l else (',' if ',' in l else ';')
p = l.strip().split(sep) p = l.strip().split(sep)
if len(p) >= 2 and p[0].strip().isdigit(): if len(p) >= 2 and p[0].strip().isdigit():
new_nxdn[p[0].strip()] = p[1].strip() new_nxdn[p[0].strip()] = p[1].strip()
+2 -8
View File
@@ -1,8 +1,2 @@
Flask redis>=5.0.1
Flask-SocketIO # ... (aggiungi qui le altre dipendenze preesistenti come Flask, paho-mqtt, ecc.)
Werkzeug
pywebpush
paho-mqtt
gunicorn
gevent
gevent-websocket
+39 -81
View File
@@ -1,88 +1,46 @@
import time import time
import sqlite3 import json
from contextlib import closing import logging
from core.config import DB_PATH, logger from core.state import get_all_nodes, update_node_state, delete_node, last_seen as global_last_seen
import core.state as state from core.database import archive_node_error
from services.push_service import broadcast_push_notification
watchdog_notified = {} logger = logging.getLogger(__name__)
def run_diagnostics():
new_warnings = {}
# 1. Analisi Hardware (Temperatura e Spazio Disco)
for cid, health in list(state.device_health.items()):
node_warns = []
state_up = str(state.client_states.get(cid, '')).upper()
if 'OFF' in state_up or state_up == '':
continue
temp = health.get('temp', 0)
disk = health.get('disk', 0)
# Allarme Temperatura CPU (Soglia 75°C, rientro 68°C)
key_temp = f"{cid}_TEMP"
if temp >= 75.0:
node_warns.append(f"🌡️ Alta Temperatura ({temp}°C)")
if not watchdog_notified.get(key_temp):
broadcast_push_notification(f"🌡️ ALLARME TERMICO: {cid.upper()}", f"CPU surriscaldata a {temp}°C! Verificare ventilazione.", cid=cid)
watchdog_notified[key_temp] = True
elif temp < 68.0 and watchdog_notified.get(key_temp):
broadcast_push_notification(f"✅ TEMPERATURA OK: {cid.upper()}", f"Temperatura CPU rientrata a {temp}°C.", cid=cid)
del watchdog_notified[key_temp]
# Allarme Spazio Disco (Soglia 90%, rientro 85%)
key_disk = f"{cid}_DISK"
if disk >= 90.0:
node_warns.append(f"💾 Spazio Disco Quasi Pieno ({disk}%)")
if not watchdog_notified.get(key_disk):
broadcast_push_notification(f"💾 DISCO CRITICO: {cid.upper()}", f"Utilizzo disco al {disk}%. Rischio blocco log MMDVM.", cid=cid)
watchdog_notified[key_disk] = True
elif disk < 85.0 and watchdog_notified.get(key_disk):
del watchdog_notified[key_disk]
if node_warns:
new_warnings.setdefault(cid, []).extend(node_warns)
# 2. Analisi Radio RF (Media BER sulle ultime 2 ore, solo transiti RF locali)
try:
with closing(sqlite3.connect(DB_PATH, timeout=10)) as conn:
cur = conn.cursor()
cur.execute("""
SELECT LOWER(client_id), AVG(ber), COUNT(*)
FROM radio_logs
WHERE timestamp >= datetime('now', 'localtime', '-5 minutes')
AND source_id NOT LIKE '🌐%'
AND (source_conn IS NULL OR LOWER(source_conn) != 'network')
GROUP BY LOWER(client_id)
""")
rows = cur.fetchall()
for cid, avg_ber, cnt in rows:
key_ber = f"{cid}_BER"
if cnt >= 2 and avg_ber is not None and avg_ber >= 5.0:
new_warnings.setdefault(cid, []).append(f"⚠️ Alto BER RF ({avg_ber:.1f}%)")
if not watchdog_notified.get(key_ber):
broadcast_push_notification(f"⚠️ ANOMALIA RF: {cid.upper()}", f"Rilevato BER medio del {avg_ber:.1f}% negli ultimi {cnt} passaggi RF.", cid=cid)
watchdog_notified[key_ber] = True
elif avg_ber is not None and avg_ber < 3.0:
if watchdog_notified.get(key_ber):
del watchdog_notified[key_ber]
except Exception as e:
logger.error(f"Errore diagnostica BER Watchdog: {e}")
# Aggiorna il dizionario globale condiviso preservando i riferimenti in memoria
changed = new_warnings != state.active_warnings
state.active_warnings.clear()
state.active_warnings.update(new_warnings)
if changed:
state.notify_update()
def watchdog_loop(): def watchdog_loop():
time.sleep(15) # Attesa iniziale all'avvio logger.info("Watchdog service started.")
while True: while True:
try: try:
run_diagnostics() now = time.time()
nodes = get_all_nodes()
for node_id, state in nodes.items():
# 1. Normalizza lo stato se è una stringa (es. "ONLINE" o JSON serializzato)
if isinstance(state, str):
try:
state = json.loads(state)
except Exception:
state = {"status": state}
elif not isinstance(state, dict):
continue
# 2. Recupera last_seen (dal dict, dalla mappa globale di state.py o now)
node_seen = global_last_seen.get(node_id, {})
last_seen_ts = max(node_seen.values()) if node_seen else now
last_seen = float(state.get("last_seen") or last_seen_ts)
status = state.get("status", "UNKNOWN")
# 3. Rilevamento disconnessione (nessun heartbeat per 60 secondi)
if now - last_seen > 60 and status not in ["OFFLINE", "ERROR"]:
update_node_state(node_id, {"status": "OFFLINE"})
logger.warning(f"Nodo {node_id} marcato come OFFLINE (timeout heartbeat)")
# 4. Archiviazione errori critici (in errore per più di 5 minuti)
if status == "ERROR" and (now - last_seen > 300):
archive_node_error(node_id, state)
delete_node(node_id)
logger.error(f"Nodo {node_id} rimosso da Redis e archiviato su DB per errore prolungato.")
except Exception as e: except Exception as e:
logger.error(f"Errore nel loop del Watchdog: {e}") logger.error(f"Errore nel ciclo di watchdog: {e}")
time.sleep(60)
time.sleep(10)
-11
View File
@@ -1,11 +0,0 @@
"""Invia una notifica push di prova a tutti gli iscritti, senza avviare MQTT e i thread dell'app.
Uso (dalla cartella del progetto, con il venv attivo): python test-push.py
"""
from core.database import init_db
from services.push_service import _send_push
init_db()
print("🚀 Starting notification test...")
_send_push("⚠️ ALLERTA FLOTTA", "Test Push Notification: System Online!", None)
print("✅ Command sent.")
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -e
echo "=== Aggiornamento Fleet Control Server (Redis Migration) ==="
# 1. Verifica e installazione del pacchetto di sistema Redis
if ! command -v redis-server &> /dev/null; then
echo "📦 Installazione del demone Redis via APT..."
sudo apt update
sudo apt install -y redis-server
else
echo "✅ redis-server già presente nel sistema."
fi
# 2. Avvio e abilitazione del servizio al boot
echo "🚀 Verifica stato servizio Redis..."
sudo systemctl enable --now redis-server
sudo systemctl restart redis-server
# Test rapido di risposta
if redis-cli ping | grep -q "PONG"; then
echo "✅ Redis è attivo e risponde (PONG)."
else
echo "❌ Errore: Redis non risponde sulla porta 6379."
exit 1
fi
# 3. Installazione della libreria Python nel Virtualenv
if [ -d "venv" ]; then
echo "🐍 Aggiornamento dipendenze nel virtualenv (incluso modulo redis)..."
venv/bin/pip install --upgrade pip
venv/bin/pip install -r requirements.txt
else
echo "⚠️ Cartella venv non trovata. Esegui 'pip install -r requirements.txt' nel tuo ambiente Python."
fi
# 4. Aggiornamento automatico di config.json
if [ -f "migrate.py" ]; then
echo "⚙️ Controllo e migrazione di config.json..."
python3 migrate.py
fi
echo ""
echo "🎉 Aggiornamento completato con successo!"
echo "Ora puoi riavviare il servizio: sudo systemctl restart fleet-control"