From 971d9edfe726ad6e3f9728812a1f4a02d78aed7c Mon Sep 17 00:00:00 2001 From: Roby Date: Fri, 16 Oct 2026 10:58:46 +0200 Subject: [PATCH 1/5] enanched security --- node_config_json.example | 72 ++ requirements.txt | 4 +- system_monitor.py | 1386 +++++++++++++++++++++++++++----------- 3 files changed, 1082 insertions(+), 380 deletions(-) create mode 100644 node_config_json.example diff --git a/node_config_json.example b/node_config_json.example new file mode 100644 index 0000000..b38a0b5 --- /dev/null +++ b/node_config_json.example @@ -0,0 +1,72 @@ +{ + "client_id": "repeater_id", + "mqtt": { + "broker": "127.0.0.1", + "port": 1883, + "user": "mmdvm", + "password": "password", + "base_topic": "servizi/repeater_id", + "keepalive": 15, + "tls": false + }, + "paths": { + "file_list": "/opt/file_list.txt", + "process_list": "/opt/process_list.txt", + "last_profile": "/opt/last_profile.txt", + "state_file": "/opt/agent_state.json" + }, + "settings": { + "auto_healing": true, + "update_interval": 60, + "usb_reset_id": "", + "gpio_reset_pin": null, + "gpio_reset_auto": true, + "verify_seconds": 8, + "max_offline_seconds": 300, + "exit_on_disconnect": false, + "heal_grace_seconds": 20, + "startup_grace_seconds": 60, + "mask_passwords": true + }, + "telegram": { + "enabled": false, + "token": "TOKEN ID", + "chat_id": "CHAT ID", + "quiet_hours": { + "start": 23, + "end": 7 + } + }, + "profiles": { + "A": { + "label": "Config BM", + "services": [ + { + "name": "mmdvmhost.service", + "source": "/etc/MMDVMHost_BM.ini", + "target": "/etc/MMDVMHost.ini" + }, + { + "name": "dmrgateway.service", + "source": "/etc/DMRGateway_BM.ini", + "target": "/etc/DMRGateway.ini" + } + ] + }, + "B": { + "label": "Config PC", + "services": [ + { + "name": "mmdvmhost.service", + "source": "/etc/MMDVMHost_PC.ini", + "target": "/etc/MMDVMHost.ini" + }, + { + "name": "dmrgateway.service", + "source": "/etc/DMRGateway_PC.ini", + "target": "/etc/DMRGateway.ini" + } + ] + } + } +} diff --git a/requirements.txt b/requirements.txt index da68d72..1e0b0ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ -paho-mqtt +paho-mqtt>=2.0 psutil requests +# Facoltativo, solo per il reset GPIO dell'HAT (Raspberry Pi) / Optional, only for the HAT GPIO reset: +# RPi.GPIO diff --git a/system_monitor.py b/system_monitor.py index 6a047fa..d1ee431 100644 --- a/system_monitor.py +++ b/system_monitor.py @@ -1,3 +1,10 @@ +#!/usr/bin/env python3 +""" +Fleet Control - Node Agent (v2.0) + +Compatibile con node_config.json esistenti: tutte le chiavi nuove sono OPZIONALI. +Vedi AGENT_CHANGELOG.md per l'elenco completo delle modifiche e dei nuovi parametri. +""" import paho.mqtt.client as mqtt import json import time @@ -8,16 +15,21 @@ import threading import sys import os import shutil +import re +import queue +import random +import signal import requests from pathlib import Path -import configparser import logging from logging.handlers import RotatingFileHandler +AGENT_VERSION = "2.0" + # --- CALCOLO PERCORSI DINAMICI --- AGENT_DIR = os.path.dirname(os.path.abspath(__file__)) LOG_FILE = os.path.join(AGENT_DIR, 'node_agent.log') -CONFIG_FILE = os.path.join(AGENT_DIR, 'node_config.json') +CONFIG_FILE = os.environ.get('FLEET_AGENT_CONFIG', os.path.join(AGENT_DIR, 'node_config.json')) # ========================================== # 0. LOGGING & HARDWARE CONFIGURATION @@ -39,134 +51,435 @@ try: except ImportError: GPIO_AVAILABLE = False logger.warning("RPi.GPIO library not found. Hardware reset disabled.") +except RuntimeError as e: + # La libreria c'è ma non funziona su questa scheda (l'import stesso solleva RuntimeError): non deve bloccare l'agent + GPIO_AVAILABLE = False + logger.warning(f"RPi.GPIO not usable on this board ({e}). GPIO hardware reset disabled.") # ========================================== -# 1. UNIFIED CONFIGURATION LOADING +# 1. CONFIGURATION LOADING (con validazione) # ========================================== CONFIG_PATH = Path(CONFIG_FILE) + def load_config(): try: if not CONFIG_PATH.exists(): logger.error(f"ERROR: File {CONFIG_PATH} not found!") sys.exit(1) with open(CONFIG_PATH, 'r') as f: - return json.load(f) + data = json.load(f) except Exception as e: logger.error(f"CRITICAL JSON ERROR: {e}") sys.exit(1) + if not isinstance(data, dict): + logger.error("CRITICAL: node_config.json must contain a JSON object") + sys.exit(1) + + missing = [] + if not str(data.get('client_id', '')).strip(): + missing.append('client_id') + mqtt_cfg = data.get('mqtt') if isinstance(data.get('mqtt'), dict) else {} + for k in ('broker', 'port', 'user', 'password'): + if k not in mqtt_cfg: + missing.append(f'mqtt.{k}') + if missing: + logger.error(f"CRITICAL: missing required settings in node_config.json: {', '.join(missing)}") + sys.exit(1) + + # sezioni opzionali: sempre presenti come dict (niente KeyError in giro per il codice) + for section in ('paths', 'settings', 'telegram', 'profiles'): + if not isinstance(data.get(section), dict): + data[section] = {} + return data + + cfg = load_config() # Identifiers and Topics -CLIENT_ID = cfg.get('client_id', 'iv3jdv').lower() -BASE_TOPIC = cfg.get('mqtt', {}).get('base_topic', f"servizi/{CLIENT_ID}") +CLIENT_ID = str(cfg['client_id']).strip().lower() +MQTT_CFG = cfg['mqtt'] +BASE_TOPIC = MQTT_CFG.get('base_topic', f"servizi/{CLIENT_ID}") +if BASE_TOPIC != f"servizi/{CLIENT_ID}": + logger.warning(f"mqtt.base_topic='{BASE_TOPIC}' diverso da 'servizi/{CLIENT_ID}': " + f"il server di controllo usa SEMPRE servizi/ e non ti vedrebbe.") TOPIC_CMD = f"{BASE_TOPIC}/cmnd" TOPIC_STAT = f"{BASE_TOPIC}/stat" +TOPIC_REQUEST = "devices/control/request" +TOPIC_CONTROL = f"devices/{CLIENT_ID}/control" +TOPIC_CONFIG_SET = f"devices/{CLIENT_ID}/config_set/" +TOPIC_LOGS = f"devices/{CLIENT_ID}/logs" +TOPIC_SERVICES = f"devices/{CLIENT_ID}/services" + +# Percorsi (tutti configurabili in "paths") +FILE_LIST_PATH = cfg['paths'].get('file_list') or '/opt/file_list.txt' +PROCESS_LIST_PATH = cfg['paths'].get('process_list') or '/opt/process_list.txt' +LAST_PROFILE_PATH = cfg['paths'].get('last_profile') or '/opt/last_profile.txt' +STATE_PATH = cfg['paths'].get('state_file') or '/opt/agent_state.json' # Global Status Variables -boot_recovered = False +mqtt_client = None current_status = "ONLINE" auto_healing_counter = {} +stopped_by_admin = set() # servizi fermati volontariamente dalla dashboard (l'auto-healing li ignora) +ops_lock = threading.RLock() # una sola operazione "pesante" alla volta (profilo, config, reset, update...) +heal_pause_until = 0.0 +disconnected_since = None +published_sig = {} # percorso file -> (mtime_ns, size) dell'ultima versione pubblicata +_warned_units = set() + +cmd_queue = queue.Queue(maxsize=20) +tg_queue = queue.Queue(maxsize=50) + +UNIT_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.@\-]{0,63}$') +USB_ID_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9:/_.\- ]{0,63}$') +SLUG_RE = re.compile(r'^[A-Za-z0-9_.\-]{1,64}$') +SENSITIVE_KEY_RE = re.compile(r'^(.*password|passwd|pass|secret|.*token|authkey|apikey)$', re.I) +MAX_CONFIG_BYTES = 65536 + + +def int_setting(name, default, minimum=0): + try: + value = int(cfg['settings'].get(name, default)) + except (TypeError, ValueError): + value = default + return max(minimum, value) + + +def get_update_interval(): + return int_setting('update_interval', 30, 10) + # ========================================== -# 2. TELEGRAM NOTIFICATION FUNCTION +# 2. HELPERS: SICUREZZA, FILE, COMANDI # ========================================== -def send_telegram_message(message): - t_cfg = cfg.get('telegram', {}) - if not t_cfg.get('enabled', False): return - +def redact(text): + """Toglie dai testi (log, messaggi) token Telegram e credenziali negli URL.""" + text = str(text) + token = str(cfg['telegram'].get('token') or '') + if token and token != "TOKEN ID": + text = text.replace(token, '***') + text = re.sub(r'bot\d+:[A-Za-z0-9_\-]+', 'bot***', text) + text = re.sub(r'(https?://)[^/@\s]+@', r'\1***@', text) + return text + + +def safe_write_bytes(path, data): + """Scrittura atomica (file temporaneo + fsync + replace) mantenendo permessi e proprietario. + Se la directory non è scrivibile (es. /etc con file concesso all'utente) ripiega sulla scrittura in-place.""" + path = str(path) + tmp = f"{path}.tmp.{os.getpid()}" + try: + st = None + try: + st = os.stat(path) + except FileNotFoundError: + pass + with open(tmp, 'wb') as f: + f.write(data) + f.flush() + os.fsync(f.fileno()) + if st is not None: + os.chmod(tmp, st.st_mode & 0o777) + try: + os.chown(tmp, st.st_uid, st.st_gid) + except (PermissionError, OSError): + pass + os.replace(tmp, path) + return + except OSError as e: + try: + os.unlink(tmp) + except OSError: + pass + logger.warning(f"Scrittura atomica di {path} non possibile ({e}): scrivo in-place.") + with open(path, 'wb') as f: + f.write(data) + f.flush() + os.fsync(f.fileno()) + + +def safe_write_text(path, text): + safe_write_bytes(path, text.encode('utf-8')) + + +def run_cmd(args, timeout=30, cwd=None): + """Esegue un comando SENZA shell e con timeout. rc=124 timeout, rc=127 comando mancante.""" + try: + return subprocess.run(args, capture_output=True, text=True, timeout=timeout, cwd=cwd) + except subprocess.TimeoutExpired: + logger.error(f"Timeout ({timeout}s) eseguendo: {' '.join(map(str, args))}") + return subprocess.CompletedProcess(args, 124, '', 'timeout') + except FileNotFoundError: + return subprocess.CompletedProcess(args, 127, '', 'command not found') + except Exception as e: + return subprocess.CompletedProcess(args, 1, '', str(e)) + + +def read_lines(path): + """Righe non vuote e non commentate di un file di testo ([] se manca).""" + try: + p = Path(str(path)) + if not str(path) or not p.is_file(): + return [] + return [l.strip() for l in p.read_text(encoding='utf-8', errors='ignore').splitlines() + if l.strip() and not l.strip().startswith('#')] + except Exception as e: + logger.error(f"Errore lettura {path}: {e}") + return [] + + +def connected(): + return mqtt_client is not None and mqtt_client.is_connected() + + +def log_to_broker(msg): + try: + if mqtt_client is not None: + mqtt_client.publish(TOPIC_LOGS, msg) + except Exception as e: + logger.error(f"Publish log error: {e}") + + +def publish_status(): + """Stato del nodo (retained). Va ripubblicato a ogni connessione: dopo un blip il LWT l'ha messo OFFLINE.""" + try: + if mqtt_client is not None: + mqtt_client.publish(TOPIC_STAT, current_status, qos=1, retain=True) + except Exception as e: + logger.error(f"Publish status error: {e}") + + +def note_ops_done(): + """Dopo un'operazione l'auto-healing resta in pausa qualche secondo (i servizi stanno ripartendo).""" + global heal_pause_until + heal_pause_until = time.time() + int_setting('heal_grace_seconds', 20, 0) + + +# ========================================== +# 3. TELEGRAM (invio in un thread dedicato) +# ========================================== +def in_quiet_hours(hour): + q = cfg['telegram'].get('quiet_hours', {"start": 23, "end": 7}) + if q is False or q is None or not isinstance(q, dict): + return False + try: + start, end = int(q.get('start', 23)), int(q.get('end', 7)) + except (TypeError, ValueError): + return False + if start == end: + return False + if start < end: + return start <= hour < end + return hour >= start or hour < end + + +def send_telegram_message(message, urgent=False): + t_cfg = cfg['telegram'] + if not t_cfg.get('enabled', False): + return + current_hour = int(time.strftime("%H")) - if current_hour >= 23 or current_hour < 7: - logger.info(f"🌙 Late night ({current_hour}:00): Telegram notification skipped.") + if not urgent and in_quiet_hours(current_hour): + logger.info(f"🌙 Quiet hours ({current_hour}:00): Telegram notification skipped.") return token = t_cfg.get('token') chat_id = t_cfg.get('chat_id') - if not token or not chat_id or token == "TOKEN ID": return + if not token or not chat_id or token == "TOKEN ID": + return + clean_msg = str(message).replace("", "").replace("", "") try: - clean_msg = message.replace("", "").replace("", "") - url = f"https://api.telegram.org/bot{token}/sendMessage" - payload = {"chat_id": chat_id, "text": f"[{CLIENT_ID.upper()}]\n{clean_msg}"} - requests.post(url, json=payload, timeout=10) - except Exception as e: - logger.error(f"Telegram sending error: {e}") + tg_queue.put_nowait((token, chat_id, f"[{CLIENT_ID.upper()}]\n{clean_msg}")) + except queue.Full: + logger.warning("Telegram queue full: message dropped") + + +def telegram_worker(): + while True: + token, chat_id, text = tg_queue.get() + try: + r = requests.post(f"https://api.telegram.org/bot{token}/sendMessage", + json={"chat_id": chat_id, "text": text}, timeout=10) + if not r.ok: + logger.error(f"Telegram HTTP {r.status_code}") + except Exception as e: + # l'eccezione di requests contiene l'URL con il token: va ripulita + logger.error(f"Telegram sending error: {type(e).__name__}: {redact(e)}") + # ========================================== -# 3. MULTIPLE PROFILE SWITCH LOGIC +# 4. STATO PERSISTENTE (profilo attivo, servizi fermati a mano) # ========================================== +def norm_name(name): + n = str(name).strip().lower() + return n[:-8] if n.endswith('.service') else n + + +def load_state(): + global stopped_by_admin + try: + with open(STATE_PATH, 'r') as f: + data = json.load(f) + stopped_by_admin = {norm_name(x) for x in data.get('stopped', [])} + except FileNotFoundError: + pass + except Exception as e: + logger.warning(f"Impossibile leggere {STATE_PATH}: {e}") + + +def save_state(): + try: + safe_write_text(STATE_PATH, json.dumps({"stopped": sorted(stopped_by_admin)})) + except Exception as e: + logger.warning(f"Impossibile salvare {STATE_PATH}: {e}") + + def get_actual_config_from_disk(): try: - path = "/opt/last_profile.txt" - if os.path.exists(path): - with open(path, "r") as f: + if os.path.exists(LAST_PROFILE_PATH): + with open(LAST_PROFILE_PATH, "r") as f: label = f.read().strip() if label: return f"ONLINE - {label}" except Exception as e: logger.error(f"Errore lettura memoria profilo: {e}") - - return "ONLINE" # Default se il file non esiste o è vuoto + return "ONLINE" # Default se il file non esiste o è vuoto -def switch_config(config_type): - profile = cfg.get('profiles', {}).get(config_type) - - if not profile: - return f"ERROR: Profile {config_type} not found in JSON" - - label = profile.get('label', f"Profile {config_type}") - services = profile.get('services', []) - - if not services: - return f"ERROR: No services configured for {config_type}" - + +def save_last_profile(label): try: - for s in services: - subprocess.run(["sudo", "systemctl", "stop", s['name']], check=False) - - for s in services: - if not os.path.exists(s['source']): - return f"ERROR: Missing source file {s['source']}" - shutil.copy(s['source'], s['target']) - - for s in services: - subprocess.run(["sudo", "systemctl", "start", s['name']], check=False) - - # Save the current profile to disk to remember it on reboot - with open("/opt/last_profile.txt", "w") as f: - f.write(label) - - send_telegram_message(f"✅ Multiple switch completed: {label}") - return f"ONLINE - {label}" - + safe_write_text(LAST_PROFILE_PATH, label) except Exception as e: - return f"ERROR: {str(e)}" + logger.error(f"Impossibile salvare il profilo attivo in {LAST_PROFILE_PATH}: {e}") -def force_online_if_needed(client): - global boot_recovered, current_status - if not boot_recovered: - logger.info("⚠️ Memory recovery skipped. Setting status from disk...") - current_status = get_actual_config_from_disk() - client.publish(TOPIC_STAT, current_status, retain=True) - boot_recovered = True # ========================================== -# 4. TELEMETRY AND AUTO-HEALING +# 5. SYSTEMD: NOMI UNITÀ (whitelist) E CONTROLLI +# ========================================== +def build_unit_map(): + """nome normalizzato -> nome unità. Solo ciò che il nodo conosce: process_list, profili, settings.service_map.""" + m = {} + for name in read_lines(PROCESS_LIST_PATH): + n = norm_name(name) + if UNIT_RE.match(n): + m[n] = n + for profile in cfg['profiles'].values(): + for s in (profile.get('services', []) if isinstance(profile, dict) else []): + name = str(s.get('name', '')) + if UNIT_RE.match(name): + m[norm_name(name)] = name + for k, v in (cfg['settings'].get('service_map') or {}).items(): + if UNIT_RE.match(str(v)): + m[norm_name(k)] = str(v) + return m + + +def resolve_unit(name): + return build_unit_map().get(norm_name(name)) + + +def systemctl(action, unit, timeout=60): + return run_cmd(["sudo", "systemctl", action, unit], timeout=timeout) + + +def unit_is_active(unit): + r = run_cmd(["systemctl", "is-active", unit], timeout=10) + return r.returncode == 0 and r.stdout.strip() == 'active' + + +def unit_exists(unit): + return run_cmd(["systemctl", "cat", unit], timeout=10).returncode == 0 + + +def unit_is_disabled(unit): + r = run_cmd(["systemctl", "is-enabled", unit], timeout=10) + return r.stdout.strip() in ('disabled', 'masked') + + +def wait_units_active(units, seconds): + """Attende fino a `seconds` che tutte le unità siano attive. Ritorna la lista di quelle NON attive.""" + deadline = time.monotonic() + seconds + pending = list(units) + while True: + pending = [u for u in pending if not unit_is_active(u)] + if not pending or time.monotonic() >= deadline: + return pending + time.sleep(1) + + +# ========================================== +# 6. RESET HARDWARE (USB / GPIO) +# ========================================== +def gpio_pin_config(): + pin = cfg['settings'].get('gpio_reset_pin') + if pin in (None, '', 0, False): + return None # nessun default implicito: il GPIO si usa solo se configurato + try: + return int(pin) + except (TypeError, ValueError): + return None + + +def hardware_reset(auto=False): + """Reset del modem. Ritorna {"attempted", "ok", "method", "msg"}.""" + usb_id = str(cfg['settings'].get('usb_reset_id') or '').strip() + if usb_id: + if not USB_ID_RE.match(usb_id): + return {"attempted": False, "ok": False, "method": "USB", "msg": f"⚠️ usb_reset_id non valido: {usb_id!r}"} + logger.info(f"Esecuzione reset USB per il dispositivo {usb_id}...") + r = run_cmd(["sudo", "usbreset", usb_id], timeout=15) + time.sleep(1.5) + if r.returncode == 0: + return {"attempted": True, "ok": True, "method": "USB", "msg": f"🔌 USB Reset ({usb_id}) inviato!"} + return {"attempted": True, "ok": False, "method": "USB", + "msg": f"⚠️ USB Reset ({usb_id}) fallito (rc={r.returncode}): {r.stderr.strip()[:120]}"} + + pin = gpio_pin_config() + if pin is not None and GPIO_AVAILABLE and (not auto or cfg['settings'].get('gpio_reset_auto', True)): + logger.info(f"Esecuzione reset GPIO (HAT) sul pin {pin}...") + try: + GPIO.setwarnings(False) + GPIO.setmode(GPIO.BCM) + GPIO.setup(pin, GPIO.OUT) + GPIO.output(pin, GPIO.LOW) + time.sleep(0.5) + GPIO.output(pin, GPIO.HIGH) + GPIO.cleanup(pin) + time.sleep(1.5) + return {"attempted": True, "ok": True, "method": "HAT", "msg": f"🔌 GPIO Pulse (Pin {pin}) inviato!"} + except Exception as e: + logger.error(f"Errore GPIO: {e}") + return {"attempted": True, "ok": False, "method": "HAT", "msg": f"⚠️ Errore GPIO (Pin {pin}): {e}"} + + return {"attempted": False, "ok": False, "method": "", + "msg": "⚠️ Nessun metodo di reset (USB o GPIO) configurato o disponibile sul nodo."} + + +# ========================================== +# 7. TELEMETRIA # ========================================== def get_cpu_temperature(): temp = 0.0 try: - temps = psutil.sensors_temperatures() - if 'cpu_thermal' in temps: temp = temps['cpu_thermal'][0].current - elif 'coretemp' in temps: temp = temps['coretemp'][0].current + temps = psutil.sensors_temperatures() if hasattr(psutil, 'sensors_temperatures') else {} + if temps.get('cpu_thermal'): + temp = temps['cpu_thermal'][0].current + elif temps.get('coretemp'): + temp = temps['coretemp'][0].current elif platform.system() == "Linux": - res = os.popen('vcgencmd measure_temp').readline() - if res: temp = float(res.replace("temp=","").replace("'C\n","")) - except: pass + r = run_cmd(["vcgencmd", "measure_temp"], timeout=5) + m = re.search(r'temp=([\d.]+)', r.stdout) if r.returncode == 0 else None + if m: + temp = float(m.group(1)) + except Exception as e: + logger.debug(f"Temperature read error: {e}") return round(temp, 1) + def get_system_status(): status = { "cpu_usage_percent": psutil.cpu_percent(interval=0.5), @@ -175,360 +488,675 @@ def get_system_status(): "disk_usage_percent": psutil.disk_usage('/').percent, "processes": {}, "timestamp": time.strftime("%H:%M:%S"), - "profiles": { k: v.get('label', f'PROFILE {k}') for k, v in cfg.get('profiles', {}).items() } + "profiles": {k: (v.get('label', f'PROFILE {k}') if isinstance(v, dict) else f'PROFILE {k}') + for k, v in cfg['profiles'].items()}, + "update_interval": get_update_interval(), # il server ne ricava le soglie "agent muto" + "agent_version": AGENT_VERSION, } - proc_path = Path(cfg['paths'].get('process_list', '')) - if proc_path.exists(): + names = read_lines(PROCESS_LIST_PATH) + if names: try: - target_processes = proc_path.read_text(encoding="utf-8").splitlines() - running_names = {p.info['name'].lower() for p in psutil.process_iter(['name'])} - for name in target_processes: - name = name.strip().lower() - if name: status["processes"][name] = "online" if name in running_names else "offline" - except Exception as e: logger.error(f"Process check error: {e}") + running = {(p.info.get('name') or '').lower() for p in psutil.process_iter(['name'])} + for name in names: + n = name.strip().lower() + if n: + status["processes"][n] = "online" if n in running else "offline" + except Exception as e: + logger.error(f"Process check error: {e}") return status -def check_auto_healing(client, status): - if not cfg['settings'].get('auto_healing', False): return - for proc_name, state in status["processes"].items(): - if state == "offline": - attempts = auto_healing_counter.get(proc_name, 0) - if attempts < 3: - auto_healing_counter[proc_name] = attempts + 1 - msg = f"🛠 Auto-healing: {proc_name} offline. Restarting {attempts+1}/3..." - client.publish(f"devices/{CLIENT_ID}/logs", msg) - send_telegram_message(msg) - - # --- INIZIO LOGICA RESET HARDWARE PER MMDVMHOST --- - if proc_name.lower() == "mmdvmhost": - usb_id = cfg.get('settings', {}).get('usb_reset_id', "").strip() - gpio_pin = cfg.get('settings', {}).get('gpio_reset_pin', 21) - - if usb_id: - logger.info(f"Esecuzione reset USB per il dispositivo {usb_id} prima del riavvio...") - try: - subprocess.run(["sudo", "usbreset", usb_id], check=False) - time.sleep(1.5) - client.publish(f"devices/{CLIENT_ID}/logs", f"🔌 USB Reset ({usb_id}) inviato!") - except Exception as e: - logger.error(f"Errore durante il reset USB in auto-healing: {e}") - - elif GPIO_AVAILABLE and gpio_pin: - logger.info(f"Esecuzione reset GPIO (HAT) sul pin {gpio_pin} prima del riavvio...") - try: - RESET_PIN = int(gpio_pin) - GPIO.setwarnings(False) - GPIO.setmode(GPIO.BCM) - GPIO.setup(RESET_PIN, GPIO.OUT) - GPIO.output(RESET_PIN, GPIO.LOW) - time.sleep(0.5) - GPIO.output(RESET_PIN, GPIO.HIGH) - GPIO.cleanup(RESET_PIN) - time.sleep(1.5) - client.publish(f"devices/{CLIENT_ID}/logs", f"🔌 GPIO Pulse (Pin {RESET_PIN}) inviato!") - except Exception as e: - logger.error(f"Errore GPIO in auto-healing: {e}") - # --- FINE LOGICA MODIFICA --- - subprocess.run(["sudo", "systemctl", "restart", proc_name]) - elif attempts == 3: - msg = f"🚨 CRITICAL: {proc_name} failed!" - client.publish(f"devices/{CLIENT_ID}/logs", msg) - send_telegram_message(msg) - auto_healing_counter[proc_name] = 4 - else: - auto_healing_counter[proc_name] = 0 - -def publish_all(client): +def publish_telemetry(): status = get_system_status() - file_list_path = Path(cfg['paths'].get('file_list', '')) - status["config_files"] = [] - status["files"] = [] - - if file_list_path.exists(): - try: - files = file_list_path.read_text(encoding="utf-8").splitlines() - extracted_names = [Path(f.strip()).stem for f in files if f.strip()] - status["config_files"] = extracted_names - status["files"] = extracted_names - except: pass - - client.publish(f"devices/{CLIENT_ID}/services", json.dumps(status), qos=1) - - if file_list_path.exists(): - try: - files = file_list_path.read_text(encoding="utf-8").splitlines() - for f in files: - p = Path(f.strip()) - if p.exists(): - client.publish(f"data/{CLIENT_ID}/{p.stem}/full_config", json.dumps({"raw_text": p.read_text(encoding="utf-8")}), qos=1, retain=True) - except: pass + stems = [Path(f).stem for f in read_lines(FILE_LIST_PATH)] + status["config_files"] = stems + status["files"] = stems + if connected(): + mqtt_client.publish(TOPIC_SERVICES, json.dumps(status), qos=1) return status -def publish_all_ini_files(client): - file_list_path = cfg.get('paths', {}).get('file_list', '/opt/file_list.txt') - if not os.path.exists(file_list_path): return +# ========================================== +# 8. PUBBLICAZIONE CONFIGURAZIONI (solo se cambiate) +# ========================================== +def parse_ini(text): + ini_data = {} + current_section = None + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith(('#', ';')): + continue + if line.startswith('[') and line.endswith(']'): + current_section = line[1:-1].strip() + ini_data[current_section] = {} + elif '=' in line and current_section is not None: + k, v = line.split('=', 1) + k, v = k.strip(), v.strip() + if k in ini_data[current_section]: + ini_data[current_section][k] = str(ini_data[current_section][k]) + "," + v + else: + ini_data[current_section][k] = v + return ini_data + + +def mask_sensitive(ini_data): + """Nasconde Password/Secret/Token nei topic per sezione (NON tocca PassAllTG/PassAllPC, che servono al server).""" + for payload in ini_data.values(): + for k, v in payload.items(): + if SENSITIVE_KEY_RE.match(k) and str(v).strip(): + payload[k] = "********" + return ini_data + + +def profile_only_sources(): + """File usati solo come sorgente di un profilo (es. *_BM.ini): per l'editor basta full_config.""" + sources, targets = set(), set() + for profile in cfg['profiles'].values(): + for s in (profile.get('services', []) if isinstance(profile, dict) else []): + if s.get('source'): + sources.add(s['source']) + if s.get('target'): + targets.add(s['target']) + return sources - targets + + +def publish_configs(force=False): + """Pubblica full_config e sezioni INI (retained) SOLO per i file cambiati dall'ultima volta.""" + if not connected(): + return + skip_sections = profile_only_sources() + mask = bool(cfg['settings'].get('mask_passwords', True)) + for fp in read_lines(FILE_LIST_PATH): + try: + if not os.path.isfile(fp): + continue + st = os.stat(fp) + sig = (st.st_mtime_ns, st.st_size) + if not force and published_sig.get(fp) == sig: + continue + text = Path(fp).read_text(encoding='utf-8', errors='replace') + stem = Path(fp).stem + mqtt_client.publish(f"data/{CLIENT_ID}/{stem}/full_config", + json.dumps({"raw_text": text}), qos=1, retain=True) + if fp not in skip_sections: + ini_data = parse_ini(text) + if mask: + mask_sensitive(ini_data) + for section, payload in ini_data.items(): + part = re.sub(r'[/+#]', '_', section) + if part: + mqtt_client.publish(f"data/{CLIENT_ID}/{stem}/{part}", json.dumps(payload), qos=1, retain=True) + published_sig[fp] = sig + except Exception as e: + logger.error(f"Error publishing config {fp}: {e}") + + +# ========================================== +# 9. CAMBIO PROFILO +# ========================================== +def find_profile(key): + k = str(key).strip().lower() + for pkey, profile in cfg['profiles'].items(): + if str(pkey).lower() == k and isinstance(profile, dict): + return pkey, profile + return None, None + + +def switch_config(key): + """Ritorna (ok, label_o_errore). Non tocca nessun servizio se i controlli preliminari falliscono; + se qualcosa va storto ripristina la configurazione precedente e riavvia i servizi.""" + pkey, profile = find_profile(key) + if not profile: + return False, f"Profile {key} not found in JSON" + label = profile.get('label', f"Profile {pkey}") + services = profile.get('services', []) + if not services: + return False, f"No services configured for {pkey}" + + # --- CONTROLLI PRELIMINARI (prima di fermare qualsiasi cosa) --- + for s in services: + name, src, dst = s.get('name'), s.get('source'), s.get('target') + if not (name and src and dst): + return False, f"Invalid service entry in profile {pkey}" + if not UNIT_RE.match(str(name)): + return False, f"Invalid unit name {name!r}" + if not os.path.isfile(src): + return False, f"Missing source file {src}" + if not os.path.isdir(os.path.dirname(str(dst)) or '.'): + return False, f"Target directory missing for {dst}" + + verify = int_setting('verify_seconds', 8, 0) + with ops_lock: + previous = {} + for s in services: + dst = s['target'] + try: + previous[dst] = Path(dst).read_bytes() if os.path.exists(dst) else None + except Exception as e: + return False, f"Cannot read target {dst}: {e}" + + error = None + try: + for s in services: + systemctl("stop", s['name']) + for s in services: + safe_write_bytes(s['target'], Path(s['source']).read_bytes()) + except Exception as e: + error = f"Copy failed: {e}" + + if error is None: + for s in services: + systemctl("start", s['name']) + if verify > 0: + failed = wait_units_active([s['name'] for s in services], verify) + if failed: + error = f"Units not active after switch: {', '.join(failed)}" + + if error: + logger.error(f"Cambio profilo {pkey} fallito ({error}): ripristino la configurazione precedente.") + for s in services: + systemctl("stop", s['name']) + for dst, data in previous.items(): + if data is not None: + try: + safe_write_bytes(dst, data) + except Exception as e: + logger.error(f"Rollback di {dst} fallito: {e}") + for s in services: + systemctl("start", s['name']) + note_ops_done() + return False, f"{error} - rolled back to previous configuration" + + for s in services: + stopped_by_admin.discard(norm_name(s['name'])) + save_state() + save_last_profile(label) + note_ops_done() + return True, label + + +# ========================================== +# 10. SCRITTURA CONFIG DA REMOTO +# ========================================== +def service_for_file(path): + """Servizio da riavviare dopo la modifica di un file di configurazione (None = nessuno).""" + stem = Path(path).stem + unit = resolve_unit(stem) + if unit: + return unit + guess = stem.lower() + if str(path) not in profile_only_sources() and UNIT_RE.match(guess) and unit_exists(guess): + return guess + return None + + +def apply_config_set(slug, payload): try: - with open(file_list_path, 'r') as f: - files_to_parse = [line.strip() for line in f if line.strip()] + data = json.loads(payload) except Exception as e: - logger.error(f"Error reading {file_list_path}: {e}") + logger.error(f"Config {slug}: JSON non valido ({e})") + log_to_broker(f"❌ Config {slug}: invalid JSON") + return + raw = data.get('raw_text') if isinstance(data, dict) else None + if not isinstance(raw, str) or not raw.strip(): + logger.error(f"Config {slug}: raw_text mancante o vuoto: rifiutato") + log_to_broker(f"❌ Config {slug}: empty or missing raw_text, refused") + return + raw_bytes = raw.encode('utf-8') + if len(raw_bytes) > MAX_CONFIG_BYTES: + logger.error(f"Config {slug}: troppo grande ({len(raw_bytes)} byte)") + log_to_broker(f"❌ Config {slug}: too large") return - for file_path in files_to_parse: - if not os.path.exists(file_path): continue - try: - base_name = os.path.splitext(os.path.basename(file_path))[0] - ini_data = {} - current_section = None - - with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: - for line in f: - line = line.strip() - if not line or line.startswith(('#', ';')): continue - if line.startswith('[') and line.endswith(']'): - current_section = line[1:-1].strip() - ini_data[current_section] = {} - elif '=' in line and current_section is not None: - k, v = line.split('=', 1) - k, v = k.strip(), v.strip() - if k in ini_data[current_section]: - ini_data[current_section][k] = str(ini_data[current_section][k]) + "," + v - else: - ini_data[current_section][k] = v - - for section, payload in ini_data.items(): - topic = f"data/{CLIENT_ID}/{base_name}/{section}" - client.publish(topic, json.dumps(payload), retain=True) - except Exception as e: - logger.error(f"Error parsing INI for {file_path}: {e}") + target = None + for line in read_lines(FILE_LIST_PATH): + if Path(line).stem.lower() == slug.lower(): + target = Path(line) + break + if target is None: + logger.error(f"Config {slug}: file non presente in file_list") + log_to_broker(f"❌ Config {slug}: not in file list") + return -def write_config_from_json(slug, json_payload): - file_list_path = Path(cfg['paths'].get('file_list', '')) - if not file_list_path.exists(): return + verify = int_setting('verify_seconds', 8, 0) + with ops_lock: + previous = target.read_bytes() if target.exists() else None + if previous is not None: + try: + shutil.copy2(target, str(target) + ".bak") + except Exception as e: + logger.warning(f"Backup .bak non creato per {target}: {e}") + safe_write_bytes(target, raw_bytes) + + unit = service_for_file(target) + if unit: + systemctl("restart", unit) + if verify > 0: + failed = wait_units_active([unit], verify) + if failed: + logger.error(f"Config {slug}: {unit} non riparte con la nuova configurazione: ripristino.") + if previous is not None: + safe_write_bytes(target, previous) + systemctl("restart", unit) + log_to_broker(f"❌ Config {slug}: service did not start, previous configuration restored") + send_telegram_message(f"❌ Config {slug.upper()} rejected: service did not start (restored).") + note_ops_done() + return + else: + logger.info(f"Config {slug} salvata: nessun servizio da riavviare.") + note_ops_done() + + send_telegram_message(f"📝 Config {slug.upper()} updated via Web.") + logger.info(f"Configuration {slug} updated successfully.") + + +# ========================================== +# 11. AUTO-HEALING +# ========================================== +def check_auto_healing(status): + if not cfg['settings'].get('auto_healing', False): + return + if time.time() < heal_pause_until: + return + if not ops_lock.acquire(blocking=False): # c'è un'operazione in corso: ci riprovo al prossimo giro + return try: - files = file_list_path.read_text(encoding="utf-8").splitlines() - for f in files: - p = Path(f.strip()) - if p.stem.lower() == slug.lower(): - new_data = json.loads(json_payload) - shutil.copy(p, str(p) + ".bak") - with open(p, 'w', encoding="utf-8") as file: file.write(new_data.get("raw_text", "")) - os.system(f"sudo systemctl restart {slug}") - send_telegram_message(f"📝 Config {slug.upper()} updated via Web.") - logger.info(f"Configuration {slug} updated successfully.") - break - except Exception as e: logger.error(f"Config writing error: {e}") + _heal(status) + finally: + ops_lock.release() + + +def _heal(status): + state_changed = False + for proc_name, state in status["processes"].items(): + norm = norm_name(proc_name) + + if state == "online": + auto_healing_counter[proc_name] = 0 + if norm in stopped_by_admin: + stopped_by_admin.discard(norm) + state_changed = True + continue + + if norm in stopped_by_admin: + continue # fermato volontariamente dalla dashboard + unit = resolve_unit(norm) + if not unit or not unit_exists(unit) or unit_is_disabled(unit): + if norm not in _warned_units: + _warned_units.add(norm) + logger.info(f"Auto-healing: '{proc_name}' ignorato (unità assente o disabilitata su questo nodo).") + continue + + attempts = auto_healing_counter.get(proc_name, 0) + if attempts < 3: + auto_healing_counter[proc_name] = attempts + 1 + msg = f"🛠 Auto-healing: {proc_name} offline. Restarting {attempts + 1}/3..." + log_to_broker(msg) + send_telegram_message(msg) + + if norm == "mmdvmhost": + res = hardware_reset(auto=True) + if res["attempted"]: + log_to_broker(res["msg"]) + if not res["ok"]: + logger.warning(res["msg"]) + + systemctl("restart", unit) + elif attempts == 3: + msg = f"🚨 CRITICAL: {proc_name} failed!" + log_to_broker(msg) + send_telegram_message(msg, urgent=True) + auto_healing_counter[proc_name] = 4 + + if state_changed: + save_state() + # ========================================== -# 5. MQTT CALLBACKS +# 12. COMANDI (eseguiti da un thread dedicato, mai nella callback MQTT) # ========================================== +def enqueue(name, fn): + try: + cmd_queue.put_nowait((name, fn)) + return True + except queue.Full: + logger.error(f"Coda comandi piena: '{name}' scartato") + log_to_broker(f"❌ Command queue full: {name} discarded") + return False + + +def command_worker(): + while True: + name, fn = cmd_queue.get() + try: + fn() + except Exception as e: + logger.exception(f"Errore eseguendo il comando '{name}'") + log_to_broker(f"❌ ERROR ({name}): {redact(e)}"[:300]) + publish_status() + + +def do_switch_profile(key): + global current_status + ok, text = switch_config(key) + if ok: + current_status = f"ONLINE - {text}" + send_telegram_message(f"✅ Multiple switch completed: {text}") + else: + logger.error(f"Cambio profilo fallito: {text}") + log_to_broker(f"❌ {text}") + send_telegram_message(f"❌ Profile switch failed: {text}") + publish_status() # anche in caso di errore: il server esce dallo stato "Sent..." + publish_telemetry() + publish_configs() + + +def do_update_agent(): + log_to_broker("🔄 Avvio aggiornamento OTA (git pull)...") + logger.info("Ricevuto comando di UPDATE_AGENT. Esecuzione git pull...") + with ops_lock: + head = run_cmd(["git", "rev-parse", "HEAD"], timeout=15, cwd=AGENT_DIR) + old = head.stdout.strip() if head.returncode == 0 else None + + pull = run_cmd(["git", "pull", "--ff-only"], timeout=120, cwd=AGENT_DIR) + if pull.returncode != 0: + err = redact(pull.stderr or pull.stdout)[:500] + log_to_broker(f"❌ Errore aggiornamento:\n{err}") + logger.error(f"Errore git pull:\n{err}") + publish_status() + return + + head = run_cmd(["git", "rev-parse", "HEAD"], timeout=15, cwd=AGENT_DIR) + new = head.stdout.strip() if head.returncode == 0 else None + if old and new and old == new: + log_to_broker("✅ Già aggiornato (nessuna modifica).") + publish_status() + return + + # La nuova versione deve almeno essere sintatticamente valida, altrimenti il nodo resterebbe irraggiungibile + check = run_cmd([sys.executable, "-c", + "import ast,sys; ast.parse(open(sys.argv[1], encoding='utf-8').read())", + os.path.abspath(__file__)], timeout=30) + if check.returncode != 0: + if old: + run_cmd(["git", "reset", "--hard", old], timeout=30, cwd=AGENT_DIR) + msg = redact(check.stderr)[:300] + log_to_broker(f"❌ Nuova versione non valida: ripristinata la precedente.\n{msg}") + logger.error(f"Nuova versione non valida, rollback a {old}: {msg}") + publish_status() + return + + log_to_broker("✅ Git pull completato. Riavvio in corso...") + logger.info(f"Aggiornamento completato:\n{pull.stdout}") + time.sleep(1) + # Systemd (Restart=always) lo fa ripartire con il nuovo codice + os._exit(1) + + +def do_reboot(): + mqtt_client.publish(TOPIC_STAT, f"OFFLINE - Rebooting {CLIENT_ID.upper()}...", qos=1, retain=True) + logger.info("REBOOT command received. Rebooting system...") + time.sleep(1) + r = run_cmd(["sudo", "reboot"], timeout=30) + if r.returncode != 0: + logger.error(f"Reboot fallito (rc={r.returncode}): {r.stderr.strip()[:200]}") + log_to_broker("❌ Reboot failed") + publish_status() + + +def do_reset_hat(): + with ops_lock: + res = hardware_reset(auto=False) + if not res["attempted"]: + logger.warning(res["msg"]) + log_to_broker(res["msg"]) + publish_status() + return + logger.info("Restarting MMDVMHost...") + unit = resolve_unit("mmdvmhost") or "mmdvmhost" + systemctl("restart", unit) + if res["ok"]: + mqtt_client.publish(f"fleet/{CLIENT_ID}/status", f"{res['method']} RESET + MMDVM RESTART OK") + log_to_broker(f"{res['msg']} + MMDVMHost Restarted") + else: + mqtt_client.publish(f"fleet/{CLIENT_ID}/status", f"RESET ERROR: {res['msg']}") + log_to_broker(f"{res['msg']} (MMDVMHost restarted anyway)") + note_ops_done() + publish_status() + + +def do_toggle_telegram(new_state): + try: + # si parte dal file su disco (non dalla copia in memoria) per non perdere modifiche fatte a mano + with open(CONFIG_PATH, 'r') as f: + disk = json.load(f) + if not isinstance(disk.get('telegram'), dict): + disk['telegram'] = {} + disk['telegram']['enabled'] = new_state + safe_write_text(CONFIG_PATH, json.dumps(disk, indent=4)) + cfg['telegram']['enabled'] = new_state + log_to_broker(f"{'🔔' if new_state else '🔇'} Notifications {'ON' if new_state else 'OFF'}") + if new_state: + send_telegram_message("Notifications reactivated!") + except Exception as e: + logger.error(f"Error saving Telegram status: {e}") + publish_status() + + +def do_service_control(action, service): + unit = resolve_unit(service) + if not unit: + logger.warning(f"Servizio non consentito: {service[:40]!r}") + log_to_broker(f"❌ Service not allowed: {service[:40]}") + publish_status() + return + with ops_lock: + r = systemctl(action, unit) + if r.returncode == 0: + n = norm_name(service) + if action == "stop": + stopped_by_admin.add(n) # l'auto-healing non lo rialzerà + else: + stopped_by_admin.discard(n) + save_state() + log_to_broker(f"✅ {action.upper()}: {service}") + logger.info(f"Service command executed: {action.upper()} {service}") + else: + err = r.stderr.strip()[:200] + log_to_broker(f"❌ ERROR: {action} {service}: {err}") + logger.error(f"Error executing service command {action} {service}: {err}") + note_ops_done() + publish_telemetry() + publish_status() + + +def do_global_request(): + time.sleep(random.uniform(0, 5)) # evita che tutta la flotta risponda nello stesso istante + publish_telemetry() + publish_configs(force=True) + publish_status() + + +def initial_publish(): + publish_telemetry() + publish_configs(force=True) + + +# ========================================== +# 13. MQTT CALLBACKS (leggere: validano e mettono in coda) +# ========================================== +def handle_command(payload): + cmd = payload.upper() + pkey, profile = find_profile(payload) + if profile: + enqueue(f"profile {pkey}", lambda: do_switch_profile(pkey)) + elif cmd == "UPDATE_AGENT": + enqueue("update_agent", do_update_agent) + elif cmd == "REBOOT": + enqueue("reboot", do_reboot) + elif cmd == "RESET_HAT": + enqueue("reset_hat", do_reset_hat) + elif cmd in ("TG:OFF", "TG:ON"): + enqueue("toggle_telegram", lambda: do_toggle_telegram(cmd == "TG:ON")) + else: + logger.warning(f"Comando sconosciuto ignorato: {payload[:40]!r}") + log_to_broker(f"❓ Unknown command: {payload[:40]}") + publish_status() + + +def handle_control(payload): + action, _, service = payload.partition(":") + action, service = action.strip().lower(), service.strip() + if action not in ("restart", "stop", "start") or not service: + logger.warning(f"Comando di controllo non valido: {payload[:60]!r}") + log_to_broker(f"❌ Invalid control command: {payload[:60]}") + return + enqueue(f"{action} {service}", lambda: do_service_control(action, service)) + + def on_connect(client, userdata, flags, reason_code, properties=None): if reason_code == 0: logger.info(f"✅ Connected to MQTT broker: {CLIENT_ID.upper()}") - client.subscribe([(TOPIC_CMD, 0), (TOPIC_STAT, 0)]) client.subscribe([ - ("devices/control/request", 0), - (f"devices/{CLIENT_ID}/control", 0), - (f"devices/{CLIENT_ID}/config_set/#", 0) + (TOPIC_CMD, 0), + (TOPIC_REQUEST, 0), + (TOPIC_CONTROL, 0), + (TOPIC_CONFIG_SET + "#", 0) ]) - threading.Timer(5.0, force_online_if_needed, [client]).start() - publish_all(client) - publish_all_ini_files(client) + publish_status() # a ogni connessione: dopo un blip il LWT ha messo il nodo OFFLINE + enqueue("initial publish", initial_publish) else: logger.error(f"❌ MQTT connection error. Code: {reason_code}") + +def on_connect_fail(client, userdata): + logger.warning("MQTT: connessione al broker non riuscita, riprovo...") + + def on_disconnect(client, userdata, disconnect_flags, reason_code, properties=None): - logger.error(f"⚠️ Disconnessione rilevata a basso livello (Codice: {reason_code}). Forzo il riavvio tramite systemd...") - os._exit(1) + logger.warning(f"⚠️ Disconnesso dal broker MQTT (codice: {reason_code}).") + if cfg['settings'].get('exit_on_disconnect', False): + logger.error("exit_on_disconnect attivo: riavvio tramite systemd...") + os._exit(1) + def on_message(client, userdata, msg): - global boot_recovered, current_status, cfg - payload = msg.payload.decode().strip() - topic = msg.topic - - if topic == TOPIC_STAT and not boot_recovered: - if not any(x in payload.upper() for x in ["OFFLINE", "ERROR", "REBOOT"]): - current_status = payload - boot_recovered = True - client.publish(TOPIC_STAT, current_status, retain=True) + try: + topic = msg.topic + payload = msg.payload.decode('utf-8', errors='replace').strip() - elif topic == TOPIC_CMD: - cmd = payload.upper() - if cmd in cfg.get('profiles', {}).keys(): - current_status = switch_config(cmd) - client.publish(TOPIC_STAT, current_status, retain=True) - boot_recovered = True - publish_all(client) - - elif cmd == "UPDATE_AGENT": - client.publish(f"devices/{CLIENT_ID}/logs", "🔄 Avvio aggiornamento OTA (git pull)...") - logger.info("Ricevuto comando di UPDATE_AGENT. Esecuzione git pull...") - try: - # Esegue il comando git pull nella directory dell'agente - result = subprocess.run( - ["git", "pull"], - cwd=AGENT_DIR, - capture_output=True, text=True, check=True - ) - client.publish(f"devices/{CLIENT_ID}/logs", f"✅ Git pull completato. Riavvio in corso...") - logger.info(f"Aggiornamento completato:\n{result.stdout}") - time.sleep(1) - - # Uccide il processo. Systemd lo farà ripartire istantaneamente con il nuovo codice! - os._exit(1) - - except subprocess.CalledProcessError as e: - client.publish(f"devices/{CLIENT_ID}/logs", f"❌ Errore aggiornamento:\n{e.stderr}") - logger.error(f"Errore git pull:\n{e.stderr}") - - elif cmd == "REBOOT": - client.publish(TOPIC_STAT, f"OFFLINE - Rebooting {CLIENT_ID.upper()}...", retain=False) - logger.info("REBOOT command received. Rebooting system...") - time.sleep(1) - subprocess.run(["sudo", "reboot"], check=True) - - elif cmd == 'RESET_HAT': - usb_id = cfg.get('settings', {}).get('usb_reset_id', "").strip() - gpio_pin = cfg.get('settings', {}).get('gpio_reset_pin', 21) - - if usb_id: - try: - logger.info(f"RESET USB manuale inviato al dispositivo {usb_id}") - subprocess.run(["sudo", "usbreset", usb_id], check=False) - time.sleep(1.5) - logger.info("Restarting MMDVMHost...") - subprocess.run(["sudo", "systemctl", "restart", "mmdvmhost"], check=False) - client.publish(f"fleet/{CLIENT_ID}/status", "USB RESET + MMDVM RESTART OK") - client.publish(f"devices/{CLIENT_ID}/logs", f"🔌 USB Reset ({usb_id}) + MMDVMHost Restarted") - except Exception as e: - logger.error(f"Errore durante il reset USB/MMDVMHost: {e}") - client.publish(f"fleet/{CLIENT_ID}/status", f"RESET ERROR: {e}") - - elif GPIO_AVAILABLE and gpio_pin: - try: - RESET_PIN = int(gpio_pin) - GPIO.setwarnings(False) - GPIO.setmode(GPIO.BCM) - GPIO.setup(RESET_PIN, GPIO.OUT) - GPIO.output(RESET_PIN, GPIO.LOW) - time.sleep(0.5) - GPIO.output(RESET_PIN, GPIO.HIGH) - GPIO.cleanup(RESET_PIN) - logger.info(f"Pulsazione di RESET inviata al GPIO {RESET_PIN}") - time.sleep(1.5) - logger.info("Restarting MMDVMHost...") - subprocess.run(["sudo", "systemctl", "restart", "mmdvmhost"], check=False) - client.publish(f"fleet/{CLIENT_ID}/status", "HAT RESET + MMDVM RESTART OK") - client.publish(f"devices/{CLIENT_ID}/logs", f"🔌 GPIO Reset (Pin {RESET_PIN}) + MMDVMHost Restarted") - except Exception as e: - logger.error(f"Errore durante il reset GPIO/MMDVMHost: {e}") - client.publish(f"fleet/{CLIENT_ID}/status", f"RESET ERROR: {e}") + if getattr(msg, 'retain', False): + logger.warning(f"Messaggio retained ignorato su {topic} (i comandi non vanno mai pubblicati retained)") + return + + if topic == TOPIC_CMD: + handle_command(payload) + elif topic == TOPIC_REQUEST: + if payload.lower() in ("status", "update"): + logger.info("📥 Received global update command (REQ CONFIG)") + enqueue("global request", do_global_request) + elif topic == TOPIC_CONTROL: + handle_control(payload) + elif topic.startswith(TOPIC_CONFIG_SET): + slug = topic[len(TOPIC_CONFIG_SET):] + if SLUG_RE.match(slug): + enqueue(f"config {slug}", lambda: apply_config_set(slug, payload)) else: - msg = "⚠️ Nessun metodo di reset (USB o GPIO) configurato o disponibile sul nodo." - logger.warning(msg) - client.publish(f"devices/{CLIENT_ID}/logs", msg) + logger.warning(f"config_set con nome non valido: {slug[:40]!r}") + except Exception: + logger.exception("Errore in on_message") - elif cmd in ["TG:OFF", "TG:ON"]: - new_state = (cmd == "TG:ON") - cfg['telegram']['enabled'] = new_state - try: - with open(CONFIG_PATH, 'w') as f: json.dump(cfg, f, indent=4) - client.publish(f"devices/{CLIENT_ID}/logs", f"{'🔔' if new_state else '🔇'} Notifications {'ON' if new_state else 'OFF'}") - if new_state: send_telegram_message("Notifications reactivated!") - except Exception as e: logger.error(f"Error saving Telegram status: {e}") - elif topic == "devices/control/request" and payload.lower() in ["status", "update"]: - logger.info("📥 Received global update command (REQ CONFIG)") - publish_all(client) - publish_all_ini_files(client) - client.publish(TOPIC_STAT, current_status, retain=True) - - elif topic == f"devices/{CLIENT_ID}/control": - if ":" in payload: - action, service = payload.split(":") - if action.lower() in ["restart", "stop", "start"]: - try: - subprocess.run(["sudo", "systemctl", action.lower(), service.lower()], check=True) - client.publish(f"devices/{CLIENT_ID}/logs", f"✅ {action.upper()}: {service}") - logger.info(f"Service command executed: {action.upper()} {service}") - publish_all(client) - except Exception as e: - client.publish(f"devices/{CLIENT_ID}/logs", f"❌ ERROR: {str(e)}") - logger.error(f"Error executing service command: {e}") - - elif topic.startswith(f"devices/{CLIENT_ID}/config_set/"): - slug = topic.split("/")[-1] - write_config_from_json(slug, payload) - time.sleep(1) - publish_all(client) - publish_all_ini_files(client) - -def auto_publish_task(client): - disconnect_counter = 0 - +# ========================================== +# 14. THREAD DI SERVIZIO: TELEMETRIA E WATCHDOG +# ========================================== +def telemetry_loop(): while True: try: - # Forza la conversione a intero per evitare crash silenziosi su time.sleep - update_interval = int(cfg['settings'].get('update_interval', 30)) - - if not client.is_connected(): - disconnect_counter += 1 - logger.warning(f"⚠️ Broker MQTT irraggiungibile. Watchdog {disconnect_counter}/5...") - - if disconnect_counter >= 5: - logger.error("🚨 Connessione persa definitivamente. Hard Kill del processo...") - os._exit(1) + if connected(): + status = publish_telemetry() + publish_configs() else: - if disconnect_counter > 0: - logger.info("✅ Connessione MQTT ripristinata spontaneamente.") - disconnect_counter = 0 - - # Le funzioni di sistema e file IO sono ora blindate per non far crashare il loop - status = publish_all(client) - publish_all_ini_files(client) - check_auto_healing(client, status) - + status = get_system_status() + check_auto_healing(status) # l'auto-healing è locale: funziona anche senza broker except Exception as e: - # Cattura e scrive nei log qualsiasi anomalia interna che prima uccideva il thread - logger.error(f"❌ Errore critico nel thread Watchdog: {e}") - + logger.error(f"❌ Errore critico nel thread telemetria: {e}") + time.sleep(get_update_interval()) + + +def watchdog_tick(now=None): + """Se il broker resta irraggiungibile oltre max_offline_seconds il processo esce e systemd lo rilancia.""" + global disconnected_since + now = time.time() if now is None else now + if connected(): + if disconnected_since is not None: + logger.info("✅ Connessione MQTT ripristinata.") + disconnected_since = None + return + if disconnected_since is None: + disconnected_since = now + return + limit = int_setting('max_offline_seconds', 300, 30) + if now - disconnected_since >= limit: + logger.error(f"🚨 Broker MQTT irraggiungibile da {int(now - disconnected_since)}s. Riavvio del processo...") + os._exit(1) + + +def watchdog_loop(): + while True: try: - time.sleep(update_interval) + watchdog_tick() except Exception as e: - logger.error(f"⚠️ Errore nel timer di sleep: {e}") - time.sleep(30) # Fallback + logger.error(f"Watchdog error: {e}") + time.sleep(10) + + +def shutdown_handler(signum, frame): + """systemctl stop/restart: con una disconnessione 'pulita' il LWT non parte, quindi OFFLINE lo pubblichiamo noi.""" + logger.info("Shutdown richiesto: pubblico OFFLINE e chiudo.") + try: + info = mqtt_client.publish(TOPIC_STAT, f"OFFLINE - {CLIENT_ID.upper()}", qos=1, retain=True) + info.wait_for_publish(timeout=3) + mqtt_client.disconnect() + except Exception: + pass + os._exit(0) + def start_service(): - global current_status + global current_status, mqtt_client, heal_pause_until + load_state() current_status = get_actual_config_from_disk() + heal_pause_until = time.time() + int_setting('startup_grace_seconds', 60, 0) + client = mqtt.Client(callback_api_version=mqtt.CallbackAPIVersion.VERSION2, client_id=CLIENT_ID.upper()) - client.will_set(TOPIC_STAT, payload=f"OFFLINE - {CLIENT_ID.upper()}", qos=1, retain=False) - client.username_pw_set(cfg['mqtt']['user'], cfg['mqtt']['password']) + mqtt_client = client + # LWT retained: se il nodo muore il broker sostituisce anche il messaggio "ONLINE" conservato + client.will_set(TOPIC_STAT, payload=f"OFFLINE - {CLIENT_ID.upper()}", qos=1, retain=True) + client.username_pw_set(MQTT_CFG['user'], MQTT_CFG['password']) + if MQTT_CFG.get('tls'): + client.tls_set(ca_certs=MQTT_CFG.get('ca_cert') or None) + client.reconnect_delay_set(min_delay=1, max_delay=30) client.on_connect = on_connect + client.on_connect_fail = on_connect_fail client.on_disconnect = on_disconnect client.on_message = on_message - - # 1. Start the telemetry "engine" ONLY ONCE - threading.Thread(target=auto_publish_task, args=(client,), daemon=True).start() - + + for target, name in ((command_worker, "commands"), (telegram_worker, "telegram"), + (telemetry_loop, "telemetry"), (watchdog_loop, "watchdog")): + threading.Thread(target=target, name=name, daemon=True).start() + + signal.signal(signal.SIGTERM, shutdown_handler) + signal.signal(signal.SIGINT, shutdown_handler) + + logger.info(f"Node agent v{AGENT_VERSION} avviato ({CLIENT_ID.upper()}). Connessione al broker...") + # connect_async + loop_start: paho gestisce da solo primo collegamento e riconnessioni + client.connect_async(MQTT_CFG['broker'], int(MQTT_CFG['port']), int(MQTT_CFG.get('keepalive', 15))) + client.loop_start() + while True: - try: - logger.info("Attempting connection to MQTT broker...") - client.connect(cfg['mqtt']['broker'], cfg['mqtt']['port'], 15) - - # 2. Start network manager in background (handles reconnections automatically!) - client.loop_start() - - # 3. Pause main thread indefinitely - while True: - time.sleep(1) - - except Exception as e: - # This triggers ONLY if the broker is down when the node boots - logger.error(f"Broker unreachable at boot ({e}). Retrying in 10 seconds...") - time.sleep(10) + time.sleep(3600) + if __name__ == "__main__": start_service() From dd3848fbacf4c7a050f224e92ebd7add21c2c89b Mon Sep 17 00:00:00 2001 From: Roby Date: Fri, 16 Oct 2026 11:02:11 +0200 Subject: [PATCH 2/5] update README --- README.md | 418 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 370 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index d6429b2..ed5af3f 100644 --- a/README.md +++ b/README.md @@ -9,37 +9,168 @@ The **Fleet Node Agent** is a lightweight monitoring and control script designed for remote MMDVM nodes. It acts as the bridge between your radio hardware and the Central Console. -> ℹ️ **Note:** This is the Remote Agent repository, intended to be installed on individual repeater nodes. To manage and monitor all your fleet from a single web dashboard, you must also install the [Fleet Control Console (Server)](https://git.arifvg.it/iv3jdv/fleet-control-server) on your central hub. +> ℹ️ **Note:** This is the Remote Agent repository, intended to be installed on individual repeater nodes. To manage and monitor all your fleet from a single web dashboard, you must also install the [Fleet Control Console (Server)](https://git.arifvg.it/iv3jdv/fleet-control-server) on your central hub. Use the latest console version: older ones do not recognise the `OFFLINE - ` / `ONLINE - ` states sent by this agent, so node push notifications would not fire. +**Contents:** [Features](#en-features) · [How it works](#en-topics) · [Safe operations](#en-safe) · [Profiles](#en-profiles) · [Hardware reset](#en-reset) · [Telegram](#en-telegram) · [Security](#en-security) · [Configuration](#en-config) · [Installation](#en-install) · [Updating](#en-update) · [Troubleshooting](#en-trouble) + + ### ✨ Key Features -* **Real-Time Telemetry:** Streams CPU, RAM, Disk usage, and Temperature data via MQTT. -* **Service Management:** Allows remote Start/Stop/Restart of system daemons (MMDVMHost, DMRGateway, etc.). -* **Remote Configuration:** Enables the Central Server to edit `.ini` files remotely. -* **Hardware Reset:** Supports physical MMDVM HAT reset via GPIO pins (requires RPi.GPIO). -* **Auto-Healing:** Automatically detects service failures and attempts to restart them. +* **Real-Time Telemetry:** CPU, RAM, disk, temperature and the status of the monitored processes, sent via MQTT every `update_interval` seconds (together with the agent version and the interval, so the console can tell when an agent has gone silent). +* **Service Management:** Remote Start/Stop/Restart of system daemons (MMDVMHost, DMRGateway, etc.). Only services that the node knows about can be controlled (see [Security](#en-security)). +* **Remote Configuration:** The Central Server can edit your `.ini` files remotely, with a safety net: empty or oversized content is refused, a `.bak` copy is kept, and if the service does not come back up with the new file the previous version is restored automatically. +* **Dynamic Profiles:** Switch between complete configurations (e.g. BrandMeister ⇄ DMR+) with automatic rollback if something goes wrong. +* **Auto-Healing:** Detects service failures and restarts them (3 attempts, then a CRITICAL alert). It never restarts a service you stopped on purpose from the dashboard. +* **Hardware Reset:** Surgical reset of a stuck modem via USB or GPIO, without rebooting the node. +* **Safe OTA Update:** One-click `git pull` from the console, with a syntax check of the new code and automatic rollback if it is not valid. +* **Telegram Alerts:** Real-time notifications with configurable quiet hours; CRITICAL alerts are always delivered. +* **Resilient MQTT:** Automatic reconnection, retained last-will so that a dead node is never shown as online, and a watchdog for a broker that stays unreachable. +* **Lightweight:** Configuration is published only when files actually change, and passwords are masked in the per-section topics. +> 👥 **On the console:** restart/stop/start of services, the config editor, reboot, HAT reset and the OTA update are reserved to *admin* users. Profile switching and the Telegram toggle are available to operators allowed on that node. + + +### 📡 How it works (MQTT topics) +Everything goes through your MQTT broker. `` is your `client_id` in lowercase. + +| Topic | Direction | Content | +|---|---|---| +| `servizi//cmnd` | agent ⬅ console | Commands: a profile key (`A`, `B`, …), `TG:ON`, `TG:OFF`, `REBOOT`, `RESET_HAT`, `UPDATE_AGENT` | +| `servizi//stat` | agent ➡ console (retained) | `ONLINE - ` / `OFFLINE - ` (also used as Last Will) | +| `devices//services` | agent ➡ console | Telemetry JSON (CPU, RAM, disk, temperature, processes, profiles, files, `update_interval`, `agent_version`) | +| `devices//control` | agent ⬅ console | `start:`, `stop:`, `restart:` | +| `devices//config_set/` | agent ⬅ console | `{"raw_text": "..."}` – new content of a file from `file_list.txt` | +| `devices/control/request` | agent ⬅ console | `status` / `update`: republish everything (each node answers after a random 0–5 s delay) | +| `devices//logs` | agent ➡ console | Outcome of commands and auto-healing messages | +| `data///full_config` | agent ➡ console (retained) | Full text of each configuration file (used by the editor) | +| `data///
` | agent ➡ console (retained) | Each INI section as JSON (passwords masked) | +| `fleet//status` | agent ➡ console | Result of a `RESET_HAT` | + +> The console always uses `servizi/` as the base topic, so leave `mqtt.base_topic` at its default (`servizi/`). +> The dashboard reads the **Info/General** data from a file named `MMDVMHost.ini` or `MMDVM.ini`, and the DMR networks from `DMRGateway.ini`. Keep these names in `file_list.txt`. + + +### 🛡️ Safe operations +Every operation that touches services or files is executed by a **single worker thread**, one at a time, never inside the MQTT callback (so a slow `git pull` or `systemctl` can no longer cause a disconnection). + +* **Profile switch:** all source files and units are checked *before* anything is stopped. Then services are stopped, files copied atomically, services started, and (after up to `verify_seconds`) every unit is verified to be active. If anything fails, the previous configuration is restored and the services restarted. The node keeps its previous status and the error is reported to the console and Telegram. +* **Remote config write:** content must be a non-empty text up to 64 KB. The previous file is saved as `.bak`, the new one is written atomically and the *right* service is restarted (the one mapped to that file). If it does not start, the old content is restored. Files that are only a *source* of a profile (e.g. `MMDVMHost_BM.ini`) are saved without restarting anything. +* **OTA update (`UPDATE_AGENT`):** `git pull --ff-only` with a timeout. If the code changed, its syntax is checked; if invalid, the previous commit is restored (`git reset --hard`) and the agent keeps running. If valid, the agent exits and systemd restarts it with the new code. Credentials in git URLs are never published. +* **Auto-healing:** a service reported offline is restarted up to 3 times (with a hardware reset first for `mmdvmhost`, if configured), then a CRITICAL alert is sent once. It is skipped for services you stopped from the dashboard (remembered across restarts in `state_file`), for units that do not exist or are disabled on that node, while another operation is running (and for `heal_grace_seconds` after it), and during the first `startup_grace_seconds` after boot. It works even if the broker is unreachable. +* **Atomic writes:** configuration, last profile and state files are written to a temporary file and then renamed, keeping permissions. If the directory is not writable (e.g. a single file in `/etc` made writable for the agent user) it falls back to an in-place write. + + ### 🔄 Dynamic Profiles (Multiple Configurations) -The agent supports unlimited dynamic profile switching (e.g., switching between a BrandMeister and a DMR+ configuration on the fly). +The agent supports unlimited dynamic profile switching (e.g., switching between a BrandMeister and a DMR+ configuration on the fly). -In your `node_config.json`, you can define any number of custom profiles under the `"profiles"` key. For each profile, specify a label and the list of services/files to swap. -The Central Console dashboard will automatically read this configuration and dynamically generate a button for each profile. If you don't need profile switching for a specific node, simply leave the dictionary empty (`"profiles": {}` or remove the block entirely). The dashboard will gracefully hide the profile controls for that node, keeping the UI clean. +In your `node_config.json`, define any number of custom profiles under the `"profiles"` key. For each profile, specify a `label` and the list of services/files to swap (`name` of the systemd unit, `source` file to copy, `target` file to overwrite): +```json +"profiles": { + "A": { + "label": "Config BM", + "services": [ + { "name": "mmdvmhost.service", "source": "/etc/MMDVMHost_BM.ini", "target": "/etc/MMDVMHost.ini" }, + { "name": "dmrgateway.service", "source": "/etc/DMRGateway_BM.ini", "target": "/etc/DMRGateway.ini" } + ] + } +} +``` +The Central Console dashboard reads this configuration and generates a button for each profile. Profile keys are matched case-insensitively. If you don't need profile switching for a node, leave the dictionary empty (`"profiles": {}`) or remove the block: the dashboard hides the controls for that node. The active profile is remembered in `paths.last_profile` and restored at boot. + + +### 🔌 Hardware Resets (USB & GPIO) +The agent can reset a stuck modem without rebooting the whole node, either on demand (**Reset HAT** button) or automatically before restarting `mmdvmhost` during auto-healing. Configure it under `"settings"`: +* **USB modems (e.g., STM32 Nucleo):** set `"usb_reset_id"` to the device ID (e.g. `"0483:374b"`). Requires the `usbreset` tool (package `usbutils`) and, if the agent is not root, a sudoers rule. +* **GPIO HATs:** set `"gpio_reset_pin"` to your BCM pin (e.g. `21`). Requires the `RPi.GPIO` library (optional: if it is missing, or cannot be loaded on your board, the GPIO reset is simply disabled and a warning is written to the log). **There is no default pin**: GPIO is used only if the pin is configured. With `"gpio_reset_auto": false` the GPIO is excluded from auto-healing while the manual *Reset HAT* keeps working. + +USB has priority: if `"usb_reset_id"` is empty (`""`) the agent falls back to GPIO. The real outcome is always reported (a failed `usbreset` is no longer reported as "sent"). On a node without a HAT, leave `gpio_reset_pin` as `null`. + + ### 🤖 Telegram Bot Integration -The agent features built-in support for **Telegram Bot** notifications. This allows system administrators to receive real-time alerts regarding node status, offline services, and critical system events directly on their mobile devices. Notifications are highly manageable: they can be dynamically enabled or muted for each individual node directly from the central NOC Fleet Console, ensuring you only get notified when it matters. +Real-time alerts about node status, offline services and critical events on your phone. Configure the `telegram` block: -### 🔌 Dynamic Hardware Resets (USB & GPIO) -The agent can perform surgical hardware resets to stuck modems without rebooting the entire node. You can configure this dynamically in your `node_config.json` under the `"settings"` block: -* **USB Modems (e.g., STM32 Nucleo):** Set `"usb_reset_id"` to your device's ID (e.g., `"0483:374b"`). This requires the `usbutils` package installed on the host. -* **GPIO HATs:** Set `"gpio_reset_pin"` to your specific BCM pin (e.g., `21`). -The agent will automatically prioritize the USB reset. If `"usb_reset_id"` is left empty (`""`), it will gracefully fall back to the GPIO method. +```json +"telegram": { + "enabled": true, + "token": "123456:ABC...", + "chat_id": "123456789", + "quiet_hours": { "start": 23, "end": 7 } +} +``` +* **Quiet hours** (default 23:00–07:00, local time of the node) silence normal messages; **CRITICAL alerts always get through**. Use `"quiet_hours": false` to disable them. +* Notifications can be turned on or muted for each node from the Fleet Console (`TG:ON` / `TG:OFF`); the choice is saved in `node_config.json`. +* Messages are sent by a dedicated thread, and the bot token never appears in the logs. -### 🛠️ Core Configuration Files -To enable full functionality, you must configure these three files: -* **`node_config.json`**: The main configuration. Here you set your MQTT broker, your unique `client_id` (e.g., `IR3XXX`), and the paths for the lists below. -* **`process_list.txt`**: List the names of the system services you want to monitor (e.g., `mmdvmhost`, `dmrgateway`). The agent will check their status and attempt to restart them if they crash (Auto-healing). -* **`file_list.txt`**: List the full absolute paths of the configuration files (e.g., `/etc/MMDVMHost.ini`) that you want to edit remotely from the Dashboard. + +### 🔐 Security +The agent executes commands received over MQTT, so protect the channel: +* **Whitelist:** remote service control only works on names the node knows: `process_list.txt`, the services of the profiles and `settings.service_map`. Anything else (including systemctl options) is rejected. If the dashboard cannot control a service, add it to `process_list.txt`. +* **Passwords:** keys such as `Password`, `Pass`, `Secret`, `*Token`, `AuthKey` are masked in the per-section topics (`PassAllTG`/`PassAllPC` are not touched). The `full_config` topic, used by the editor, contains the **full file, passwords included**: restrict who can read `data/#` on the broker. Masking can be disabled with `mask_passwords: false`. +* **Broker:** use one MQTT user per node with ACLs limited to its own topics (`servizi//#`, `devices//#`, `data//#`, `fleet//#`), a separate user for the server, and TLS (`mqtt.tls`) if the broker is not on a LAN/VPN. MMDVMHost and the gateways use their own credentials, so they need rules too. +* **Retained commands are ignored**, so a stale message can never re-run a reboot. +* **Files:** `chmod 600 node_config.json`, and never commit it (add `node_config.json`, `node_agent.log*` and `*.bak` to `.gitignore`). +* **sudo:** the provided `fleet-agent.service` runs the agent as `root`, so no sudoers rule is needed. If you change `User=` to a non-root user, allow only what the agent needs (check paths with `which systemctl reboot usbreset`, validate with `visudo -c`): + ``` + # /etc/sudoers.d/fleet-agent (replace "pi" with the agent user) + pi ALL=(root) NOPASSWD: /usr/bin/systemctl start *, /usr/bin/systemctl stop *, /usr/bin/systemctl restart *, /usr/sbin/reboot, /usr/bin/usbreset * + ``` + +### 🛠️ Configuration +Three files drive the agent: +* **`node_config.json`**: main configuration (MQTT broker, your unique `client_id`, paths, settings, Telegram, profiles). `client_id` and the `mqtt` credentials are **required**; everything else is optional. Use only letters, digits, `_` and `-` in the `client_id` (the console accepts nothing else), and make it **unique**: two nodes with the same ID kick each other off the broker. +* **`process_list.txt`**: names of the processes/services to monitor, one per line (e.g. `mmdvmhost`, `dmrgateway`). They are checked, shown on the dashboard, auto-healed and controllable remotely. Lines starting with `#` are ignored. +* **`file_list.txt`**: absolute paths of the configuration files to expose to the editor (e.g. `/etc/MMDVMHost.ini`), one per line. The topic name is the file name without extension. + +Complete example (`node_config.json.example` contains the same keys): +```json +{ + "client_id": "ir3xxx", + "mqtt": { "broker": "127.0.0.1", "port": 1883, "user": "mmdvm", "password": "password", + "base_topic": "servizi/ir3xxx", "keepalive": 15, "tls": false }, + "paths": { + "file_list": "/opt/fleet-node-agent/file_list.txt", + "process_list": "/opt/fleet-node-agent/process_list.txt", + "last_profile": "/opt/last_profile.txt", + "state_file": "/opt/agent_state.json" + }, + "settings": { + "auto_healing": true, + "update_interval": 60, + "usb_reset_id": "", + "gpio_reset_pin": null, + "verify_seconds": 8, + "max_offline_seconds": 300 + }, + "telegram": { "enabled": false, "token": "TOKEN ID", "chat_id": "CHAT ID", "quiet_hours": { "start": 23, "end": 7 } }, + "profiles": {} +} +``` + +| Key | Default | Description | +|---|---|---| +| `settings.auto_healing` | `false` | Restart failed services (see above) | +| `settings.update_interval` | `30` | Seconds between telemetry messages (minimum 10) | +| `settings.usb_reset_id` | – | USB ID for `usbreset` (priority over GPIO) | +| `settings.gpio_reset_pin` | none | BCM pin of the HAT reset line. No implicit default | +| `settings.gpio_reset_auto` | `true` | `false`: never pulse the GPIO from auto-healing | +| `settings.verify_seconds` | `8` | How long to wait for services to be active after a profile switch / config write (`0` = don't verify) | +| `settings.max_offline_seconds` | `300` | If the broker stays unreachable this long, the process exits so that systemd restarts it | +| `settings.exit_on_disconnect` | `false` | `true`: exit at *every* disconnection (the old behaviour) | +| `settings.heal_grace_seconds` | `20` | Auto-healing pause after an operation | +| `settings.startup_grace_seconds` | `60` | Auto-healing pause after the agent starts | +| `settings.mask_passwords` | `true` | Mask passwords in per-section topics | +| `settings.service_map` | `{}` | `{"process": "unit.service"}` when the process name differs from the unit name | +| `paths.file_list` / `process_list` | `/opt/file_list.txt` / `/opt/process_list.txt` | Point them to where you created the files | +| `paths.last_profile` | `/opt/last_profile.txt` | Active profile label (must be writable) | +| `paths.state_file` | `/opt/agent_state.json` | Services stopped on purpose (must be writable) | +| `mqtt.keepalive` | `15` | MQTT keepalive in seconds | +| `mqtt.tls` / `mqtt.ca_cert` | `false` / – | Enable TLS (optional CA file) | +| `telegram.quiet_hours` | 23 → 7 | `{"start": h, "end": h}` or `false` | + + ### 🚀 Installation Guide +**Requirements:** Python 3.8+, `git`, and the packages in `requirements.txt` (`paho-mqtt>=2.0`, `psutil`, `requests`); optionally `RPi.GPIO` (GPIO reset, install it with `pip install RPi.GPIO` inside the venv) and `usbutils` (USB reset). #### 0. Root Privileges All installation steps must be executed as the `root` user. Before starting, elevate your privileges by running: @@ -50,7 +181,7 @@ sudo su #### 1. Clone the Repository Clone the agent into `/opt` to ensure path consistency: ```bash -sudo git clone https://git.arifvg.it/iv3jdv/fleet-node-agent.git /opt/fleet-node-agent +git clone https://git.arifvg.it/iv3jdv/fleet-node-agent.git /opt/fleet-node-agent cd /opt/fleet-node-agent ``` @@ -62,11 +193,12 @@ pip install -r requirements.txt ``` #### 3. Configuration -1. **Main Config**: `cp node_config.json.example node_config.json` and edit it with your MQTT credentials. +1. **Main Config**: `cp node_config.json.example node_config.json` and edit it with your `client_id` and MQTT credentials. 2. **Processes**: `cp process_list.txt.example process_list.txt` and add your service names (one per line). 3. **Files**: `cp file_list.txt.example file_list.txt` and add the absolute paths to your `.ini` files. - -**Note:** To enable Telegram notifications, ensure you have configured your `TELEGRAM_TOKEN` and `CHAT_ID` in the agent's configuration file. +4. **Paths**: make sure `paths.file_list` and `paths.process_list` in `node_config.json` point to the files you just created (e.g. `/opt/fleet-node-agent/file_list.txt`). +5. **Permissions**: `chmod 600 node_config.json`. +6. **Telegram** (optional): fill in the `telegram` block (`enabled`, `token`, `chat_id`). #### 4. Systemd Service (Auto-start) ```bash @@ -75,6 +207,35 @@ systemctl daemon-reload systemctl enable fleet-agent systemctl start fleet-agent ``` +> The provided unit runs the agent as `root` with the venv's Python and already has `Restart=always` and `RestartSec=10`. Keep both: the OTA update and the watchdog rely on systemd to restart the agent. + +Check that it is running: `journalctl -u fleet-agent -f`. The node should appear online in the console within a few seconds. + + +### 🔄 Updating +* **From the console:** use the **UPDATE OTA** button of the node (admin only). +* **By hand:** `cd /opt/fleet-node-agent && git pull && systemctl restart fleet-agent`. + +The OTA update needs the agent folder to be a git clone where `git pull` works without prompts (public repository or stored credentials). Keep local files (`node_config.json`, logs, `*.bak`) out of git to avoid conflicts. + +**Upgrading from 1.x:** your `node_config.json` keeps working unchanged. Three things behave differently: (1) a broker disconnection no longer restarts the process (paho reconnects by itself; see `max_offline_seconds` / `exit_on_disconnect`); (2) the GPIO reset needs `gpio_reset_pin` to be set explicitly; (3) `OFFLINE` is now a retained message and the agent republishes its status at every connection. + + +### 🩺 Troubleshooting +The node card on the console shows a line like `MMDVM 12s · GW 3m · AGENT 3s` with the age of the last message received for each source. +* **AGENT** is a periodic heartbeat: a warning (⚠) means the agent stopped publishing although the node is shown online (stuck connection?). +* **MMDVM** / **GW** arrive only when something happens (a transmission, a mode change…), so on a quiet repeater a long age is normal. A ⚠ on MMDVM means **nothing has ever been received** since the console started: check that MMDVMHost is connected to the broker (its MQTT section: broker, credentials, topic). + +| Symptom | Things to check | +|---|---| +| Node always offline | Broker address/port, credentials and ACLs; unique `client_id`; `journalctl -u fleet-agent` | +| A service cannot be controlled from the dashboard | It must be in `process_list.txt` (or in a profile / `service_map`) | +| Profile switch reports an error | The message names the cause (missing file, unit not active…); the previous profile has been restored. Increase `verify_seconds` for slow services | +| Edited config "disappears" | The service failed to start with it and the agent restored the old file (see the log message) | +| Reset HAT does nothing | `gpio_reset_pin` not set, `RPi.GPIO` not installed, or `usb_reset_id`/`usbreset` missing | +| A stopped service is not restarted by auto-healing | By design: it was stopped from the dashboard. Use *start*/*restart* to hand it back to auto-healing | + +Logs: `journalctl -u fleet-agent -f` and `node_agent.log` in the agent folder (rotating, 2 MB × 3). --- @@ -83,37 +244,168 @@ systemctl start fleet-agent Il **Fleet Node Agent** è lo script di monitoraggio e controllo per i nodi remoti MMDVM. Funge da ponte tra l'hardware radio e la Console Centrale. -> ℹ️ **Nota:** Questo è il repository dell'Agent Remoto, da installare sui singoli nodi ripetitore. Per gestire e monitorare tutta la flotta da un'unica dashboard web, devi installare anche la [Fleet Control Console (Server)](https://git.arifvg.it/iv3jdv/fleet-control-server) sul tuo hub centrale. +> ℹ️ **Nota:** Questo è il repository dell'Agent Remoto, da installare sui singoli nodi ripetitore. Per gestire e monitorare tutta la flotta da un'unica dashboard web, devi installare anche la [Fleet Control Console (Server)](https://git.arifvg.it/iv3jdv/fleet-control-server) sul tuo hub centrale. Usa la versione più recente della console: le precedenti non riconoscono gli stati `OFFLINE - ` / `ONLINE - ` inviati da questo agent, quindi le notifiche push dei nodi non partirebbero. +**Indice:** [Funzionalità](#it-features) · [Come funziona](#it-topics) · [Operazioni sicure](#it-safe) · [Profili](#it-profiles) · [Reset hardware](#it-reset) · [Telegram](#it-telegram) · [Sicurezza](#it-security) · [Configurazione](#it-config) · [Installazione](#it-install) · [Aggiornamento](#it-update) · [Risoluzione problemi](#it-trouble) + + ### ✨ Funzionalità Principali -* **Telemetria Real-Time:** Invia dati su CPU, RAM, Disco e Temperatura via MQTT. -* **Gestione Servizi:** Permette l'avvio, l'arresto o il riavvio remoto dei demoni di sistema (MMDVMHost, DMRGateway, ecc.). -* **Configurazione Remota:** Consente al Server Centrale di modificare i file `.ini` a distanza. -* **Reset Hardware:** Supporta il reset fisico della scheda MMDVM HAT tramite pin GPIO (richiede RPi.GPIO). -* **Auto-Healing:** Rileva automaticamente i crash dei servizi e tenta di riavviarli. +* **Telemetria Real-Time:** CPU, RAM, disco, temperatura e stato dei processi monitorati, inviati via MQTT ogni `update_interval` secondi (insieme a versione dell'agent e intervallo, così la console capisce quando un agent è diventato muto). +* **Gestione Servizi:** Avvio, arresto o riavvio remoto dei demoni di sistema (MMDVMHost, DMRGateway, ecc.). Si possono controllare solo i servizi che il nodo conosce (vedi [Sicurezza](#it-security)). +* **Configurazione Remota:** Il Server Centrale può modificare i file `.ini` a distanza, con una rete di sicurezza: contenuti vuoti o troppo grandi vengono rifiutati, viene conservata una copia `.bak` e, se il servizio non riparte con il nuovo file, la versione precedente viene ripristinata automaticamente. +* **Profili Dinamici:** Cambio tra configurazioni complete (es. BrandMeister ⇄ DMR+) con rollback automatico se qualcosa va storto. +* **Auto-Healing:** Rileva i crash dei servizi e li riavvia (3 tentativi, poi un allarme CRITICAL). Non riavvia mai un servizio che hai fermato di proposito dalla dashboard. +* **Reset Hardware:** Reset "chirurgico" di un modem bloccato via USB o GPIO, senza riavviare il nodo. +* **Aggiornamento OTA sicuro:** `git pull` con un click dalla console, con controllo di sintassi del nuovo codice e rollback automatico se non è valido. +* **Avvisi Telegram:** Notifiche in tempo reale con ore silenziose configurabili; gli allarmi CRITICAL arrivano sempre. +* **MQTT resiliente:** Riconnessione automatica, last-will retained (un nodo morto non risulta mai online) e watchdog per un broker che resta irraggiungibile. +* **Leggero:** Le configurazioni vengono pubblicate solo quando i file cambiano davvero e le password sono mascherate nei topic per sezione. +> 👥 **Sulla console:** riavvio/stop/start dei servizi, editor di configurazione, reboot, reset HAT e aggiornamento OTA sono riservati agli utenti *admin*. Cambio profilo e interruttore Telegram sono disponibili agli operatori abilitati su quel nodo. + + +### 📡 Come funziona (topic MQTT) +Tutto passa dal tuo broker MQTT. `` è il tuo `client_id` in minuscolo. + +| Topic | Direzione | Contenuto | +|---|---|---| +| `servizi//cmnd` | agent ⬅ console | Comandi: chiave di un profilo (`A`, `B`, …), `TG:ON`, `TG:OFF`, `REBOOT`, `RESET_HAT`, `UPDATE_AGENT` | +| `servizi//stat` | agent ➡ console (retained) | `ONLINE - ` / `OFFLINE - ` (usato anche come Last Will) | +| `devices//services` | agent ➡ console | JSON di telemetria (CPU, RAM, disco, temperatura, processi, profili, file, `update_interval`, `agent_version`) | +| `devices//control` | agent ⬅ console | `start:`, `stop:`, `restart:` | +| `devices//config_set/` | agent ⬅ console | `{"raw_text": "..."}` – nuovo contenuto di un file di `file_list.txt` | +| `devices/control/request` | agent ⬅ console | `status` / `update`: ripubblica tutto (ogni nodo risponde dopo un'attesa casuale di 0–5 s) | +| `devices//logs` | agent ➡ console | Esito dei comandi e messaggi dell'auto-healing | +| `data///full_config` | agent ➡ console (retained) | Testo completo di ogni file di configurazione (usato dall'editor) | +| `data///` | agent ➡ console (retained) | Ogni sezione INI come JSON (password mascherate) | +| `fleet//status` | agent ➡ console | Esito di un `RESET_HAT` | + +> La console usa sempre `servizi/` come topic base: lascia `mqtt.base_topic` al valore predefinito (`servizi/`). +> La dashboard legge i dati **Info/General** da un file chiamato `MMDVMHost.ini` o `MMDVM.ini`, e le reti DMR da `DMRGateway.ini`. Mantieni questi nomi in `file_list.txt`. + + +### 🛡️ Operazioni sicure +Ogni operazione che tocca servizi o file è eseguita da **un unico thread di lavoro**, una alla volta, mai dentro la callback MQTT (così un `git pull` o un `systemctl` lenti non possono più provocare una disconnessione). + +* **Cambio profilo:** tutti i file sorgente e le unità vengono controllati *prima* di fermare qualsiasi cosa. Poi si fermano i servizi, si copiano i file in modo atomico, si avviano i servizi e (entro `verify_seconds`) si verifica che ogni unità sia attiva. Se qualcosa fallisce, la configurazione precedente viene ripristinata e i servizi riavviati. Il nodo mantiene lo stato precedente e l'errore viene segnalato alla console e a Telegram. +* **Scrittura config da remoto:** il contenuto deve essere un testo non vuoto fino a 64 KB. Il file precedente viene salvato come `.bak`, il nuovo viene scritto in modo atomico e viene riavviato il servizio *giusto* (quello associato al file). Se non parte, torna il contenuto precedente. I file che sono solo *sorgente* di un profilo (es. `MMDVMHost_BM.ini`) vengono salvati senza riavviare nulla. +* **Aggiornamento OTA (`UPDATE_AGENT`):** `git pull --ff-only` con timeout. Se il codice è cambiato ne viene controllata la sintassi; se non è valido si torna al commit precedente (`git reset --hard`) e l'agent continua a girare. Se è valido, l'agent esce e systemd lo riavvia con il nuovo codice. Le credenziali negli URL di git non vengono mai pubblicate. +* **Auto-healing:** un servizio segnalato offline viene riavviato fino a 3 volte (prima con un reset hardware per `mmdvmhost`, se configurato), poi parte una sola volta un allarme CRITICAL. Viene saltato per i servizi che hai fermato dalla dashboard (ricordati anche dopo un riavvio in `state_file`), per le unità inesistenti o disabilitate su quel nodo, mentre un'altra operazione è in corso (e per `heal_grace_seconds` dopo), e nei primi `startup_grace_seconds` dopo l'avvio. Funziona anche se il broker è irraggiungibile. +* **Scritture atomiche:** configurazione, ultimo profilo e file di stato vengono scritti su un file temporaneo e poi rinominati, mantenendo i permessi. Se la directory non è scrivibile (es. un singolo file in `/etc` reso scrivibile all'utente dell'agent) si ripiega sulla scrittura in-place. + + ### 🔄 Profili Dinamici Multipli (Dynamic Profiles) L'agent supporta il cambio rapido tra un numero illimitato di configurazioni (es. per passare "al volo" da un setup BrandMeister a uno DMR+). -Nel file `node_config.json`, puoi definire quanti profili desideri sotto la chiave `"profiles"`. Per ogni profilo, ti basta specificare un'etichetta (label) e i servizi/file da scambiare. -La dashboard centrale leggerà questa configurazione e genererà dinamicamente un pulsante per ogni profilo inserito. Se un nodo non necessita di questa funzione, è sufficiente lasciare il dizionario vuoto (`"profiles": {}` o rimuovere del tutto il blocco). La dashboard capirà in automatico di dover nascondere i controlli dei profili per quel nodo specifico, mantenendo l'interfaccia pulita. +Nel file `node_config.json` puoi definire quanti profili desideri sotto la chiave `"profiles"`. Per ogni profilo specifica un'etichetta (`label`) e l'elenco di servizi/file da scambiare (`name` dell'unità systemd, file `source` da copiare, file `target` da sovrascrivere): +```json +"profiles": { + "A": { + "label": "Config BM", + "services": [ + { "name": "mmdvmhost.service", "source": "/etc/MMDVMHost_BM.ini", "target": "/etc/MMDVMHost.ini" }, + { "name": "dmrgateway.service", "source": "/etc/DMRGateway_BM.ini", "target": "/etc/DMRGateway.ini" } + ] + } +} +``` +La dashboard centrale legge questa configurazione e genera un pulsante per ogni profilo. Le chiavi dei profili sono confrontate senza distinguere maiuscole e minuscole. Se un nodo non necessita di questa funzione, lascia il dizionario vuoto (`"profiles": {}`) o rimuovi il blocco: la dashboard nasconde i controlli per quel nodo. Il profilo attivo viene ricordato in `paths.last_profile` e ripristinato all'avvio. + + +### 🔌 Reset Hardware (USB & GPIO) +L'agent può resettare un modem bloccato senza riavviare l'intero nodo, su richiesta (pulsante **Reset HAT**) oppure in automatico prima di riavviare `mmdvmhost` durante l'auto-healing. Si configura sotto `"settings"`: +* **Modem USB (es. STM32 Nucleo):** imposta `"usb_reset_id"` con l'ID del dispositivo (es. `"0483:374b"`). Richiede lo strumento `usbreset` (pacchetto `usbutils`) e, se l'agent non è root, una regola sudoers. +* **HAT GPIO:** imposta `"gpio_reset_pin"` con il tuo pin BCM (es. `21`). Richiede la libreria `RPi.GPIO` (facoltativa: se manca, o non può essere caricata sulla tua scheda, il reset GPIO viene semplicemente disattivato e nel log compare un avviso). **Non esiste un pin di default**: il GPIO si usa solo se il pin è configurato. Con `"gpio_reset_auto": false` il GPIO è escluso dall'auto-healing mentre il *Reset HAT* manuale continua a funzionare. + +L'USB ha la priorità: se `"usb_reset_id"` è vuoto (`""`) si ripiega sul GPIO. L'esito reale viene sempre riportato (un `usbreset` fallito non viene più segnalato come "inviato"). Su un nodo senza HAT lascia `gpio_reset_pin` a `null`. + + ### 🤖 Telegram Bot Integration -L'agent dispone del supporto nativo per le notifiche tramite **Bot Telegram**. Questo permette agli amministratori di sistema di ricevere alert in tempo reale sullo stato dei nodi, sui servizi offline e sugli eventi critici direttamente sul proprio smartphone. La gestione degli avvisi è centralizzata: le notifiche possono essere attivate o silenziate dinamicamente per ogni singolo nodo direttamente dalla Fleet Console centrale, evitando spam inutile. +Alert in tempo reale sullo stato dei nodi, sui servizi offline e sugli eventi critici direttamente sullo smartphone. Configura il blocco `telegram`: -### 🔌 Reset Hardware Dinamico (USB & GPIO) -L'agente può eseguire reset hardware "chirurgici" ai modem bloccati senza dover riavviare l'intero nodo. Puoi configurare questa funzione dinamicamente nel file `node_config.json` sotto il blocco `"settings"`: -* **Modem USB (es. STM32 Nucleo):** Imposta `"usb_reset_id"` con l'ID del tuo dispositivo (es. `"0483:374b"`). Richiede il pacchetto `usbutils` installato sul nodo. -* **HAT GPIO:** Imposta `"gpio_reset_pin"` con il tuo pin BCM specifico (es. `21`). -L'agente darà sempre priorità al reset USB. Se il campo `"usb_reset_id"` viene lasciato vuoto (`""`), il sistema ripiegherà automaticamente sul metodo GPIO. +```json +"telegram": { + "enabled": true, + "token": "123456:ABC...", + "chat_id": "123456789", + "quiet_hours": { "start": 23, "end": 7 } +} +``` +* Le **ore silenziose** (predefinite 23:00–07:00, ora locale del nodo) silenziano i messaggi normali; gli **allarmi CRITICAL passano sempre**. Con `"quiet_hours": false` si disattivano. +* Le notifiche si possono attivare o silenziare per ogni nodo dalla Fleet Console (`TG:ON` / `TG:OFF`); la scelta viene salvata in `node_config.json`. +* I messaggi partono da un thread dedicato e il token del bot non compare mai nei log. -### 🛠️ File di Configurazione Chiave -Per il corretto funzionamento, è necessario definire i parametri in questi tre file: -* **`node_config.json`**: La configurazione principale. Qui imposti il broker MQTT, il tuo `client_id` univoco (es. `IR3XXX`) e i percorsi per le liste sottostanti. -* **`process_list.txt`**: Elenca i nomi dei servizi di sistema da monitorare (es. `mmdvmhost`, `dmrgateway`). L'agente ne controllerà lo stato e proverà a riavviarli in caso di crash (Auto-healing). -* **`file_list.txt`**: Elenca i percorsi completi dei file di configurazione (es. `/etc/MMDVMHost.ini`) che desideri poter modificare remotamente dalla Dashboard. + +### 🔐 Sicurezza +L'agent esegue comandi ricevuti via MQTT, quindi proteggi il canale: +* **Whitelist:** il controllo remoto dei servizi funziona solo sui nomi che il nodo conosce: `process_list.txt`, i servizi dei profili e `settings.service_map`. Tutto il resto (comprese le opzioni di systemctl) viene rifiutato. Se la dashboard non riesce a controllare un servizio, aggiungilo a `process_list.txt`. +* **Password:** chiavi come `Password`, `Pass`, `Secret`, `*Token`, `AuthKey` sono mascherate nei topic per sezione (`PassAllTG`/`PassAllPC` non vengono toccate). Il topic `full_config`, usato dall'editor, contiene il **file completo, password comprese**: limita chi può leggere `data/#` sul broker. Il mascheramento si disattiva con `mask_passwords: false`. +* **Broker:** un utente MQTT per nodo con ACL limitate ai suoi topic (`servizi//#`, `devices//#`, `data//#`, `fleet//#`), un utente separato per il server e TLS (`mqtt.tls`) se il broker non è in LAN/VPN. MMDVMHost e i gateway usano le proprie credenziali, quindi servono regole anche per loro. +* **I comandi retained vengono ignorati**, quindi un messaggio vecchio non può rieseguire un reboot. +* **File:** `chmod 600 node_config.json` e non committarlo mai (aggiungi `node_config.json`, `node_agent.log*` e `*.bak` al `.gitignore`). +* **sudo:** il `fleet-agent.service` fornito fa girare l'agent come `root`, quindi non serve nessuna regola sudoers. Se cambi `User=` con un utente non-root, consenti solo ciò che serve all'agent (verifica i percorsi con `which systemctl reboot usbreset`, controlla con `visudo -c`): + ``` + # /etc/sudoers.d/fleet-agent (sostituisci "pi" con l'utente dell'agent) + pi ALL=(root) NOPASSWD: /usr/bin/systemctl start *, /usr/bin/systemctl stop *, /usr/bin/systemctl restart *, /usr/sbin/reboot, /usr/bin/usbreset * + ``` + +### 🛠️ Configurazione +Tre file guidano l'agent: +* **`node_config.json`**: configurazione principale (broker MQTT, `client_id` univoco, percorsi, impostazioni, Telegram, profili). `client_id` e le credenziali `mqtt` sono **obbligatori**; tutto il resto è facoltativo. Nel `client_id` usa solo lettere, numeri, `_` e `-` (la console non accetta altro) e fai in modo che sia **univoco**: due nodi con lo stesso ID si scollegano a vicenda dal broker. +* **`process_list.txt`**: nomi dei processi/servizi da monitorare, uno per riga (es. `mmdvmhost`, `dmrgateway`). Vengono controllati, mostrati in dashboard, riavviati dall'auto-healing e controllabili da remoto. Le righe che iniziano con `#` sono ignorate. +* **`file_list.txt`**: percorsi assoluti dei file di configurazione da esporre all'editor (es. `/etc/MMDVMHost.ini`), uno per riga. Il nome del topic è il nome del file senza estensione. + +Esempio completo (`node_config.json.example` contiene le stesse chiavi): +```json +{ + "client_id": "ir3xxx", + "mqtt": { "broker": "127.0.0.1", "port": 1883, "user": "mmdvm", "password": "password", + "base_topic": "servizi/ir3xxx", "keepalive": 15, "tls": false }, + "paths": { + "file_list": "/opt/fleet-node-agent/file_list.txt", + "process_list": "/opt/fleet-node-agent/process_list.txt", + "last_profile": "/opt/last_profile.txt", + "state_file": "/opt/agent_state.json" + }, + "settings": { + "auto_healing": true, + "update_interval": 60, + "usb_reset_id": "", + "gpio_reset_pin": null, + "verify_seconds": 8, + "max_offline_seconds": 300 + }, + "telegram": { "enabled": false, "token": "TOKEN ID", "chat_id": "CHAT ID", "quiet_hours": { "start": 23, "end": 7 } }, + "profiles": {} +} +``` + +| Chiave | Default | Descrizione | +|---|---|---| +| `settings.auto_healing` | `false` | Riavvia i servizi caduti (vedi sopra) | +| `settings.update_interval` | `30` | Secondi tra i messaggi di telemetria (minimo 10) | +| `settings.usb_reset_id` | – | ID USB per `usbreset` (priorità sul GPIO) | +| `settings.gpio_reset_pin` | nessuno | Pin BCM della linea di reset dell'HAT. Nessun default implicito | +| `settings.gpio_reset_auto` | `true` | `false`: l'auto-healing non dà mai impulsi sul GPIO | +| `settings.verify_seconds` | `8` | Quanto attendere che i servizi siano attivi dopo un cambio profilo / scrittura config (`0` = non verificare) | +| `settings.max_offline_seconds` | `300` | Se il broker resta irraggiungibile così a lungo, il processo esce e systemd lo riavvia | +| `settings.exit_on_disconnect` | `false` | `true`: esce a *ogni* disconnessione (il vecchio comportamento) | +| `settings.heal_grace_seconds` | `20` | Pausa dell'auto-healing dopo un'operazione | +| `settings.startup_grace_seconds` | `60` | Pausa dell'auto-healing dopo l'avvio dell'agent | +| `settings.mask_passwords` | `true` | Maschera le password nei topic per sezione | +| `settings.service_map` | `{}` | `{"processo": "unita.service"}` quando il nome del processo differisce da quello dell'unità | +| `paths.file_list` / `process_list` | `/opt/file_list.txt` / `/opt/process_list.txt` | Puntali a dove hai creato i file | +| `paths.last_profile` | `/opt/last_profile.txt` | Etichetta del profilo attivo (deve essere scrivibile) | +| `paths.state_file` | `/opt/agent_state.json` | Servizi fermati di proposito (deve essere scrivibile) | +| `mqtt.keepalive` | `15` | Keepalive MQTT in secondi | +| `mqtt.tls` / `mqtt.ca_cert` | `false` / – | Abilita TLS (file CA facoltativo) | +| `telegram.quiet_hours` | 23 → 7 | `{"start": h, "end": h}` oppure `false` | + + ### 🚀 Guida all'Installazione +**Requisiti:** Python 3.8+, `git` e i pacchetti di `requirements.txt` (`paho-mqtt>=2.0`, `psutil`, `requests`); facoltativi `RPi.GPIO` (reset GPIO, si installa con `pip install RPi.GPIO` dentro il venv) e `usbutils` (reset USB). #### 0. Privilegi di Root Tutti i passaggi di installazione devono essere eseguiti come utente `root`. Prima di iniziare, eleva i tuoi privilegi eseguendo: @@ -122,9 +414,9 @@ sudo su ``` #### 1. Clonazione del Repository -Clona l'agente nella cartella `/opt` per garantire la coerenza con i servizi systemd: +Clona l'agent nella cartella `/opt` per garantire la coerenza con i servizi systemd: ```bash -sudo git clone https://git.arifvg.it/iv3jdv/fleet-node-agent.git /opt/fleet-node-agent +git clone https://git.arifvg.it/iv3jdv/fleet-node-agent.git /opt/fleet-node-agent cd /opt/fleet-node-agent ``` @@ -136,11 +428,12 @@ pip install -r requirements.txt ``` #### 3. Personalizzazione -1. **Config Principale**: `cp node_config.json.example node_config.json` e inserisci i dati MQTT. +1. **Config Principale**: `cp node_config.json.example node_config.json` e inserisci il tuo `client_id` e i dati MQTT. 2. **Processi**: `cp process_list.txt.example process_list.txt` e aggiungi i nomi dei tuoi servizi (uno per riga). 3. **File**: `cp file_list.txt.example file_list.txt` e inserisci i percorsi assoluti dei tuoi file `.ini`. - -**Nota:** Per abilitare le notifiche Telegram, assicurati di aver configurato il tuo `TELEGRAM_TOKEN` e il `CHAT_ID` nel file di configurazione dell'agent. +4. **Percorsi**: verifica che `paths.file_list` e `paths.process_list` in `node_config.json` puntino ai file appena creati (es. `/opt/fleet-node-agent/file_list.txt`). +5. **Permessi**: `chmod 600 node_config.json`. +6. **Telegram** (facoltativo): compila il blocco `telegram` (`enabled`, `token`, `chat_id`). #### 4. Esecuzione come Servizio (systemd) ```bash @@ -149,6 +442,35 @@ systemctl daemon-reload systemctl enable fleet-agent systemctl start fleet-agent ``` +> L'unità fornita fa girare l'agent come `root` con il Python del venv e ha già `Restart=always` e `RestartSec=10`. Mantienili entrambi: l'aggiornamento OTA e il watchdog si affidano a systemd per riavviare l'agent. + +Verifica che giri: `journalctl -u fleet-agent -f`. Il nodo dovrebbe comparire online nella console entro pochi secondi. + + +### 🔄 Aggiornamento +* **Dalla console:** usa il pulsante **UPDATE OTA** del nodo (solo admin). +* **A mano:** `cd /opt/fleet-node-agent && git pull && systemctl restart fleet-agent`. + +L'aggiornamento OTA richiede che la cartella dell'agent sia un clone git in cui `git pull` funzioni senza richieste interattive (repository pubblico o credenziali salvate). Tieni i file locali (`node_config.json`, log, `*.bak`) fuori da git per evitare conflitti. + +**Passando dalla 1.x:** il tuo `node_config.json` continua a funzionare senza modifiche. Tre cose si comportano in modo diverso: (1) una disconnessione dal broker non riavvia più il processo (paho si riconnette da solo; vedi `max_offline_seconds` / `exit_on_disconnect`); (2) il reset GPIO richiede `gpio_reset_pin` impostato esplicitamente; (3) `OFFLINE` ora è un messaggio retained e l'agent ripubblica il proprio stato a ogni connessione. + + +### 🩺 Risoluzione problemi +La card del nodo nella console mostra una riga come `MMDVM 12s · GW 3m · AGENT 3s` con l'età dell'ultimo messaggio ricevuto da ciascuna fonte. +* **AGENT** è un heartbeat periodico: un avviso (⚠) significa che l'agent ha smesso di pubblicare pur risultando online (connessione bloccata?). +* **MMDVM** / **GW** arrivano solo quando succede qualcosa (una trasmissione, un cambio modo…), quindi su un ripetitore tranquillo un'età lunga è normale. Un ⚠ su MMDVM significa che **non è mai arrivato nulla** da quando la console è attiva: controlla che MMDVMHost sia collegato al broker (sezione MQTT: broker, credenziali, topic). + +| Sintomo | Cosa controllare | +|---|---| +| Nodo sempre offline | Indirizzo/porta del broker, credenziali e ACL; `client_id` univoco; `journalctl -u fleet-agent` | +| Un servizio non si controlla dalla dashboard | Deve essere in `process_list.txt` (o in un profilo / `service_map`) | +| Il cambio profilo segnala un errore | Il messaggio indica la causa (file mancante, unità non attiva…); il profilo precedente è stato ripristinato. Aumenta `verify_seconds` per i servizi lenti | +| La config modificata "sparisce" | Il servizio non partiva con quella config e l'agent ha ripristinato il file vecchio (vedi il messaggio nel log) | +| Reset HAT non fa nulla | `gpio_reset_pin` non impostato, `RPi.GPIO` non installata, oppure mancano `usb_reset_id`/`usbreset` | +| Un servizio fermato non viene riavviato dall'auto-healing | È voluto: l'hai fermato dalla dashboard. Usa *start*/*restart* per restituirlo all'auto-healing | + +Log: `journalctl -u fleet-agent -f` e `node_agent.log` nella cartella dell'agent (a rotazione, 2 MB × 3). --- *Created by IV3JDV @ ARIFVG - 2026* From 47dfe7c53125afe2f0c674c0530a6d3917dd7fae Mon Sep 17 00:00:00 2001 From: Roby Date: Fri, 16 Oct 2026 11:08:10 +0200 Subject: [PATCH 3/5] clean --- node_config.json.example | 57 ---------------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 node_config.json.example diff --git a/node_config.json.example b/node_config.json.example deleted file mode 100644 index 69b2259..0000000 --- a/node_config.json.example +++ /dev/null @@ -1,57 +0,0 @@ -{ - "client_id": "repeater_id", - "mqtt": { - "broker": "127.0.0.1", - "port": 1883, - "user": "mmdvm", - "password": "password", - "base_topic": "servizi/repeater_id" - }, - "paths": { - "file_list": "/opt/file_list.txt", - "process_list": "/opt/process_list.txt" - }, - "settings": { - "auto_healing": true, - "update_interval": 60, - "usb_reset_id": "", - "gpio_reset_pin": 21 - }, - "telegram": { - "enabled": false, - "token": "TOKEN ID", - "chat_id": "CHAT ID" - }, - "profiles": { - "A": { - "label": "Config BM", - "services": [ - { - "name": "mmdvmhost.service", - "source": "/etc/MMDVMHost_BM.ini", - "target": "/etc/MMDVMHost.ini" - }, - { - "name": "dmrgateway.service", - "source": "/etc/DMRGateway_BM.ini", - "target": "/etc/DMRGateway.ini" - } - ] - }, - "B": { - "label": "Config PC", - "services": [ - { - "name": "mmdvmhost.service", - "source": "/etc/MMDVMHost_PC.ini", - "target": "/etc/MMDVMHost.ini" - }, - { - "name": "dmrgateway.service", - "source": "/etc/DMRGateway_PC.ini", - "target": "/etc/DMRGateway.ini" - } - ] - } - } -} From f5d115bf828333c50fde763163361a826abb52d9 Mon Sep 17 00:00:00 2001 From: Roby Date: Fri, 16 Oct 2026 11:09:47 +0200 Subject: [PATCH 4/5] clean --- node_config_json.example => node_config.json.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename node_config_json.example => node_config.json.example (100%) diff --git a/node_config_json.example b/node_config.json.example similarity index 100% rename from node_config_json.example rename to node_config.json.example From de4aded818df7aa0a958f31d58bff86c5508e408 Mon Sep 17 00:00:00 2001 From: Roby Date: Fri, 16 Oct 2026 11:34:19 +0200 Subject: [PATCH 5/5] update path --- node_config.json.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node_config.json.example b/node_config.json.example index b38a0b5..409309d 100644 --- a/node_config.json.example +++ b/node_config.json.example @@ -10,10 +10,10 @@ "tls": false }, "paths": { - "file_list": "/opt/file_list.txt", - "process_list": "/opt/process_list.txt", - "last_profile": "/opt/last_profile.txt", - "state_file": "/opt/agent_state.json" + "file_list": "/opt/fleet-node-agent/file_list.txt", + "process_list": "/opt/fleet-node-agent/process_list.txt", + "last_profile": "/opt/fleet-node-agent/last_profile.txt", + "state_file": "/opt/fleet-node-agent/agent_state.json" }, "settings": { "auto_healing": true,