#!/usr/bin/env python3 """ Applotus Sentinel Node Agent ============================ A lightweight, zero-dependency node agent for Linux/Unix VPS. Reports system telemetry, public IP, open sockets, and executes remote commands. Supports SSL certificate verification bypass via --insecure / standard flags. """ import io import os import re import sys import time import json import ssl import socket import platform import subprocess import signal import argparse import urllib.request import urllib.error import urllib.parse import http.client import struct import base64 import threading import select import collections from concurrent.futures import ThreadPoolExecutor import gzip import zlib import zipfile import py_compile import shutil import warnings import tempfile warnings.filterwarnings('ignore', category=DeprecationWarning) try: import pty import fcntl import termios except ImportError: pty = None fcntl = None termios = None try: import ctypes if platform.system() == "Windows": class MEMORYSTATUSEX(ctypes.Structure): _fields_ = [ ("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong), ("ullTotalPhys", ctypes.c_ulonglong), ("ullAvailPhys", ctypes.c_ulonglong), ("ullTotalPageFile", ctypes.c_ulonglong), ("ullAvailPageFile", ctypes.c_ulonglong), ("ullTotalVirtual", ctypes.c_ulonglong), ("ullAvailVirtual", ctypes.c_ulonglong), ("ullAvailExtendedVirtual", ctypes.c_ulonglong), ] except Exception: ctypes = None VERSION = "1.82.4" # The agent has never captured its own stdout/stderr before -- output just goes wherever its # supervisor sends it (a terminal, a nohup file, the systemd journal), invisible to Applotus # itself. Tee'd into a bounded ring buffer here so a browser can ask for recent log lines the same # way it already can for a container's logs (get_container_logs below), without needing every one # of this file's ~160 print() call sites to route through anything new. AGENT_LOG_MAX_LINES = 1000 AGENT_LOG_MAX_BYTES = 200 * 1024 _agent_log_buffer = collections.deque(maxlen=AGENT_LOG_MAX_LINES) _agent_log_lock = threading.Lock() class _TeeStream: def __init__(self, real_stream): self._real = real_stream self._pending = "" def write(self, data): # Backstop for an unencodable character, below _force_utf8_stdio's stream-level fix. # # ~101 of this file's print() lines carry emoji or box characters. On Windows, when stdout # is a PIPE rather than a console (which is exactly what PowerShell gives a native command # it is capturing), Python encodes with the legacy ANSI code page -- cp1252 on most # installs -- and the first such line raises UnicodeEncodeError from charmap_encode. # Reported from Windows 11 as the agent dying immediately after desktop discovery: the # service-mesh banner is simply the first emoji line that executes. # # _force_utf8_stdio() below normally prevents this outright. This stays because a log line # must never be able to terminate the agent even if that reconfigure was refused (an # exotic stream, a Python too old to expose reconfigure, a supervisor substituting its # own object): a mangled character in a log line is a cosmetic loss, a dead agent is not. try: self._real.write(data) except UnicodeEncodeError: enc = getattr(self._real, "encoding", None) or "ascii" self._real.write(data.encode(enc, "replace").decode(enc, "replace")) self._pending += data while "\n" in self._pending: line, self._pending = self._pending.split("\n", 1) with _agent_log_lock: _agent_log_buffer.append(line) return len(data) def flush(self): self._real.flush() def isatty(self): return self._real.isatty() def _force_utf8_stdio(): """Makes stdout/stderr able to carry this file's non-ASCII log characters on Windows. Must run BEFORE _install_agent_log_capture wraps the streams, since _TeeStream holds the real stream it was constructed with and does not expose reconfigure itself. errors='replace' rather than the default 'strict': if some stream still cannot represent a character after this, the right outcome is a '?' in a log line, not a dead agent. line_buffering because a piped stdout is block-buffered by default, which is why a crashed agent's install transcript arrives truncated mid-startup -- the startup banners are still sitting in a 8KB buffer that never gets flushed. Costs nothing at this file's log volume. """ for name in ("stdout", "stderr"): stream = getattr(sys, name, None) reconfigure = getattr(stream, "reconfigure", None) if reconfigure is None: continue try: reconfigure(encoding="utf-8", errors="replace", line_buffering=True) except Exception: # Already-detached, replaced by a supervisor, or a build that refuses the change -- # _TeeStream.write's own fallback covers it from here. pass def _install_agent_log_capture(): _force_utf8_stdio() sys.stdout = _TeeStream(sys.stdout) sys.stderr = _TeeStream(sys.stderr) def get_agent_logs(tail=200): """Returns this agent's own recently-captured log lines -- same response shape as get_container_logs, but sourced from the in-memory tee buffer above rather than a container runtime's log store, since nothing else captures this process's own stdout/stderr.""" try: tail_n = max(1, min(int(tail or 200), AGENT_LOG_MAX_LINES)) except (TypeError, ValueError): tail_n = 200 with _agent_log_lock: lines = list(_agent_log_buffer)[-tail_n:] combined = "\n".join(lines) if len(combined) > AGENT_LOG_MAX_BYTES: combined = combined[-AGENT_LOG_MAX_BYTES:] return {"success": True, "logs": combined, "tail": tail_n} # Unverified SSL Context for --insecure / self-signed / mismatched certificates ssl_unverified_context = ssl._create_unverified_context() # Track active/recent PTY session IDs to prevent duplicate concurrent threads active_pty_sessions = set() active_pty_lock = threading.Lock() # Track active Remote Desktop sessions active_desktop_sessions = set() active_desktop_lock = threading.Lock() # Below this, deflate's own framing overhead tends to cost more than a small message's own # repetition saves -- not worth spending the CPU on before even checking the result. _WS_DEFLATE_MIN_SIZE = 32 # Above this, payloads are almost always already-compressed pixel/media chunks -- desktop # framebuffer updates and container-port traffic relayed by bridge_socket_over_ws in up-to-64KB # chunks (VNC Tight/JPEG-encoded pixels, in particular) are high-entropy and essentially never # shrink under deflate. Below it are the small, genuinely-compressible cases (JSON control # messages, PTY output) this was originally sized for. Trying anyway on every large chunk was # pure wasted CPU on the hot desktop-streaming path with zero payoff. _WS_DEFLATE_MAX_SIZE = 16384 def _xor_mask(data, mask): """Applies a 4-byte WebSocket frame mask to `data` in bulk instead of per byte. A naive `for i in range(len(data)): data[i] ^= mask[i % 4]` loop runs at the Python interpreter's per-byte speed, which becomes the dominant cost once desktop/container-port streaming pushes tens of KB per frame through here continuously. Repeating the mask out to the payload length and XOR-ing via Python's arbitrary-precision integers instead does the whole buffer in one C-level operation. """ length = len(data) if length == 0: return b"" data = bytes(data) mask_full = (mask * (length // 4 + 1))[:length] return (int.from_bytes(data, "big") ^ int.from_bytes(mask_full, "big")).to_bytes(length, "big") class SimpleWebSocketClient: """Zero-dependency RFC 6455 WebSocket client using Python standard library.""" def __init__(self, url, insecure=False, host_header=None, headers=None, offer_deflate=True): parsed = urllib.parse.urlparse(url) self.scheme = parsed.scheme.lower() self.host = parsed.hostname or "127.0.0.1" self.port = parsed.port or (443 if self.scheme in ("wss", "https") else 80) self.path = (parsed.path or "/") + ("?" + parsed.query if parsed.query else "") self.insecure = insecure self.host_header = host_header or (f"{self.host}:{self.port}" if parsed.port and parsed.port not in (80, 443) else self.host) self.extra_headers = headers or {} # Whether to offer permessage-deflate in the handshake at all. # # False for the raw byte-stream relays (desktop/RFB, container ports). Those carry payloads # that are already compressed -- VNC Tight/JPEG pixels, or whatever a container port speaks -- # so deflate cannot shrink them and measurably makes small frames slightly larger, while # costing CPU on both ends of every frame. Declining the extension here is better than # skipping compression per frame: the server then has nothing to inflate either, so the saving # lands on both this host and the relay. # # The size-based skip in send_frame below was the earlier attempt at this, and its threshold # was the wrong test -- on a desktop session the small frames are already-compressed pixels # just as much as the large ones are. self.offer_deflate = offer_deflate # Set by connect() once the handshake response is known -- only true when the server # actually echoed the extension back, so a peer that does not support it (or a rolling # deploy where the two sides briefly differ) falls back to uncompressed with no error. self._deflate_enabled = False self._inflator = None self.sock = None def connect(self, timeout=15): raw_sock = socket.create_connection((self.host, self.port), timeout=timeout) try: raw_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) raw_sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) except Exception: pass if self.scheme in ("wss", "https"): ctx = ssl.create_default_context() if self.insecure: ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE # Set timeout specifically for TLS handshake raw_sock.settimeout(timeout) self.sock = ctx.wrap_socket(raw_sock, server_hostname=self.host) else: self.sock = raw_sock key = base64.b64encode(os.urandom(16)).decode("ascii") req_lines = [ f"GET {self.path} HTTP/1.1", f"Host: {self.host_header}", "Upgrade: websocket", "Connection: Upgrade", f"Sec-WebSocket-Key: {key}", "Sec-WebSocket-Version: 13", f"User-Agent: Applotus-Sentinel-Agent/{VERSION}", ] if self.offer_deflate: # client_no_context_takeover: this client compresses every message with a fresh, # stateless deflate context (see send_frame) rather than a sliding window carried # across messages, so it never depends on the order or presence of earlier messages -- # simpler and impossible to desync on a reconnect. Not requesting # server_no_context_takeover: this client's own decompressor is a single persistent # object kept for the life of the connection (see read_frame), which decodes correctly # either way, so there is nothing to gain by constraining what the server does. req_lines.append("Sec-WebSocket-Extensions: permessage-deflate; client_no_context_takeover") for k, v in self.extra_headers.items(): req_lines.append(f"{k}: {v}") req_str = "\r\n".join(req_lines) + "\r\n\r\n" self.sock.sendall(req_str.encode("utf-8")) res = b"" while b"\r\n\r\n" not in res: chunk = self.sock.recv(1024) if not chunk: break res += chunk header_str = res.decode("latin1", errors="replace") if "101 " not in header_str: raise ConnectionError(f"WebSocket upgrade rejected: {header_str[:120]}") # Only trust this when the server actually echoed the extension back -- a peer that does # not support it (or an older agent talking to this same code path) gets plain frames with # no error, exactly as before this was added. self._deflate_enabled = any( line.strip().lower().startswith("sec-websocket-extensions:") and "permessage-deflate" in line.lower() for line in header_str.split("\r\n")) self._inflator = zlib.decompressobj(-15) if self._deflate_enabled else None self.sock.setblocking(True) return True def send_frame(self, data, opcode=1): if self.sock is None: return if isinstance(data, str): data = data.encode("utf-8") rsv1 = 0 # permessage-deflate (RFC 7692). Never for control frames (close/ping/pong opcodes 8-10) # -- the spec forbids RSV1 there, and they are too small to benefit anyway. A fresh # compressor per call (no context takeover) rather than one kept across calls: see the # handshake comment in connect() for why that is deliberate, not a missed optimization. # Capped at _WS_DEFLATE_MAX_SIZE -- see its comment for why large chunks skip this # entirely instead of paying for a compressobj that will almost certainly lose the # size-check below anyway. if self._deflate_enabled and opcode in (1, 2) and _WS_DEFLATE_MIN_SIZE <= len(data) <= _WS_DEFLATE_MAX_SIZE: co = zlib.compressobj(6, zlib.DEFLATED, -15) compressed = co.compress(bytes(data)) + co.flush(zlib.Z_SYNC_FLUSH) if compressed.endswith(b"\x00\x00\xff\xff"): compressed = compressed[:-4] # Deflate's own framing overhead can make a small or already-dense payload larger, not # smaller -- only switch to the compressed form when it actually paid off. if len(compressed) < len(data): data = compressed rsv1 = 0x40 length = len(data) mask = os.urandom(4) if length < 126: hdr = struct.pack("!BB", 0x80 | rsv1 | opcode, 0x80 | length) elif length < 65536: hdr = struct.pack("!BBH", 0x80 | rsv1 | opcode, 0x80 | 126, length) else: hdr = struct.pack("!BBQ", 0x80 | rsv1 | opcode, 0x80 | 127, length) masked = _xor_mask(data, mask) try: self.sock.sendall(hdr + mask + masked) except Exception: pass def read_frame(self): try: hdr = self._recv_exact(2) if not hdr or len(hdr) < 2: return None, None b1, b2 = hdr[0], hdr[1] opcode = b1 & 0x0F rsv1 = bool(b1 & 0x40) has_mask = bool(b2 & 0x80) length = b2 & 0x7F if length == 126: raw_len = self._recv_exact(2) if not raw_len: return None, None length = struct.unpack("!H", raw_len)[0] elif length == 127: raw_len = self._recv_exact(8) if not raw_len: return None, None length = struct.unpack("!Q", raw_len)[0] mask = self._recv_exact(4) if has_mask else None payload = self._recv_exact(length) if length > 0 else b"" if mask and payload: payload = _xor_mask(payload, mask) if rsv1 and self._inflator is not None and opcode not in (8, 9, 10): # One decompressor kept for this connection's whole life (not recreated per # message): correct regardless of whether the server took the context-takeover # default or not, since a persistent inflator can always replay a stream produced # either way -- see the handshake comment in connect(). payload = self._inflator.decompress(bytes(payload) + b"\x00\x00\xff\xff") return opcode, payload except Exception: return None, None def _recv_exact(self, n): buf = bytearray() while len(buf) < n: chunk = self.sock.recv(n - len(buf)) if not chunk: break buf.extend(chunk) return bytes(buf) def close(self): if self.sock: try: self.send_frame(b"", opcode=8) except Exception: pass try: self.sock.close() except Exception: pass self.sock = None # The server authenticates agent WebSocket endpoints. An agent with an API key sends it; an agent # running in tenant-only mode (--tenant, no key) must still identify its workspace or the control # socket is closed and retried forever -- which surfaces as "agent has not connected yet" and a very # slow terminal connect, not as an error. # # Module-level rather than threaded through run_pty_session/run_desktop_session/ # run_container_port_session, none of which take a tenant today. _AGENT_TENANT_ID = "" # Set once the server issues a node-scoped key (or one is loaded from disk at startup). WS helpers # below fall back to it so a key adopted mid-run applies to sockets too, not just HTTP -- otherwise # a migrated agent would keep opening sockets with the tenant header the server no longer accepts. _AGENT_API_KEY = "" # Set when this node has been told (over the control WS, or via a heartbeat response) to go # dormant: a tenant removed it from their fleet, so it should check in rarely and quietly instead # of at its normal cadence. Module-level and in-memory on purpose -- a process restart (e.g. a # fresh agent run started by hand on the same host) starts clear, which is exactly what makes "run # a new agent on this host" reactivate immediately instead of needing an admin's wakeup. _dormant_mode = threading.Event() def _ws_auth_headers(api_key=None, host_header=None): h = {} effective_key = api_key or _AGENT_API_KEY if effective_key: h["x-api-key"] = effective_key elif _AGENT_TENANT_ID: h["x-tenant-id"] = _AGENT_TENANT_ID if host_header: h["Host"] = host_header return h import ntpath # Windows path semantics regardless of host, for the shell lookup below. def _pty_report(ws, msg, level="error"): """Writes a human-readable line into the browser's terminal. The terminal relay carries raw bytes straight into xterm.js, so an ANSI-coloured line is all that is needed -- no side channel, and it matches how the server already writes its own status lines. This exists because every failure in the terminal bridge used to be printed to the agent's own stdout and nowhere else. The server optimistically tells the browser "PTY bridge established! Spawning interactive shell..." as soon as the agent's socket connects -- before any shell has been started -- so a spawn failure left the user looking at a success message followed by silence, indistinguishable from a shell that started and simply said nothing. Diagnosing it required reading the agent's log, which needs an operator session and a dispatch the user may not have. Best-effort by design: if the socket is already gone there is nothing useful to do about it. """ colour = "31" if level == "error" else ("33" if level == "warn" else "36") prefix = "[!]" if level in ("error", "warn") else "[*]" try: if ws: ws.send_frame(f"\r\n\x1b[{colour}m{prefix} {msg}\x1b[0m\r\n".encode("utf-8"), opcode=1) except Exception: pass def _windows_appinstaller_dir(): r"""The directory holding the real winget.exe, or None. winget is an MSIX app alias registered per user, so SYSTEM's alias directory is empty and the name never resolves there. The package's own directory under WindowsApps does hold the binary and is readable, and the binary runs as SYSTEM -- verified against --version, `source list` and manifest resolution, all of which succeed. Naming the directory is therefore enough; nothing about winget needed working around. """ base = os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "WindowsApps") try: names = os.listdir(base) except Exception: return None best = None for name in names: if not name.startswith("Microsoft.DesktopAppInstaller_"): continue # The split/scale packages carry no executable; only the architecture one does. cand = os.path.join(base, name, "winget.exe") try: if os.path.isfile(cand): # Highest version wins when several are registered, which is the normal state after # an App Installer update. if best is None or name > best[0]: best = (name, os.path.join(base, name)) except Exception: pass return best[1] if best else None def _windows_interactive_path(): r"""PATH for an interactive Windows shell: the live machine PATH, winget's directory, then ours. os.environ is a snapshot taken when this process started. For a boot-start service that is whatever the machine looked like at boot, so a terminal opened today cannot see anything installed since -- which is how `winget`, `podman` and `java` were all unresolvable on a host that had all three. The existing process PATH is appended rather than discarded: whatever the service environment legitimately carries stays available, with the live values taking precedence. """ parts = [] def add(d): if d and d not in parts: parts.append(d) try: for d in _windows_machine_path_dirs(): add(d) except Exception: pass try: add(_windows_appinstaller_dir()) except Exception: pass for d in (os.environ.get("PATH") or "").split(os.pathsep): add(d.strip()) return os.pathsep.join(p for p in parts if p) def _windows_shell_candidates(which=None, isfile=None, sysroot=None): """Windows shells to try, in preference order, as argv lists. Factored out and parameterised purely so the ordering and the absolute-path fallbacks are testable off Windows -- `which` and `isfile` default to the real implementations. The previous logic was `shell_cmd = ["powershell.exe", ...]` and only switched to cmd.exe when `shutil.which("cmd.exe")` succeeded. So if PATH was thin enough that neither resolved -- entirely possible for a service started outside a normal login, which is how the agent usually runs -- it kept powershell.exe anyway, Popen raised FileNotFoundError, and the failure was printed to the agent's own stdout and nowhere else. Hence: try each candidate, and include absolute System32 paths so an empty PATH is survivable rather than fatal. """ which = which or shutil.which isfile = isfile or os.path.isfile if sysroot is None: sysroot = os.environ.get("SystemRoot") or os.environ.get("windir") or r"C:\Windows" candidates = [] def add(argv): # De-duplicate on the resolved executable, so a PATH hit and its absolute System32 twin collapse # to one entry. Deliberately NOT on basename: a PowerShell 7 install shadowing 5.1 lives at a # different path and is a legitimate second candidate, not a duplicate. if argv and argv[0] and not any(c[0].lower() == argv[0].lower() for c in candidates): candidates.append(argv) # PATH first: respects a deliberately-chosen shell. shutil.which returns a full path and only returns # one when the file is actually executable, so these are verified by the lookup itself. p = which("powershell.exe") if p: add([p, "-NoLogo"]) c = which("cmd.exe") if c: add([c]) # Then absolute locations, for the thin-PATH service case. Each is existence-checked as it is added, # rather than filtering the whole list afterwards -- an earlier version re-ran which() over the # finished list, which meant re-resolving paths that had already been resolved and made the PATH # entries depend on a second lookup succeeding for no reason. # # ntpath.join rather than os.path.join: these are Windows paths by definition, and os.path.join # follows the *host* platform, so building them anywhere else produced C:\Windows/System32/... with # forward slashes. Invisible in normal use -- this only runs on Windows, where the two agree -- but a # path that is correct only by coincidence of platform is worth not writing. ps_abs = ntpath.join(sysroot, "System32", "WindowsPowerShell", "v1.0", "powershell.exe") cmd_abs = ntpath.join(sysroot, "System32", "cmd.exe") if isfile(ps_abs): add([ps_abs, "-NoLogo"]) if isfile(cmd_abs): add([cmd_abs]) return candidates def run_pty_session(*args, **kwargs): """Wraps the terminal bridge in a wake lock, for the same reason the desktop one has one. A shell sitting at a prompt produces no input and no output, so it looks completely idle to the OS and an aggressive sleep setting will suspend the machine underneath a session someone still has open. The host then drops every socket at once and the terminal dies. """ session_id = kwargs.get("session_id") or (args[0] if args else "?") wake_lock = _start_session_wake_lock(session_id, "terminal") try: return _run_pty_session_bridge(*args, **kwargs) finally: _stop_session_wake_lock(wake_lock, session_id) def _run_pty_session_bridge(session_id, ws_url, insecure=False, host_header=None, api_key=None, container_id=None): """Spawns an interactive shell session (PTY on Linux/macOS, PowerShell/cmd.exe on Windows) and bridges it over WebSocket. When container_id is set, the shell runs inside that container via the runtime's own exec (podman/docker) instead of on this host directly -- this is the only way to offer a terminal for a container with no in-container agent of its own; the parent agent execs into it rather than spawning a shell locally.""" with active_pty_lock: if session_id in active_pty_sessions: # Already handled or currently active return active_pty_sessions.add(session_id) print(f"[{time.strftime('%X')}] [💻 TERMINAL SESSION] Initiating for session {session_id}" + (f" (exec into container {container_id[:12]})" if container_id else "") + "...", flush=True) ws = None try: headers = _ws_auth_headers(api_key) ws = SimpleWebSocketClient(ws_url, insecure=insecure, host_header=host_header, headers=headers) ws.connect(timeout=15) except Exception as e: print(f"[{time.strftime('%X')}] [-] Failed to connect Terminal WebSocket: {e}", flush=True) with active_pty_lock: active_pty_sessions.discard(session_id) return # ── Windows Console Fallback Bridge ─────────────────────────────────────── if pty is None or platform.system() == "Windows": proc = None try: if container_id: info = active_containers.get(container_id) or {} runtime_path = _runtime_path_for(info) or "podman" if not os.path.isabs(runtime_path): runtime_path = shutil.which(runtime_path) or runtime_path # -i and -t separately, never combined: confirmed live (podman 4.9.3) that "-it" as # one argv token makes podman exec exit 127 with no output at all, while the exact # same two flags passed as separate tokens work -- its flag parser doesn't bundle # short options the way getopt-style CLIs do. # # bash preference via `command -v`, never via a bare `exec bash || exec sh`: # confirmed live that when bash genuinely isn't installed (Alpine/busybox images, # Adminer among them), `exec bash` failing is NOT just a nonzero status the `||` # can catch -- some POSIX shells treat a failed exec as fatal and terminate the # whole shell right there with status 127, never reaching the fallback at all. # `command -v` only queries; it never risks that failure mode. shell_cmd = [runtime_path, "exec", "-i", "-t", container_id, "/bin/sh", "-c", "if command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi"] else: shell_candidates = _windows_shell_candidates() if not shell_candidates: _pty_report(ws, "No Windows shell could be found. Looked for powershell.exe and " "cmd.exe on PATH and under %%SystemRoot%%\\System32. The agent's " "PATH may be empty because it runs as a service outside a login " "session.") raise RuntimeError("no Windows shell available") shell_cmd = shell_candidates[0] env = os.environ.copy() env["TERM"] = "xterm-256color" env["COLORTERM"] = "truecolor" # A live PATH, not the one this process booted with. Without this the shell inherits a # snapshot from agent start, so nothing installed since resolves by name -- the reason # `winget` reported "not recognized" on a machine where it works, and `podman` and # `java` did the same. if platform.system() == "Windows": try: env["PATH"] = _windows_interactive_path() except Exception: pass # Each candidate is tried in turn: a shell can be present on disk and still fail to start # (blocked by policy, a broken PowerShell profile), and falling through to cmd.exe is far # better than reporting no shell at all. attempts = [] proc = None for argv in ([shell_cmd] if container_id else shell_candidates): try: proc = subprocess.Popen( argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0, creationflags=subprocess.CREATE_NO_WINDOW if hasattr(subprocess, 'CREATE_NO_WINDOW') else 0, env=env ) shell_cmd = argv break except Exception as spawn_err: attempts.append(f"{argv[0]}: {spawn_err}") proc = None if proc is None: # Previously this raised out of Popen into a handler that only printed to the agent's own # stdout, so the browser sat on the server's optimistic "spawning shell" line forever with # no way to learn why. Say it in the terminal instead. _pty_report(ws, "Could not start a shell. Tried: " + "; ".join(attempts)) raise RuntimeError("all shell candidates failed") print(f"[{time.strftime('%X')}] [✓ TERMINAL ACTIVE] Windows shell attached (PID {proc.pid}) to session {session_id}", flush=True) # Confirmation the browser can actually see. The server's "PTY bridge established" arrives # when this agent's socket connects, which is before any of the above has happened, so it is # not evidence a shell exists. This line is. _pty_report(ws, f"{os.path.basename(shell_cmd[0])} started (PID {proc.pid}).", level="info") stop_event = threading.Event() def _proc_reader(): try: while not stop_event.is_set(): data = proc.stdout.read(1024) if not data: break # Binary, not text (opcode=1's default): this is raw console output from a # process neither side controls, and a multi-byte UTF-8 character split # across a 1024-byte read boundary -- or any genuinely non-UTF-8 byte, a # legacy code page, a truecolor escape -- makes an invalid text frame. A # strict WebSocket peer (this server's own `ws` library among them) closes # the connection on exactly that, which is indistinguishable from the shell # itself crashing. See the matching fix on the POSIX PTY read loop below. ws.send_frame(data, opcode=2) except Exception: pass finally: stop_event.set() t_reader = threading.Thread(target=_proc_reader, daemon=True) t_reader.start() while not stop_event.is_set(): if proc.poll() is not None: break opcode, payload = ws.read_frame() if opcode is None or opcode == 8: break elif opcode == 9: ws.send_frame(payload, opcode=10) elif opcode in (1, 2) and payload: if payload.startswith(b'{"type":"resize"') or payload.startswith(b'{"type": "resize"'): pass else: try: proc.stdin.write(payload) proc.stdin.flush() except Exception: break except Exception as e: print(f"[{time.strftime('%X')}] [-] Windows terminal bridge error: {e}", flush=True) # The two raises above have already reported something specific; this covers everything else # so no failure in this branch can end as a silent blank terminal. if not str(e).startswith(("no Windows shell available", "all shell candidates failed")): _pty_report(ws, f"Terminal bridge error: {e}") finally: if proc: try: proc.terminate() proc.kill() except Exception: pass if ws: ws.close() with active_pty_lock: active_pty_sessions.discard(session_id) print(f"[{time.strftime('%X')}] [💻 TERMINAL CLOSED] Windows session {session_id} terminated", flush=True) return # ── POSIX PTY Implementation (Linux / macOS) ───────────────────────────── try: master, slave = pty.openpty() pid = os.fork() if pid == 0: os.close(master) os.setsid() os.dup2(slave, 0) os.dup2(slave, 1) os.dup2(slave, 2) os.close(slave) env = os.environ.copy() env["TERM"] = "xterm-256color" env["COLORTERM"] = "truecolor" env["LANG"] = env.get("LANG") or "en_US.UTF-8" try: if container_id: info = active_containers.get(container_id) or {} runtime_path = _runtime_path_for(info) or "podman" if not os.path.isabs(runtime_path): runtime_path = shutil.which(runtime_path) or runtime_path # See the identical comments in the Windows-fallback branch above: -i and -t # must be separate argv tokens (not combined as "-it"), and bash preference # goes through `command -v` rather than a bare failing `exec bash`. argv = [runtime_path, "exec", "-i", "-t", container_id, "/bin/sh", "-c", "if command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi"] os.execve(runtime_path, argv, env) else: shell = os.environ.get("SHELL") or "/bin/bash" if not os.path.exists(shell): shell = "/bin/sh" # Started as a LOGIN shell, by the argv[0] convention: a leading dash is how every # Bourne-family shell is told it is a login shell, and unlike -l it is understood by # sh/dash as well as bash and zsh. # # This matters because of where PATH actually comes from. A non-login interactive zsh # reads ~/.zshrc but not ~/.zprofile, and Homebrew's installer writes its # `eval "$(brew shellenv)"` line into ~/.zprofile -- so brew, and everything installed # by it, was simply absent from the remote terminal while being present in a normal # one. The same applies to bash and ~/.bash_profile. # # It also makes this terminal behave like an ssh login, which is what anyone opening a # shell here already expects. os.execve(shell, ["-" + os.path.basename(shell)], env) except Exception: os._exit(1) else: os.close(slave) if fcntl: fcntl.fcntl(master, fcntl.F_SETFL, os.O_NONBLOCK) print(f"[{time.strftime('%X')}] [✓ PTY ACTIVE] PID {pid} attached to session {session_id}", flush=True) def _log_pty_exit(): # Reachable from the EOF-on-read and read-exception branches below, as well as # the waitpid-detected-exit branch further down -- each is a different way this # loop can end, and every one of them was a silent "PTY CLOSED" with no explanation # before this, which is exactly the failure mode that made the instant-exit bug # (podman exec dying right after launch) impossible to diagnose. try: res = os.waitpid(pid, os.WNOHANG) except ChildProcessError: res = None if res and res[0] != 0: exit_code = os.WEXITSTATUS(res[1]) if os.WIFEXITED(res[1]) else None signal_num = os.WTERMSIG(res[1]) if os.WIFSIGNALED(res[1]) else None print(f"[{time.strftime('%X')}] [💻 PTY EXITED] session {session_id} " f"exit_code={exit_code} signal={signal_num}", flush=True) try: while True: r, _, _ = select.select([master, ws.sock], [], [], 1.0) if master in r: try: data = os.read(master, 4096) if data: # Binary, not text: a multi-byte UTF-8 character split across this # 4096-byte read boundary -- or any raw byte a TUI's ANSI/truecolor # output happens to emit -- makes an invalid WebSocket text frame, # and this server's own `ws` library closes the connection on that # per spec. The browser's onmessage already handles a Blob (binary) # frame via .text() -- a lossy decode that degrades to a stray � # instead of killing the session, which is what text frames did not # survive. Confirmed live: Hermes's TUI (box-drawing, a ✦ tip glyph, # a spinner) reproduced exactly this disconnect. ws.send_frame(data, opcode=2) else: _log_pty_exit() break except (BlockingIOError, InterruptedError): pass except Exception as e: print(f"[{time.strftime('%X')}] [💻 PTY READ ERROR] session {session_id}: {e}", flush=True) _log_pty_exit() break # Checked after the read attempt above, not before: checking first raced ahead # of draining the pty, so a process that wrote something (an error explaining # why it was about to exit, say) right before dying routinely lost that output # to this exact race -- confirmed live against a podman exec that exited # instantly with nothing ever reaching the browser or this log to explain why. res = os.waitpid(pid, os.WNOHANG) if res and res[0] != 0: # One more drain: the exec'd process's last write can still be sitting in # the pty's buffer even after waitpid confirms it's gone. try: while True: data = os.read(master, 4096) if not data: break ws.send_frame(data, opcode=2) # binary -- see the matching read above except (BlockingIOError, InterruptedError, OSError): pass exit_code = os.WEXITSTATUS(res[1]) if os.WIFEXITED(res[1]) else None signal_num = os.WTERMSIG(res[1]) if os.WIFSIGNALED(res[1]) else None print(f"[{time.strftime('%X')}] [💻 PTY EXITED] session {session_id} " f"exit_code={exit_code} signal={signal_num}", flush=True) break if ws.sock in r: opcode, payload = ws.read_frame() if opcode is None or opcode == 8: break elif opcode == 9: ws.send_frame(payload, opcode=10) elif opcode in (1, 2) and payload: if payload.startswith(b'{"type":"resize"') or payload.startswith(b'{"type": "resize"'): try: msg = json.loads(payload.decode("utf-8")) cols = int(msg.get("cols", 80)) rows = int(msg.get("rows", 24)) if termios and fcntl: fcntl.ioctl(master, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0)) except Exception: pass else: os.write(master, payload) finally: try: os.close(master) except Exception: pass try: os.kill(pid, 15) time.sleep(0.15) os.kill(pid, 9) os.waitpid(pid, 0) except Exception: pass ws.close() print(f"[{time.strftime('%X')}] [💻 PTY CLOSED] Session {session_id} terminated", flush=True) except Exception as err: print(f"[{time.strftime('%X')}] [-] PTY session error: {err}", flush=True) if ws: ws.close() finally: with active_pty_lock: active_pty_sessions.discard(session_id) def bridge_socket_over_ws(sock, ws, label="SOCKET", eof_hint=None): """Generic raw-byte relay between a local TCP socket and a WebSocket frame stream. Protocol-agnostic — used by run_desktop_session (VNC/RFB) and run_container_port_session (arbitrary container-exposed TCP ports) alike. Caller owns socket/ws lifecycle (blocking mode, close-on-exit); this function only pumps bytes until either side closes or errors.""" raw_chunk_view = bytearray(65536) while True: r, _, _ = select.select([sock, ws.sock], [], [], 1.0) if sock in r: try: nbytes = sock.recv_into(raw_chunk_view) if nbytes > 0: ws.send_frame(memoryview(raw_chunk_view)[:nbytes], opcode=2) else: print(f"[{time.strftime('%X')}] [!] {label}: local socket closed connection (EOF).{(' ' + eof_hint) if eof_hint else ''}", flush=True) break except (BlockingIOError, InterruptedError): pass except Exception as _e: print(f"[{time.strftime('%X')}] [!] {label}: socket read exception: {_e}", flush=True) break if ws.sock in r: opcode, payload = ws.read_frame() if opcode is None or opcode == 8: print(f"[{time.strftime('%X')}] [!] {label}: WebSocket client disconnected (opcode={opcode})", flush=True) break elif opcode == 9: ws.send_frame(payload, opcode=10) elif opcode in (1, 2) and payload: try: sock.sendall(payload) except Exception as _e: print(f"[{time.strftime('%X')}] [!] {label}: socket send exception: {_e}", flush=True) break def _find_xauthority_for_display(display): """Locate the Xauthority cookie for the X server actually serving `display`, by finding that server's process and using its owning user's home directory -- not a fixed/guessed path, since this agent runs on arbitrary hosts and is frequently root while the display belongs to whichever user is actually logged in (root has no cookie of its own for that display).""" try: disp_num = display.lstrip(":").split(".")[0] except Exception: return None try: import pwd except ImportError: return None # Windows/macOS have no notion of this -- has_x11vnc_helper is Linux-only anyway try: for pid_dir in os.listdir("/proc"): if not pid_dir.isdigit(): continue try: with open(f"/proc/{pid_dir}/cmdline", "rb") as f: argv = [a.decode("utf-8", "ignore") for a in f.read().split(b"\x00") if a] except Exception: continue if not argv or not any(x in argv[0] for x in ("Xorg", "Xvfb", "Xtightvnc")): continue if f":{disp_num}" not in argv: continue st = os.stat(f"/proc/{pid_dir}") home = pwd.getpwuid(st.st_uid).pw_dir candidate = os.path.join(home, ".Xauthority") if os.path.isfile(candidate): return candidate except Exception: pass return None # ── Matching the console resolution to the viewer's panel (Windows) ────────────────────────────── # noVNC asks the server to resize its framebuffer to the viewer's panel (resizeSession), which is the # clean way to avoid black bars. TightVNC Server on Windows does not implement SetDesktopSize and # structurally cannot -- it mirrors whatever the real Windows display is set to. So the request is # silently dropped, noVNC falls back to aspect-preserving scaling, and a 4:3 console in a widescreen # panel wastes about a fifth of the width. # # The remaining option is to change the console resolution itself, which is what this does. Only the # aspect ratio genuinely has to match to kill the bars; the absolute size matters just for sharpness, # and a framebuffer somewhat larger than the panel is fine because noVNC scales it down cleanly. # # Applied dynamically (flags=0), NOT with CDS_UPDATEREGISTRY: the change should last for the session # and not outlive a reboot. Every session re-applies it anyway, so persisting it would buy nothing and # would quietly redefine the machine's configured resolution. DM_BITSPERPEL = 0x00040000 DM_PELSWIDTH = 0x00080000 DM_PELSHEIGHT = 0x00100000 ENUM_CURRENT_SETTINGS = -1 DISP_CHANGE_SUCCESSFUL = 0 # Requested sizes are already bounds-checked server-side; this is the agent's own floor so a bogus # value can never ask Windows for something unusable. MIN_DISPLAY_EDGE_PX = 320 # Off by default. This is the one part of the resolution-matching feature that was never verifiable # before shipping -- it needs a real Windows host, which the development environment does not have -- # and it runs in the desktop startup path, where the failure mode is not an error but a session that # connects cleanly and then shows nothing usable. A console mode switch reconfigures the graphics # stack moments before TightVNC's framebuffer is read, so a stale or wrong-geometry framebuffer # presents exactly as "connected, no picture", with nothing logged to distinguish it. # # That is precisely what was reported: server logs show the desktop handshake completing in ~0.5s # every time while the viewer stayed unusable. Rather than leave unverified code in the default path # of a feature people rely on, it is opt-in until someone confirms it on real hardware: # # APPLOTUS_MATCH_DISPLAY_RESOLUTION=1 (or SENTINEL_MATCH_DISPLAY_RESOLUTION=1) # # Everything else about the feature stays wired up -- the viewer still sends its panel size, the # server still forwards it, and _pick_display_mode is still covered by tests. Only the syscall that # actually changes the console is gated. # # An env var turned out to be the wrong control surface, though: setting it needs shell access to the # node, and the whole point of the fleet is that nodes are reached through the server -- the one host # that actually needed this flag flipped was a Windows box nobody had a shell on. So the server can # also send it per session in the start_desktop payload, which makes enabling it a deploy rather than # an expedition. # # A locally-set env var still wins either way, truthy OR falsey: whoever is on the box keeps the final # say, including forcing it off while the server is asking for it. Absent locally, the server's flag # decides. Absent both, off. # # Read at call time rather than at import: _cfg is defined much further down this file, so evaluating # it up here would be a NameError the moment the module loads. def _match_display_resolution_enabled(server_flag=None): raw = _cfg("MATCH_DISPLAY_RESOLUTION") if raw != "": return raw.strip().lower() in ("1", "true", "yes", "on") return bool(server_flag) def _pick_display_mode(modes, req_w, req_h): """Chooses the supported display mode that best fits a viewer panel of req_w x req_h. `modes` is a list of (width, height) the adapter reports as supported. Pure and side-effect free so the selection can be tested anywhere, not just on Windows. Ranked on three keys, in this order: 1. Whether the mode covers the panel in both dimensions. A mode at least as large as the panel is scaled DOWN by the viewer, which stays sharp; a smaller one is scaled up and text visibly softens. This outranks aspect ratio deliberately, and the ordering is not obvious -- for a 1480x870 panel, 1280x768 is a closer shape (5:3 = 1.667 vs 16:9 = 1.778) and wastes only 2.0% of the width against 1600x900's 4.3% of the height. But it has to be upscaled 1.16x to get there. A few percent of black bar is a better trade than a blurry desktop, so covering wins. 2. Aspect-ratio error, which is what actually removes the bars. Rounded before comparing so modes of the same shape (1280x720 vs 1920x1080) tie and are separated by size rather than by floating-point noise. 3. Area -- smallest among the covering modes, since extra pixels beyond the panel only cost bandwidth and CPU for detail the viewer scales away; largest among the non-covering ones, for when the panel is bigger than anything the adapter offers and upscaling is unavoidable. """ if not req_w or not req_h or req_w <= 0 or req_h <= 0: return None usable = [ (int(w), int(h)) for (w, h) in modes if int(w) >= MIN_DISPLAY_EDGE_PX and int(h) >= MIN_DISPLAY_EDGE_PX ] if not usable: return None target_aspect = float(req_w) / float(req_h) def rank(mode): w, h = mode aspect_err = round(abs((float(w) / float(h)) - target_aspect), 3) covers = (w >= req_w and h >= req_h) area = w * h return (0 if covers else 1, aspect_err, area if covers else -area) return min(usable, key=rank) DISPLAY_DEVICE_ACTIVE = 0x00000001 DISPLAY_DEVICE_PRIMARY_DEVICE = 0x00000004 def _windows_session_id(): """The Windows session this process runs in, or None off Windows / on failure. Session 0 is the isolated services session. It matters here because every display call acts on the CALLING session's desktop: a boot-start agent lives in session 0, whose display is a synthetic 1024x768 device, so a resolution change made there is invisible to the person using the machine while still reporting success. """ if platform.system() != "Windows" or ctypes is None: return None try: sid = ctypes.c_ulong(0) if ctypes.windll.kernel32.ProcessIdToSessionId(os.getpid(), ctypes.byref(sid)): return int(sid.value) except Exception: pass return None def _primary_display_device(): r"""The primary adapter's device name, e.g. "\\.\DISPLAY1", or None if it cannot be resolved. Exists because passing None as the device -- "the default display" -- is not reliable. On a real 5K Windows host both EnumDisplaySettingsW(None, ...) and ChangeDisplaySettingsExW(None, ...) fail, while the same calls against the explicit device name succeed and return the true mode. Naming the device is therefore not a tidiness change; it is the difference between the feature working and silently doing nothing. Prefers the adapter flagged PRIMARY, falling back to the first ACTIVE one so a machine that flags nothing primary still gets a usable answer. """ if platform.system() != "Windows" or ctypes is None: return None try: class DISPLAY_DEVICEW(ctypes.Structure): _fields_ = [ ("cb", ctypes.c_ulong), ("DeviceName", ctypes.c_wchar * 32), ("DeviceString", ctypes.c_wchar * 128), ("StateFlags", ctypes.c_ulong), ("DeviceID", ctypes.c_wchar * 128), ("DeviceKey", ctypes.c_wchar * 128), ] user32 = ctypes.windll.user32 first_active = None for i in range(16): dd = DISPLAY_DEVICEW() dd.cb = ctypes.sizeof(DISPLAY_DEVICEW) if not user32.EnumDisplayDevicesW(None, i, ctypes.byref(dd), 0): break if not (dd.StateFlags & DISPLAY_DEVICE_ACTIVE): continue if dd.StateFlags & DISPLAY_DEVICE_PRIMARY_DEVICE: return dd.DeviceName if first_active is None: first_active = dd.DeviceName return first_active except Exception: return None # PowerShell run inside the console session to change the mode. Kept as a template rather than # assembled inline so the quoting is reviewable: this string crosses python -> a file -> schtasks -> # powershell, and every one of those layers reinterprets quotes. _CONSOLE_SETRES_PS = r""" $log = '__LOG__' function L($m) { $m | Out-File $log -Append -Encoding utf8 } "" | Out-File $log -Encoding utf8 # Written before Add-Type, which compiles C# at runtime and on a cold assembly cache can take longer # than the caller is willing to wait. With this line present a timeout means "started, unfinished"; # without it the caller cannot tell that from "never ran", and once reported a definite failure for a # change that had in fact applied. L 'STARTED' Add-Type -AssemblyName System.Windows.Forms $src = @' using System; using System.Runtime.InteropServices; public class Disp { [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] public struct DEVMODE { [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] public string dmDeviceName; public ushort dmSpecVersion, dmDriverVersion, dmSize, dmDriverExtra; public uint dmFields; public int dmPositionX, dmPositionY; public uint dmDisplayOrientation, dmDisplayFixedOutput; public short dmColor, dmDuplex, dmYResolution, dmTTOption, dmCollate; [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] public string dmFormName; public ushort dmLogPixels; public uint dmBitsPerPel, dmPelsWidth, dmPelsHeight, dmDisplayFlags, dmDisplayFrequency; public uint dmICMMethod, dmICMIntent, dmMediaType, dmDitherType; public uint dmReserved1, dmReserved2, dmPanningWidth, dmPanningHeight; } [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] public static extern int EnumDisplaySettingsW(string dev, int mode, ref DEVMODE dm); [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] public static extern int ChangeDisplaySettingsExW(string dev, ref DEVMODE dm, IntPtr hwnd, uint flags, IntPtr p); } '@ Add-Type -TypeDefinition $src $dev = [System.Windows.Forms.Screen]::PrimaryScreen.DeviceName $sz = [uint16][System.Runtime.InteropServices.Marshal]::SizeOf([type]('Disp+DEVMODE')) $dm = New-Object Disp+DEVMODE $dm.dmSize = $sz if ([Disp]::EnumDisplaySettingsW($dev, -1, [ref]$dm) -eq 0) { L 'ERR enum-failed'; exit 1 } L ('BEFORE ' + $dm.dmPelsWidth + 'x' + $dm.dmPelsHeight) if ($dm.dmPelsWidth -eq __W__ -and $dm.dmPelsHeight -eq __H__) { L ('OK __W__x__H__'); exit 0 } $dm.dmPelsWidth = __W__ $dm.dmPelsHeight = __H__ $dm.dmFields = 0x00080000 -bor 0x00100000 $rc = [Disp]::ChangeDisplaySettingsExW($dev, [ref]$dm, [IntPtr]::Zero, 0, [IntPtr]::Zero) if ($rc -ne 0) { L ('ERR rc=' + $rc); exit 1 } Start-Sleep -Seconds 1 $after = New-Object Disp+DEVMODE $after.dmSize = $sz [void][Disp]::EnumDisplaySettingsW($dev, -1, [ref]$after) L ('OK ' + $after.dmPelsWidth + 'x' + $after.dmPelsHeight) """ def _console_session_user(): """The user logged on at the physical console, e.g. "HOST\\user", or None.""" out = _run_quiet(["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", "(Get-CimInstance Win32_ComputerSystem).UserName"], timeout=25) name = (out or "").strip() return name or None def _console_log_pending(text): """True when a console-session script has started but not reached a terminal line. The scripts write STARTED first and then OK/ERR. Between those two a timeout means the work may still be in flight, which must not be reported as a definite failure -- that is how a successful resolution change came back as "the console session did not accept the change". """ seen_start = False for line in (text or "").splitlines(): line = line.strip().lstrip("\ufeff") if line.startswith("OK ") or line.startswith("ERR") or line == "OK": return False if line == "STARTED": seen_start = True return seen_start def _run_in_console_session(ps_body, tag, wait_s=20): """Runs PowerShell inside the interactive session and returns what it logged, or None. Exists because this agent is normally a SYSTEM service in session 0, where every display call reaches a synthetic 1024x768 desktop rather than the panel the user is looking at. A one-shot scheduled task created with /it runs as the logged-on user in their session without storing a password, which is the only route verified to work on this hardware. The task runs detached, so there is no exit code to read: the script writes a log and this polls for it. Shared by the read and the write paths -- they had identical copies of this sequence, and it is quoting-sensitive enough that two copies is one too many. """ user = _console_session_user() if not user: return None base = os.environ.get("ProgramData", r"C:\ProgramData") ps_path = os.path.join(base, "applotus_%s.ps1" % tag) log_path = os.path.join(base, "applotus_%s.log" % tag) task = "Applotus_%s" % tag try: with io.open(ps_path, "w", encoding="utf-8") as fh: fh.write(ps_body.replace("__LOG__", log_path)) try: os.remove(log_path) except Exception: pass _run_quiet(["schtasks", "/create", "/tn", task, "/tr", 'powershell -NoProfile -ExecutionPolicy Bypass -File "%s"' % ps_path, "/sc", "once", "/st", "23:59", "/ru", user, "/it", "/f"], timeout=30) _run_quiet(["schtasks", "/run", "/tn", task], timeout=30) for _ in range(int(wait_s)): time.sleep(1.0) try: # utf-8-sig, not utf-8: PowerShell 5.1's `Out-File -Encoding utf8` writes a BOM, and # it lands on the first line -- which is the one line every one of these scripts puts # its most important value on. It cost the current display mode on a live node. with io.open(log_path, encoding="utf-8-sig", errors="replace") as fh: text = fh.read() except Exception: continue # STARTED alone is not an answer: the script writes it before its slow work, precisely # so a timeout can be described honestly. Keep waiting for a terminal line. if text.strip() and not _console_log_pending(text): return text # Budget exhausted. Return the partial log rather than None: "STARTED and nothing else" is a # materially different situation from "no log at all", and only the caller can phrase it. try: with io.open(log_path, encoding="utf-8-sig", errors="replace") as fh: return fh.read() or None except Exception: return None except Exception as e: print("[%s] [!] console-session %s could not be dispatched: %s" % (time.strftime("%X"), tag, e), flush=True) return None finally: _run_quiet(["schtasks", "/delete", "/tn", task, "/f"], timeout=20) for p in (ps_path, log_path): try: os.remove(p) except Exception: pass # Enumerates modes from inside the console session. Only the 32bpp sizes matter here; the viewer # chooses among sizes and the agent never changes colour depth. _CONSOLE_ENUM_PS = r""" $log = '__LOG__' Add-Type -AssemblyName System.Windows.Forms $src = @' using System; using System.Runtime.InteropServices; public class DispE { [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] public struct DEVMODE { [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] public string dmDeviceName; public ushort dmSpecVersion, dmDriverVersion, dmSize, dmDriverExtra; public uint dmFields; public int dmPositionX, dmPositionY; public uint dmDisplayOrientation, dmDisplayFixedOutput; public short dmColor, dmDuplex, dmYResolution, dmTTOption, dmCollate; [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] public string dmFormName; public ushort dmLogPixels; public uint dmBitsPerPel, dmPelsWidth, dmPelsHeight, dmDisplayFlags, dmDisplayFrequency; public uint dmICMMethod, dmICMIntent, dmMediaType, dmDitherType; public uint dmReserved1, dmReserved2, dmPanningWidth, dmPanningHeight; } [DllImport("user32.dll", CharSet = CharSet.Unicode, SetLastError = true)] public static extern int EnumDisplaySettingsW(string dev, int mode, ref DEVMODE dm); } '@ Add-Type -TypeDefinition $src $dev = [System.Windows.Forms.Screen]::PrimaryScreen.DeviceName $sz = [uint16][System.Runtime.InteropServices.Marshal]::SizeOf([type]('DispE+DEVMODE')) $cur = New-Object DispE+DEVMODE $cur.dmSize = $sz $out = @() if ([DispE]::EnumDisplaySettingsW($dev, -1, [ref]$cur) -ne 0) { $out += ('CURRENT ' + $cur.dmPelsWidth + 'x' + $cur.dmPelsHeight) } $seen = @{} for ($i = 0; $i -lt 400; $i++) { $dm = New-Object DispE+DEVMODE $dm.dmSize = $sz if ([DispE]::EnumDisplaySettingsW($dev, $i, [ref]$dm) -eq 0) { break } if ($dm.dmBitsPerPel -ne 32) { continue } $k = '' + $dm.dmPelsWidth + 'x' + $dm.dmPelsHeight if (-not $seen.ContainsKey($k)) { $seen[$k] = $true; $out += ('MODE ' + $k) } } if ($out.Count -eq 0) { $out += 'ERR no-modes' } $out -join [Environment]::NewLine | Out-File $log -Encoding utf8 """ def _parse_console_modes(text): """(modes, current) from the console enumerator's log. Tolerant of partial output.""" modes, current = [], None for line in (text or "").splitlines(): line = line.strip().lstrip("\ufeff") try: if line.startswith("CURRENT "): w, h = line[8:].split("x", 1) current = (int(w), int(h)) elif line.startswith("MODE "): w, h = line[5:].split("x", 1) modes.append((int(w), int(h))) except Exception: continue return modes, current def _parse_xrandr(text): """(modes, current, reason) from `xrandr -q` output. `current` comes from the Screen line's "current W x H", not from an output's starred mode: that is the framebuffer size, which is what VNC captures and what decides bytes per frame. Modes are taken from the FIRST connected output only. Concatenating every output's modes on a multi-head host would offer resolutions that cannot both be in effect. """ current = None m = re.search(r"current\s+(\d+)\s*x\s*(\d+)", text or "") if m: current = (int(m.group(1)), int(m.group(2))) modes = [] in_output = False for line in (text or "").splitlines(): if re.match(r"^\S+\s+connected", line): if in_output: break in_output = True continue if re.match(r"^\S+\s+disconnected", line): if in_output: break continue if in_output: mm = re.match(r"^\s+(\d+)x(\d+)\s", line) if mm: modes.append((int(mm.group(1)), int(mm.group(2)))) if not modes and current: # A server with no mode list still has a size worth reporting. modes = [current] if not modes: return [], None, "xrandr reported no usable modes." return modes, current, None def _linux_display_modes(preferred_display=None): """(modes, current, reason) for whichever X display actually answers. Never raises. Tries displays rather than choosing one. Measured on a real host: :0 existed but belonged to the login greeter (cosmic-greeter) with no cookie the agent could use, while the live session was on :10 and owned by the agent's own user. Preferring :0 therefore failed on a machine that had a perfectly readable desktop. That is the same trap documented on _linux_x_displays, approached from the opposite side -- it warns against preferring :10, because auto-selection once picked an xrdp server and showed a real but idle desktop. Both preferences are guesses about which display a human is using. A display that answers is a fact, so the configured one is tried first and the rest are tried in turn. """ xrandr = shutil.which("xrandr") if not xrandr: return [], None, "xrandr is not installed on this host." candidates = [] pref = (preferred_display or "").strip() if pref: candidates.append(pref) try: for entry in (_linux_x_displays_cached() or []): disp = (entry or {}).get("display") if disp and disp not in candidates: candidates.append(disp) except Exception: pass if not candidates: # A headless server with no X at all, which is a different situation from one that is # present but unreadable, and the caller should be able to tell them apart. return [], None, "No X display was found on this host." errors = [] for display in candidates: env = dict(os.environ) env["DISPLAY"] = display # Per display, not per host: cookies are display-specific, which is exactly why the agent's # own ~/.Xauthority did not work for the greeter's :0. try: xauth = _find_xauthority_for_display(display) except Exception: xauth = None if xauth: env["XAUTHORITY"] = xauth try: r = subprocess.run([xrandr, "-q"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=12, env=env) except Exception as e: errors.append("%s: %s" % (display, e)) continue if r.returncode != 0 or not r.stdout: # xrandr's own first line is far more useful than a generic failure -- it distinguishes # "Authorization required" from "Can't open display" from a crashed server. first = [ln for ln in (r.stderr or "").splitlines() if ln.strip()] errors.append("%s: %s" % (display, first[0].strip() if first else "query failed")) continue modes, current, reason = _parse_xrandr(r.stdout) if modes: return modes, current, reason errors.append("%s: %s" % (display, reason or "no modes reported")) return [], None, "Could not read any X display (%s)." % "; ".join(errors[:3]) def _display_mode_writable(): """Whether this platform can CHANGE the resolution, as opposed to only report it. Separate from readability on purpose. Linux is readable through xrandr but applying is not implemented, and conflating the two is what made a readable node show a dead control. """ return platform.system() in ("Windows", "Darwin") def _display_modes_for_platform(preferred_display=None): """(modes, current, reason) for this host. reason is None when the list is usable. Every platform answers from a different API, and on Windows the answer depends on which session is asking -- a service agent in session 0 sees only a synthetic desktop, so the question has to be asked from the console session instead. Linux is readable but not settable; see _display_mode_writable. preferred_display only means anything on Linux, where a host can run several X servers. """ system = platform.system() if system == "Darwin": modes, current = _enumerate_darwin_display_modes() if not modes: return [], None, "The display did not report any modes." return modes, current, None if system == "Windows": if _windows_session_id() == 0: text = _run_in_console_session(_CONSOLE_ENUM_PS, "enum", wait_s=25) modes, current = _parse_console_modes(text) if not modes: return [], None, ("The agent runs as a service in session 0 and the console session " "did not report its modes.") return modes, current, None modes, current = _enumerate_windows_display_modes() if not modes: return [], None, "The display did not report any modes." return modes, current, None if system == "Linux": return _linux_display_modes(preferred_display) return [], None, "Resolution control is not available on %s targets." % (system or "this") def handle_get_display_modes(preferred_display=None): """{available, current, modes, reason} for the viewer's resolution control. Never raises. Read-only. The dropdown was previously populated from a hard-coded ladder and a stored preference, so it could not show the resolution actually in force -- which is the one thing a person looking at the control wants to know. """ try: modes, current, reason = _display_modes_for_platform(preferred_display) except Exception as e: return {"available": False, "readOnly": True, "modes": [], "current": None, "reason": "Agent error: %s: %s" % (type(e).__name__, e)} if reason: return {"available": False, "readOnly": True, "modes": [], "current": None, "reason": reason} # Largest first: the list reads as a ladder down from native, which is the direction anyone # using this is travelling. uniq = sorted({(int(w), int(h)) for w, h in modes}, key=lambda m: -(m[0] * m[1])) writable = _display_mode_writable() return { "available": True, # Readable but not settable is a real state, and conflating it with unavailable is what made # every Linux node show a dead control instead of its actual resolution. "readOnly": not writable, "current": list(current) if current else None, "modes": [[w, h] for w, h in uniq], "reason": None if writable else ("Showing this machine's current resolution. Changing it is not implemented for %s " "targets yet." % platform.system()), } def _set_display_mode_in_console_session(req_w, req_h): """Applies a mode change from inside the interactive session. (w, h) applied, or None. The change is dynamic (flags 0) and deliberately NOT written to the registry, so a reboot or a fresh logon restores the previous mode by itself. That matters: the machine's only remote access is the desktop being altered, so a bad outcome has to self-heal rather than need a visit. """ body = (_CONSOLE_SETRES_PS .replace("__W__", str(int(req_w))) .replace("__H__", str(int(req_h)))) # 45s, not 20: the script's Add-Type compiles C# at runtime and a cold assembly cache alone can # exceed 20s on a loaded host. The old budget expired while the change was still being made. text = _run_in_console_session(body, "setres", wait_s=45) for line in (text or "").splitlines(): line = line.strip() if line.startswith("OK "): try: w, h = line[3:].split("x", 1) return (int(w), int(h)) except Exception: return None if line.startswith("ERR"): print("[%s] [!] console display change failed: %s" % (time.strftime("%X"), line), flush=True) return None # Started and never finished within the budget. Deliberately distinguished from "never ran": # saying it definitely failed when it may have applied is what taught a user not to trust either # answer. if _console_log_pending(text): return "PENDING" return None def _largest_mode_within(modes, box_w, box_h): """The biggest enumerated mode that fits inside the box, or None. Picking from the adapter's own list rather than computing a size: a computed 2880x1800 that the display does not offer would pass the clamp and then be refused by the "does not offer" check a few lines later, reported as a failure. The list is the only source of modes that will actually apply. """ best = None for m in (modes or []): try: mw, mh = int(m[0]), int(m[1]) except Exception: continue if mw <= box_w and mh <= box_h: if best is None or (mw * mh) > (best[0] * best[1]): best = (mw, mh) return best def handle_set_display_mode(req_w, req_h, max_pixels=None, max_width=None, max_height=None): """Applies a requested console resolution and says what happened. Never raises. The reply always carries either the mode now in effect or the reason it could not change, because the failure this feature is most prone to is looking like it worked. """ # Not Windows-only. The agent has had a verified macOS path all along # (_enumerate_darwin_display_modes / _apply_darwin_display_mode); claiming otherwise put an inert # control in the header on every Mac. Platforms with no implementation are named individually. if not _display_mode_writable(): # The same predicate the read path reports, so the control's state and this refusal can # never disagree about what the platform supports. return {"applied": False, "reason": "Changing resolution is not implemented for %s targets yet." % platform.system()} try: w, h = int(req_w), int(req_h) except Exception: return {"applied": False, "reason": "width and height must be integers."} if w < 320 or h < 240 or w > 7680 or h > 4320: return {"applied": False, "reason": "Requested size is outside the supported range."} # Platform-aware, and on a session-0 Windows agent this asks the console session. The old call # went straight to _enumerate_windows_display_modes, which returns ([], None) there -- so every # pre-flight check silently passed and an unsupported mode was blamed on session 0. modes, current, _enum_reason = _display_modes_for_platform() # A hard ceiling on the box, distinct from the area cap below: over-size is brought DOWN to fit # rather than refused. Refusing would leave a 4K or 5K console at its native size -- the size # the policy exists to avoid carrying -- so a refusal here achieves the opposite of the intent. # # Placed after enumeration because it prefers a real mode over a computed one, and before the # area cap because clamping puts the request under that cap by construction. clamped_from = None try: box_w = int(max_width) if max_width else 0 box_h = int(max_height) if max_height else 0 except Exception: box_w = box_h = 0 if box_w > 0 and box_h > 0 and (w > box_w or h > box_h): clamped_from = (w, h) fitted = _largest_mode_within(modes, box_w, box_h) if fitted: w, h = fitted else: # No enumerated modes, or none small enough. Scale preserving aspect ratio, which is # what the framebuffer path does, and round to even for the same encoder reason. scale = min(box_w / float(w), box_h / float(h)) w = max(320, int(w * scale) // 2 * 2) h = max(240, int(h * scale) // 2 * 2) # The cap applies to RAISING, which is why it is enforced here rather than in the route: this is # the only side that knows the current mode without another round trip, and the limit is defined # relative to it. A machine already above the budget can still have its current mode re-applied; # what is refused is asking a link to carry more pixels than the policy allows. try: cap = int(max_pixels) if max_pixels else 0 except Exception: cap = 0 if cap > 0 and (w * h) > cap: cur_px = (current[0] * current[1]) if current else 0 if (w * h) > cur_px: return {"applied": False, "width": current[0] if current else None, "height": current[1] if current else None, "reason": "%dx%d is %.1f megapixels, above the %.1f megapixel limit for raising a " "remote display. Every extra pixel costs proportionally more per frame." % (w, h, (w * h) / 1e6, cap / 1e6)} def _ok(res): """Marks a successful reply with the clamp, if one happened. Applied to every success path rather than to the reason string of one: a caller that asked for 3840x2160 and received 3200x1800 has to be able to see that, and "applied: true" with the requested size nowhere in the reply is how this feature fails invisibly. """ if clamped_from: res["clamped"] = True res["requestedWidth"], res["requestedHeight"] = clamped_from res["maxWidth"], res["maxHeight"] = box_w, box_h res["reason"] = ("%s (%dx%d exceeds the %dx%d limit, so it was reduced to %dx%d.)" % (res.get("reason", "Applied."), clamped_from[0], clamped_from[1], box_w, box_h, res.get("width") or w, res.get("height") or h)) return res if current and tuple(current) == (w, h): return _ok({"applied": True, "width": w, "height": h, "reason": "Already at that resolution."}) if modes and (w, h) not in [tuple(m) for m in modes]: return {"applied": False, "width": current[0] if current else None, "height": current[1] if current else None, "reason": "The adapter does not offer %dx%d." % (w, h)} session = _windows_session_id() if session == 0: # The common case for a boot-start agent, and the one that used to fail silently. got = _set_display_mode_in_console_session(w, h) if got == "PENDING": # Honest about not knowing. The change was dispatched and had not confirmed; it may well # have applied, and claiming otherwise is the lie this branch used to tell. return {"applied": False, "width": current[0] if current else None, "height": current[1] if current else None, "reason": "The change was dispatched to the console session but did not confirm " "in time. It may still have applied -- reopen the control to see the " "current resolution."} if got: return _ok({"applied": True, "width": got[0], "height": got[1], "reason": "Applied in the console session."}) return {"applied": False, "reason": "The agent runs as a service in session 0 and the console session did not " "accept the change. Set it from the remote desktop instead."} if platform.system() == "Darwin": # The existing, verified macOS path. Restore-on-disconnect is NOT wired up here on purpose: # that means joining _acquire_desktop_display_mode's ref count, which is the one change that # can leave someone else's display at a size they did not choose, so it is handled separately. if _apply_darwin_display_mode((w, h)): return _ok({"applied": True, "width": w, "height": h, "reason": "Applied."}) return {"applied": False, "reason": "The display refused the mode change."} got = _set_windows_display_mode(w, h, True) if got: return _ok({"applied": True, "width": got[0], "height": got[1], "reason": "Applied."}) return {"applied": False, "reason": "The display refused the mode change."} def _enumerate_windows_display_modes(): """Supported (width, height) modes for the primary adapter, plus the current mode. Returns (modes, current) where current may be None. Windows-only; any failure yields ([], None) so callers degrade to leaving the resolution alone. """ if platform.system() != "Windows" or ctypes is None: return [], None try: # No `import ctypes.wintypes` here. It bound `ctypes` as a local for this whole function, # which turned the `ctypes is None` guard three lines above into a read-before-assignment -- # UnboundLocalError on every Windows call, raised before the function did anything. Nothing # in this file uses wintypes; the import existed only to prove it was importable. class DEVMODEW(ctypes.Structure): _fields_ = [ ("dmDeviceName", ctypes.c_wchar * 32), ("dmSpecVersion", ctypes.c_ushort), ("dmDriverVersion", ctypes.c_ushort), ("dmSize", ctypes.c_ushort), ("dmDriverExtra", ctypes.c_ushort), ("dmFields", ctypes.c_ulong), ("dmOrientation", ctypes.c_short), ("dmPaperSize", ctypes.c_short), ("dmPaperLength", ctypes.c_short), ("dmPaperWidth", ctypes.c_short), ("dmScale", ctypes.c_short), ("dmCopies", ctypes.c_short), ("dmDefaultSource", ctypes.c_short), ("dmPrintQuality", ctypes.c_short), ("dmColor", ctypes.c_short), ("dmDuplex", ctypes.c_short), ("dmYResolution", ctypes.c_short), ("dmTTOption", ctypes.c_short), ("dmCollate", ctypes.c_short), ("dmFormName", ctypes.c_wchar * 32), ("dmLogPixels", ctypes.c_ushort), ("dmBitsPerPel", ctypes.c_ulong), ("dmPelsWidth", ctypes.c_ulong), ("dmPelsHeight", ctypes.c_ulong), ("dmDisplayFlags", ctypes.c_ulong), ("dmDisplayFrequency", ctypes.c_ulong), ("dmICMMethod", ctypes.c_ulong), ("dmICMIntent", ctypes.c_ulong), ("dmMediaType", ctypes.c_ulong), ("dmDitherType", ctypes.c_ulong), ("dmReserved1", ctypes.c_ulong), ("dmReserved2", ctypes.c_ulong), ("dmPanningWidth", ctypes.c_ulong), ("dmPanningHeight", ctypes.c_ulong), ] user32 = ctypes.windll.user32 modes = [] seen = set() dm = DEVMODEW() dm.dmSize = ctypes.sizeof(DEVMODEW) current = None # Named explicitly: None means "the default display", which on real hardware has been # observed not to resolve at all -- both calls fail while the same calls against the # explicit device name succeed. _dev = _primary_display_device() if user32.EnumDisplaySettingsW(_dev, ENUM_CURRENT_SETTINGS, ctypes.byref(dm)): current = (int(dm.dmPelsWidth), int(dm.dmPelsHeight)) i = 0 while True: probe = DEVMODEW() probe.dmSize = ctypes.sizeof(DEVMODEW) if not user32.EnumDisplaySettingsW(_dev, i, ctypes.byref(probe)): break # 16-bit modes still get reported by some virtual adapters and look terrible over VNC; # only consider true colour, which is also what the current mode will be. if int(probe.dmBitsPerPel) >= 24: key = (int(probe.dmPelsWidth), int(probe.dmPelsHeight)) if key not in seen: seen.add(key) modes.append(key) i += 1 return modes, current except Exception as e: print(f"[{time.strftime('%X')}] [!] Could not enumerate display modes: {e}", flush=True) return [], None def _set_windows_display_mode(req_w, req_h, server_flag=None): """Best-effort match of the Windows console resolution to a req_w x req_h viewer panel. Returns the (width, height) now in effect, or None if nothing was changed. Never raises: a node whose adapter offers no suitable mode, or refuses the change, simply keeps its current resolution and the viewer falls back to the letterboxing it had before. """ if platform.system() != "Windows" or ctypes is None: return None if not req_w or not req_h: return None if not _match_display_resolution_enabled(server_flag): print(f"[{time.strftime('%X')}] [🖥️ DISPLAY] Viewer panel is {req_w}x{req_h}; console resolution " f"matching is disabled (set APPLOTUS_MATCH_DISPLAY_RESOLUTION=1 to enable) — leaving the " f"resolution alone", flush=True) return None modes, current = _enumerate_windows_display_modes() if not modes: return None chosen = _pick_display_mode(modes, req_w, req_h) if not chosen: return None if current and chosen == current: print(f"[{time.strftime('%X')}] [🖥️ DISPLAY] Already at {chosen[0]}x{chosen[1]} for a " f"{req_w}x{req_h} panel — leaving it alone", flush=True) return current try: class DEVMODEW(ctypes.Structure): _fields_ = [ ("dmDeviceName", ctypes.c_wchar * 32), ("dmSpecVersion", ctypes.c_ushort), ("dmDriverVersion", ctypes.c_ushort), ("dmSize", ctypes.c_ushort), ("dmDriverExtra", ctypes.c_ushort), ("dmFields", ctypes.c_ulong), ("dmOrientation", ctypes.c_short), ("dmPaperSize", ctypes.c_short), ("dmPaperLength", ctypes.c_short), ("dmPaperWidth", ctypes.c_short), ("dmScale", ctypes.c_short), ("dmCopies", ctypes.c_short), ("dmDefaultSource", ctypes.c_short), ("dmPrintQuality", ctypes.c_short), ("dmColor", ctypes.c_short), ("dmDuplex", ctypes.c_short), ("dmYResolution", ctypes.c_short), ("dmTTOption", ctypes.c_short), ("dmCollate", ctypes.c_short), ("dmFormName", ctypes.c_wchar * 32), ("dmLogPixels", ctypes.c_ushort), ("dmBitsPerPel", ctypes.c_ulong), ("dmPelsWidth", ctypes.c_ulong), ("dmPelsHeight", ctypes.c_ulong), ("dmDisplayFlags", ctypes.c_ulong), ("dmDisplayFrequency", ctypes.c_ulong), ("dmICMMethod", ctypes.c_ulong), ("dmICMIntent", ctypes.c_ulong), ("dmMediaType", ctypes.c_ulong), ("dmDitherType", ctypes.c_ulong), ("dmReserved1", ctypes.c_ulong), ("dmReserved2", ctypes.c_ulong), ("dmPanningWidth", ctypes.c_ulong), ("dmPanningHeight", ctypes.c_ulong), ] user32 = ctypes.windll.user32 dm = DEVMODEW() dm.dmSize = ctypes.sizeof(DEVMODEW) # Start from the current mode so colour depth and refresh rate carry over untouched; only the # two pels fields are declared in dmFields, so nothing else is asked to change. _dev = _primary_display_device() user32.EnumDisplaySettingsW(_dev, ENUM_CURRENT_SETTINGS, ctypes.byref(dm)) dm.dmPelsWidth = chosen[0] dm.dmPelsHeight = chosen[1] dm.dmFields = DM_PELSWIDTH | DM_PELSHEIGHT rc = user32.ChangeDisplaySettingsExW(_dev, ctypes.byref(dm), None, 0, None) if rc == DISP_CHANGE_SUCCESSFUL: was = f"{current[0]}x{current[1]}" if current else "unknown" print(f"[{time.strftime('%X')}] [🖥️ DISPLAY] Console set to {chosen[0]}x{chosen[1]} to match " f"a {req_w}x{req_h} viewer panel (was {was})", flush=True) return chosen print(f"[{time.strftime('%X')}] [!] Display mode change to {chosen[0]}x{chosen[1]} refused " f"(ChangeDisplaySettingsEx returned {rc}) — keeping current resolution", flush=True) return None except Exception as e: print(f"[{time.strftime('%X')}] [!] Display mode change failed: {e} — keeping current resolution", flush=True) return None def _connect_local_vnc(port, total_timeout=9.0, attempt_timeout=1.5): """Connects to a VNC server on loopback, retrying until total_timeout is spent. A single attempt was not enough. On Windows this runs moments after `Restart-Service tvnserver`, and a service reporting "started" is not the same as a socket accepting connections -- the listener comes up some time later, and on a loaded host that can be several seconds. There is also no fallback on Windows: has_x11vnc_helper is false there, so the on-demand x11vnc path below is skipped and one missed connect killed the whole session, leaving the viewer to sit until the server's 15s timeout with nothing logged to say why. Returns a connected socket, or None once the budget is exhausted. """ deadline = time.time() + total_timeout attempt = 0 last_err = None while time.time() < deadline: attempt += 1 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(attempt_timeout) try: sock.connect(("127.0.0.1", port)) if attempt > 1: print(f"[{time.strftime('%X')}] [🖥️ VNC] Connected to 127.0.0.1:{port} on attempt {attempt}", flush=True) return sock except Exception as e: last_err = e try: sock.close() except Exception: pass # Short, fixed gap rather than exponential backoff: the thing being waited on is a service # finishing its start, which takes seconds not minutes, and the whole budget has to fit # inside the server's 15s session timeout. if time.time() < deadline: time.sleep(0.5) print(f"[{time.strftime('%X')}] [!] VNC on 127.0.0.1:{port} did not accept a connection within " f"{total_timeout:.0f}s ({attempt} attempts, last error: {last_err})", flush=True) return None # Security types the agent's pre-auth interception can actually complete. Everything else -- Apple's # 30/33/35/36, TightVNC's 16, Ultra's 17, the vendor-proprietary 113-118 range -- needs a scheme that is # either unimplemented here or has no published specification. RFB_AUTH_TYPE_NAMES = { 1: "None", 2: "VNC password", 5: "RA2", 6: "RA2ne", 16: "Tight", 17: "Ultra", 18: "TLS", 19: "VeNCrypt", 20: "SASL", 30: "Apple DH", 33: "Apple", 35: "Apple", 36: "Apple", } def _describe_auth_types(types): """Renders an offered security-type list with names, so a log line is readable without a spec.""" out = [] for t in types: name = RFB_AUTH_TYPE_NAMES.get(t) if name is None: name = "vendor-proprietary" if 113 <= t <= 118 else "unknown" out.append("%d (%s)" % (t, name)) return ", ".join(out) if out else "none" def _windows_vnc_service(): """(name, status) of whichever known VNC server service is installed, or (None, None). The Windows desktop path has always assumed TightVNC -- it writes TightVNC's registry keys and restarts `tvnserver`. On a host running UltraVNC or RealVNC instead, every one of those actions targets a product that is not serving the port, which is exactly the state node-antwin was in. Knowing the real service is what lets the error name the right product. """ if platform.system() != "Windows": return None, None candidates = ["tvnserver", "uvnc_service", "winvnc", "vncserver", "RealVNC Server", "vncserver_x11", "WinVNC4"] try: ps = ("$n=@(%s); foreach($x in $n){ $s=Get-Service -Name $x -ErrorAction SilentlyContinue; " "if($s){ \"$($s.Name)=$($s.Status)\" } }" % ",".join("'%s'" % c for c in candidates)) r = subprocess.run(["powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", ps], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=12) for line in (r.stdout or "").splitlines(): if "=" in line: nm, st = line.strip().split("=", 1) return nm, st except Exception: pass return None, None def _local_rfb_auth_types(port, timeout=4.0): """(usable, detail) for a VNC server on loopback, by negotiating as far as the security types. Returns usable=True only when the server offers security type 1 (None) or 2 (VNC password), which are the two the agent's pre-auth interception implements. One step further than reading the banner, and that step is the whole point. TightVNC answers the banner perfectly well and then offers only its own security type 16, which this agent cannot authenticate -- so a banner-only probe reports health on a server that will refuse every session. The configuration that makes it offer type 1 (UseVncAuthentication=0, UseControlAuthentication=0) is read at service start, so detecting this is what tells the caller a restart is actually needed. `detail` carries the offered type list, or the server's own failure reason when it answers with zero types -- RFB's documented way of saying "rejected, here is why" (RFC 6143 7.1.2), which is how a server refusing loopback connections replies. """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) try: sock.connect(("127.0.0.1", port)) def _exact(n): buf = b"" while len(buf) < n: chunk = sock.recv(n - len(buf)) if not chunk: raise RuntimeError("closed during handshake") buf += chunk return buf banner = _exact(12) if not banner.startswith(b"RFB "): return False, "not an RFB server (got %r)" % banner[:12] sock.sendall(b"RFB 003.008\n") count = _exact(1)[0] if count == 0: try: reason = _exact(int.from_bytes(_exact(4), "big")).decode("utf-8", "replace") except Exception: reason = "no reason given" return False, "server rejected the connection: %s" % reason.strip() types = list(_exact(count)) return (1 in types or 2 in types), "offered security types %s" % (types,) except Exception as e: return False, "handshake probe failed: %s" % e finally: try: sock.close() except Exception: pass def _local_rfb_answering(port, timeout=2.0): """Whether a VNC server on 127.0.0.1:port will actually serve a loopback client. Stronger than _local_port_answering on purpose, and the difference is the whole bug it was written for. TightVNC with AllowLoopback=0 -- its default -- accepts the TCP connection and then closes it without serving, so a connect-only probe reports a healthy server where there is none. Every RFB server opens by sending a 12-byte "RFB 003.00x\n" banner unprompted, so requiring that banner distinguishes "listening and willing" from "listening and about to hang up". Returns False on refusal, on an immediate EOF, on a timeout, and on anything that is not RFB. """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) try: sock.connect(("127.0.0.1", port)) banner = b"" while len(banner) < 4: chunk = sock.recv(12 - len(banner)) if not chunk: return False # accepted then hung up: exactly the AllowLoopback=0 signature banner += chunk return banner.startswith(b"RFB ") except Exception: return False finally: try: sock.close() except Exception: pass def _local_port_answering(port, timeout=1.0): """Cheap one-shot check for an already-listening local port.""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) try: sock.connect(("127.0.0.1", port)) return True except Exception: return False finally: try: sock.close() except Exception: pass CAFFEINATE_PATH = "/usr/bin/caffeinate" def _start_session_wake_lock(session_id, kind="remote"): """Hold an OS-level no-sleep assertion for the life of a remote session (macOS only). A Mac that sleeps mid-session drops every socket to it at once. For a desktop session that loses the RFB stream and, if the lock screen asks for a password, brings it back on the login window -- which arrives as pixels inside the framebuffer, so nothing on the Applotus side can dismiss it. For a terminal session the shell simply dies. Keeping the machine awake for the duration is the one link in that chain we control. Applies to terminal sessions too, not just desktop: they were the more obvious case in hindsight, since a shell sitting at a prompt generates no activity at all and so lets an idle timer expire even sooner than a desktop session does. Returns a handle for _stop_session_wake_lock, or None when no assertion is held. """ if platform.system() != "Darwin": return None if not os.path.exists(CAFFEINATE_PATH): print(f"[{time.strftime('%X')}] [☕ WAKE LOCK] {CAFFEINATE_PATH} missing — session may be cut short by sleep", flush=True) return None try: # -w makes caffeinate release the assertion on its own if the agent is killed # abruptly. Without it a crash mid-session would orphan a process holding the Mac awake # indefinitely, with nothing left running that knows to clean it up. The normal path still # terminates it explicitly when the session ends. # # -u is deliberately absent despite reading like the obvious companion flag: caffeinate(8) # takes it with a 5 second default timeout unless -t is given, and -t would cap the whole # assertion, so on a session lasting minutes the -u part asserts essentially nothing. proc = subprocess.Popen( [CAFFEINATE_PATH, "-dims", "-w", str(os.getpid())], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) except Exception as e: print(f"[{time.strftime('%X')}] [☕ WAKE LOCK] Could not start caffeinate: {e}", flush=True) return None print(f"[{time.strftime('%X')}] [☕ WAKE LOCK] Holding no-sleep assertion (pid {proc.pid}) for {kind} session {session_id}", flush=True) _wake_display_now() return proc def _wake_display_now(): """Turns the display back on if it is already off. Separate from the hold above, and necessary. -dims prevents the display from sleeping; it does nothing about a display that went to sleep before anyone connected. That case is not cosmetic: macOS Screen Sharing serves the dark framebuffer, so the viewer showed a black screen with no login window and no desktop -- measured as 78KB in 7 frames, against 9275KB for a real screen. -u is what turns a sleeping display on (caffeinate(8)). It was deliberately left off the hold because without -t it lasts only 5 seconds, which is useless for holding -- but waking is instantaneous, so that expiry does not matter here at all. The two flags do different jobs and both are needed. Run on a thread so the ~1s lifetime of the one-shot does not delay the session, and waited on there so it is reaped rather than left as a zombie for every session. """ def _run(): try: subprocess.run([CAFFEINATE_PATH, "-u", "-t", "1"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10) except Exception as e: print(f"[{time.strftime('%X')}] [☕ WAKE LOCK] Could not wake the display: {e}", flush=True) threading.Thread(target=_run, daemon=True).start() def _stop_session_wake_lock(proc, session_id): """Release the assertion taken by _start_session_wake_lock, letting the Mac sleep normally again.""" if proc is None: return try: if proc.poll() is not None: # Already gone -- the assertion died with it, so there is nothing to release. return proc.terminate() try: proc.wait(timeout=3) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=3) print(f"[{time.strftime('%X')}] [☕ WAKE LOCK] Released for session {session_id}", flush=True) except Exception as e: print(f"[{time.strftime('%X')}] [☕ WAKE LOCK] Failed to release cleanly: {e}", flush=True) # Positional order of _run_desktop_session_bridge's parameters, so the wrapper can read one whether # the caller passed it positionally or by keyword. Kept beside the bridge so the two stay in step. _DESKTOP_BRIDGE_PARAMS = ("session_id", "ws_url", "port", "insecure", "host_header", "api_key", "vnc_password", "desired_width", "desired_height", "match_resolution", "preferred_display", "desired_from_preference") def _desktop_bridge_arg(args, kwargs, name): """One of the desktop bridge's arguments, however the caller chose to pass it.""" if name in kwargs: return kwargs[name] i = _DESKTOP_BRIDGE_PARAMS.index(name) return args[i] if len(args) > i else None # Refcounted because a node can carry two desktop sessions at once, and the second one ending must not # restore the display while the first is still drawing on it. The first session's choice stands for the # duration: re-picking per session would change resolution under a live viewer for no real gain. _display_perf_lock = threading.Lock() _display_perf_state = {"refs": 0, "original": None} def _darwin_cg(): """CoreGraphics/CoreFoundation entry points for display mode switching. ctypes is imported locally, not at module scope: it is not a module-level import in this agent, and adding one would break every platform at load time rather than only here. """ import ctypes import ctypes.util cg = ctypes.CDLL(ctypes.util.find_library("CoreGraphics")) cf = ctypes.CDLL(ctypes.util.find_library("CoreFoundation")) cg.CGMainDisplayID.restype = ctypes.c_uint32 cg.CGDisplayCopyDisplayMode.argtypes = [ctypes.c_uint32] cg.CGDisplayCopyDisplayMode.restype = ctypes.c_void_p cg.CGDisplayCopyAllDisplayModes.argtypes = [ctypes.c_uint32, ctypes.c_void_p] cg.CGDisplayCopyAllDisplayModes.restype = ctypes.c_void_p for fn in ("CGDisplayModeGetPixelWidth", "CGDisplayModeGetPixelHeight", "CGDisplayModeGetWidth", "CGDisplayModeGetHeight"): f = getattr(cg, fn) f.argtypes = [ctypes.c_void_p] f.restype = ctypes.c_size_t cg.CGDisplaySetDisplayMode.argtypes = [ctypes.c_uint32, ctypes.c_void_p, ctypes.c_void_p] cg.CGDisplaySetDisplayMode.restype = ctypes.c_int32 cg.CGDisplayModeRelease.argtypes = [ctypes.c_void_p] cf.CFArrayGetCount.argtypes = [ctypes.c_void_p] cf.CFArrayGetCount.restype = ctypes.c_long cf.CFArrayGetValueAtIndex.argtypes = [ctypes.c_void_p, ctypes.c_long] cf.CFArrayGetValueAtIndex.restype = ctypes.c_void_p cf.CFRelease.argtypes = [ctypes.c_void_p] return ctypes, cg, cf def _enumerate_darwin_display_modes(): """(modes, current) for the main display, measured in BACKING PIXELS. Pixels rather than logical points, because the framebuffer screensharingd serves is the backing store. A 27" iMac sitting in the HiDPI mode that "looks like" 1600x900 is really 3200x1800, and a picker fed logical sizes would see a number that already fits the viewer and leave 4x the pixels in place. Every mode CGDisplayCopyAllDisplayModes returns by default is 1x, so for those two numbers agree -- it is the *current* mode, which may be HiDPI and is fetched separately, where they differ. Any failure yields ([], None) so the caller leaves the resolution alone. """ if platform.system() != "Darwin": return [], None arr = None cur = None try: ctypes, cg, cf = _darwin_cg() did = cg.CGMainDisplayID() modes = [] arr = cg.CGDisplayCopyAllDisplayModes(did, None) if arr: for i in range(cf.CFArrayGetCount(arr)): m = cf.CFArrayGetValueAtIndex(arr, i) w, h = cg.CGDisplayModeGetPixelWidth(m), cg.CGDisplayModeGetPixelHeight(m) if w and h and (w, h) not in modes: modes.append((int(w), int(h))) cur = cg.CGDisplayCopyDisplayMode(did) current = None if cur: current = (int(cg.CGDisplayModeGetPixelWidth(cur)), int(cg.CGDisplayModeGetPixelHeight(cur))) return modes, current except Exception as e: print(f"[{time.strftime('%X')}] [!] Could not enumerate display modes: {e}", flush=True) return [], None finally: try: if cur: _, cg2, _ = _darwin_cg() cg2.CGDisplayModeRelease(cur) if arr: _, _, cf2 = _darwin_cg() cf2.CFRelease(arr) except Exception: pass def _apply_darwin_display_mode(target): """Switches the main display to the mode whose backing pixels are `target`. True when it took. The mode reference has to be used while the array that owns it is still alive, so the lookup and the switch happen together here rather than the ref being handed back to a caller. """ if platform.system() != "Darwin": return False arr = None try: ctypes, cg, cf = _darwin_cg() did = cg.CGMainDisplayID() arr = cg.CGDisplayCopyAllDisplayModes(did, None) if not arr: return False for i in range(cf.CFArrayGetCount(arr)): m = cf.CFArrayGetValueAtIndex(arr, i) if (int(cg.CGDisplayModeGetPixelWidth(m)), int(cg.CGDisplayModeGetPixelHeight(m))) == tuple(target): err = cg.CGDisplaySetDisplayMode(did, m, None) if err != 0: print(f"[{time.strftime('%X')}] [!] Display mode switch to " f"{target[0]}x{target[1]} failed (CGError {err})", flush=True) return False return True return False except Exception as e: print(f"[{time.strftime('%X')}] [!] Display mode switch failed: {e}", flush=True) return False finally: try: if arr: _, _, cf2 = _darwin_cg() cf2.CFRelease(arr) except Exception: pass def _acquire_desktop_display_mode(req_w, req_h, server_flag, session_id): """Drops the main display to the smallest mode that still covers the viewer panel. True if held. Enabled by default on macOS, unlike the Windows path, because this one is verifiable on real hardware and was verified. An explicit local setting still wins in either direction -- whoever is physically at the machine keeps the final say, including forcing it off: APPLOTUS_MATCH_DISPLAY_RESOLUTION=0 Failure is never fatal: every path returns False and the session proceeds at whatever resolution the display was already in. A session that connects is worth more than one that is sharp. """ if platform.system() != "Darwin": return False raw = _cfg("MATCH_DISPLAY_RESOLUTION") if raw != "" and raw.strip().lower() not in ("1", "true", "yes", "on"): return False if not req_w or not req_h or req_w <= 0 or req_h <= 0: return False with _display_perf_lock: if _display_perf_state["refs"] > 0: # Already lowered for another live session; join it rather than switching again. _display_perf_state["refs"] += 1 return True modes, current = _enumerate_darwin_display_modes() if not modes or not current: return False chosen = _pick_display_mode(modes, int(req_w), int(req_h)) if not chosen: return False if tuple(chosen) == tuple(current): print(f"[{time.strftime('%X')}] [\U0001F5A5️ DISPLAY] Already at " f"{current[0]}x{current[1]} for a {req_w}x{req_h} panel — leaving it alone", flush=True) return False if chosen[0] * chosen[1] >= current[0] * current[1]: # Only ever reduce. Raising the resolution to "cover" a large browser window would cost # bandwidth on a link that is already the bottleneck, which is the opposite of the point. print(f"[{time.strftime('%X')}] [\U0001F5A5️ DISPLAY] Best fit for a {req_w}x{req_h} " f"panel is {chosen[0]}x{chosen[1]}, not smaller than the current " f"{current[0]}x{current[1]} — leaving it alone", flush=True) return False if not _apply_darwin_display_mode(chosen): return False _display_perf_state["original"] = current _display_perf_state["refs"] = 1 saved = 1.0 - float(chosen[0] * chosen[1]) / float(current[0] * current[1]) print(f"[{time.strftime('%X')}] [\U0001F5A5️ DISPLAY] {session_id}: " f"{current[0]}x{current[1]} -> {chosen[0]}x{chosen[1]} for a {req_w}x{req_h} viewer " f"({saved * 100:.0f}% fewer pixels per frame); restores when the last session closes", flush=True) # The switch reconfigures the graphics stack; let it settle before anything reads the framebuffer. time.sleep(0.6) return True def _release_desktop_display_mode(held, session_id): """Restores the display once the last session using it has gone. The Windows path never did this and left the console at whatever it had picked. Coming back to a machine someone else silently re-resolutioned is a bad surprise, so this one always puts it back. """ if not held: return with _display_perf_lock: _display_perf_state["refs"] = max(0, _display_perf_state["refs"] - 1) if _display_perf_state["refs"] > 0: return original = _display_perf_state.get("original") _display_perf_state["original"] = None if not original: return if _apply_darwin_display_mode(original): print(f"[{time.strftime('%X')}] [\U0001F5A5️ DISPLAY] {session_id}: restored " f"{original[0]}x{original[1]}", flush=True) else: print(f"[{time.strftime('%X')}] [!] Could not restore {original[0]}x{original[1]}; the " f"display is still at the session resolution", flush=True) def run_desktop_session(*args, **kwargs): """Wraps the desktop bridge in a wake lock so the host cannot sleep mid-session. Takes *args/**kwargs rather than restating the bridge's ten parameters, which would be a second copy of that signature to keep in step for no benefit -- all this needs is the session id, and both callers pass it first. """ session_id = _desktop_bridge_arg(args, kwargs, "session_id") or "?" wake_lock = _start_session_wake_lock(session_id, "desktop") # Before the bridge, so the mode has settled before anything connects to the framebuffer. The # Windows equivalent of this lives inside the bridge and stays there; this one is out here because # it also has to be released, and the bridge has too many exits to do that reliably. display_held = _acquire_desktop_display_mode( _desktop_bridge_arg(args, kwargs, "desired_width"), _desktop_bridge_arg(args, kwargs, "desired_height"), _desktop_bridge_arg(args, kwargs, "match_resolution"), session_id) try: return _run_desktop_session_bridge(*args, **kwargs) finally: # finally, not a call at each return: the bridge exits from four places plus any exception, # and a missed one would leave the machine unable to sleep until the agent restarts -- or, now, # stuck at the session's resolution. _stop_session_wake_lock(wake_lock, session_id) _release_desktop_display_mode(display_held, session_id) def _run_desktop_session_bridge(session_id, ws_url, port=None, insecure=False, host_header=None, api_key=None, vnc_password="", desired_width=None, desired_height=None, match_resolution=None, preferred_display=None, desired_from_preference=None): """Bridges an interactive Remote Desktop session (RFB/VNC) over WebSocket with CPU, Memory, and N/W tuning.""" with active_desktop_lock: if session_id in active_desktop_sessions: return active_desktop_sessions.add(session_id) print(f"[{time.strftime('%X')}] [🖥️ DESKTOP SESSION] Initiating for session {session_id}...", flush=True) d_info = check_desktop_env() target_port = port or d_info.get("port") or 5900 if target_port == 3389 and d_info.get("has_x11vnc_helper"): target_port = 5900 # Connect to Applotus server WebSocket ws = None try: headers = _ws_auth_headers(api_key) ws = SimpleWebSocketClient(ws_url, insecure=insecure, host_header=host_header, headers=headers) ws.connect(timeout=15) # N/W Optimization: Disable Nagle algorithm on WebSocket connection for lowest latency try: ws.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except Exception: pass except Exception as e: print(f"[{time.strftime('%X')}] [-] Failed to connect Desktop WebSocket: {e}", flush=True) with active_desktop_lock: active_desktop_sessions.discard(session_id) return # On Windows: Ensure TightVNC Server registry settings allow loopback & configure correctly if platform.system() == "Windows": # Resolution first, before anything restarts the service. # # This used to run after the Restart-Service below, on the reasoning that TightVNC reports # whatever the desktop is at when a client attaches -- which is true, but attaching happens # further down either way, so doing it first satisfies that just as well. What it avoids is # changing the display mode while tvnserver is mid-restart: a mode switch reconfigures the # graphics stack, and overlapping the two put a service that was still coming up next to a # framebuffer that was still changing shape, immediately before a connect attempt that had no # retry. Ordering it first lets the mode settle while the service restarts. if desired_width and desired_height: applied = _set_windows_display_mode(desired_width, desired_height, match_resolution) if applied: # The mode switch is asynchronous inside Windows; give it a moment before a service # restart or a client attach reads the geometry. time.sleep(0.6) try: import winreg for subkey in [r"Software\TightVNC\Server", r"Software\Wow6432Node\TightVNC\Server"]: try: with winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, subkey) as key: winreg.SetValueEx(key, "AllowLoopback", 0, winreg.REG_DWORD, 1) winreg.SetValueEx(key, "LoopbackOnly", 0, winreg.REG_DWORD, 1) winreg.SetValueEx(key, "AcceptHttpConnections", 0, winreg.REG_DWORD, 0) winreg.SetValueEx(key, "UseControlAuthentication", 0, winreg.REG_DWORD, 0) winreg.SetValueEx(key, "UseVncAuthentication", 0, winreg.REG_DWORD, 0) except Exception as e: # Needs Administrator. Swallowed silently before, which is how a non-elevated agent # would fail every session with nothing logged anywhere to say why. print(f"[{time.strftime('%X')}] [!] Could not configure TightVNC under {subkey} " f"({e}) — requires Administrator", flush=True) except Exception: pass # Only restart the service when it is not already SERVING loopback -- which is a different # question from whether something is listening, and getting the two confused is what broke # node-antwin. Restarting unconditionally would tear down the very listener we are about to # connect to, pure self-inflicted downtime whenever the previous session left things working. # # But the registry writes above only reach a RUNNING tvnserver through a restart: TightVNC reads # AllowLoopback and friends at service start. So skipping the restart also skips the # configuration, and with AllowLoopback=0 (TightVNC's default) the service accepts the # connection and immediately closes it. A connect-only probe cannot tell that apart from a # healthy server, so every session died mid-handshake -- WS close 1006 after ~237 bytes -- while # the agent logged that everything was already fine. # # _local_rfb_answering requires the server's RFB banner, which it only sends to a client it is # willing to serve. Listening-but-refusing now correctly falls through to the restart. probe_port = target_port if target_port != 3389 else 5900 # Usable auth, not merely a greeting. tvnserver answers the banner and then offers only its own # security type 16, which this agent cannot authenticate -- so a banner-only check reported # health on a server that refused every session. The registry writes above make it offer type 1 # instead, and they are read at service start, which is why this decides the restart. _usable, _why = _local_rfb_auth_types(probe_port) if _usable: print(f"[{time.strftime('%X')}] [🖥️ VNC] tvnserver serving 127.0.0.1:{probe_port} " f"with usable auth ({_why}) — skipping service restart", flush=True) else: _svc_name, _svc_state = _windows_vnc_service() # Restarting tvnserver cannot help a host served by UltraVNC or RealVNC, and the registry # writes above went to keys that product does not read. Attempting it anyway produced a # failed restart and an unrelated registry dump in the log, directly after a line saying # the TightVNC configuration did not apply. _skip_restart = bool(_svc_name) and _svc_name.lower() != "tvnserver" if _skip_restart: # Restarting tvnserver cannot help a host served by a different product, and the # registry writes above went to keys it does not read. print(f"[{time.strftime('%X')}] [!] 127.0.0.1:{probe_port} cannot serve this agent " f"({_why}), and the VNC service here is {_svc_name} ({_svc_state}), not " f"tvnserver — the TightVNC configuration above does not apply to it. Enable " f"standard VNC authentication on {_svc_name} and store its password against " f"this node.", flush=True) elif not _svc_name: print(f"[{time.strftime('%X')}] [!] 127.0.0.1:{probe_port} cannot serve this agent " f"({_why}) and no recognised VNC service was found by name; restarting " f"tvnserver anyway in case it is installed under another name", flush=True) else: print(f"[{time.strftime('%X')}] [🖥️ VNC] 127.0.0.1:{probe_port} cannot serve this " f"agent ({_why}) — restarting tvnserver so the configuration above takes effect", flush=True) try: ps_cmd = ( "$keys = @('HKLM:\\Software\\TightVNC\\Server', 'HKLM:\\Software\\Wow6432Node\\TightVNC\\Server'); " "foreach ($k in $keys) { " "New-Item -Path $k -Force -ErrorAction SilentlyContinue; " "Set-ItemProperty -Path $k -Name 'AllowLoopback' -Value 1 -Type DWord -ErrorAction SilentlyContinue; " "Set-ItemProperty -Path $k -Name 'LoopbackOnly' -Value 1 -Type DWord -ErrorAction SilentlyContinue; " "Set-ItemProperty -Path $k -Name 'AcceptHttpConnections' -Value 0 -Type DWord -ErrorAction SilentlyContinue; " "Set-ItemProperty -Path $k -Name 'UseControlAuthentication' -Value 0 -Type DWord -ErrorAction SilentlyContinue; " "Set-ItemProperty -Path $k -Name 'UseVncAuthentication' -Value 0 -Type DWord -ErrorAction SilentlyContinue; " "}; Restart-Service tvnserver -ErrorAction SilentlyContinue" ) _ps = None if _skip_restart else subprocess.run([ "powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", ps_cmd ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=12) if _ps is not None and _ps.returncode != 0: # Both streams went to DEVNULL before, and the command carries # -ErrorAction SilentlyContinue, so a restart that could not run was # indistinguishable from one that worked. _err = ((_ps.stderr or "") + (_ps.stdout or "")).strip() print(f"[{time.strftime('%X')}] [!] tvnserver restart exited {_ps.returncode}: " f"{_err[:300] or 'no output'}", flush=True) elif _ps is not None: # Confirm the restart achieved the thing it was for, rather than assuming it did. for _ in range(10): time.sleep(0.5) _usable, _why = _local_rfb_auth_types(probe_port) if _usable: print(f"[{time.strftime('%X')}] [🖥️ VNC] tvnserver came back with " f"usable auth ({_why})", flush=True) break if not _usable: print(f"[{time.strftime('%X')}] [!] tvnserver restarted but still cannot serve " f"this agent ({_why}); the session will fail. Set UseVncAuthentication=0 " f"and UseControlAuthentication=0 under " f"HKLM\\Software\\TightVNC\\Server, or set a VNC password.", flush=True) except Exception: pass # Check if local VNC port is responding. Retried rather than attempted once -- see # _connect_local_vnc for why a single 2s shot was losing sessions outright on Windows. vnc_proc = None vnc_sock = _connect_local_vnc(target_port) # If port isn't open, start an on-demand x11vnc session optimized for CPU, Memory, and Network if vnc_sock is None and d_info.get("has_x11vnc_helper"): display = d_info.get("display") or os.environ.get("DISPLAY") or ":0" # The operator's explicit choice wins over auto-selection, which sorts /tmp/.X11-unix # descending and so prefers an xrdp session (:10) over the console (:0). if preferred_display: _avail = [d.get("display") for d in (d_info.get("displays") or []) if d.get("display")] if preferred_display in _avail or not _avail: # `not _avail` honours the choice when enumeration produced nothing: an empty list is # not evidence the display is gone, and refusing would strand the session. if preferred_display != display: print(f"[{time.strftime('%X')}] [\U0001F5A5️ DESKTOP] Using operator-selected " f"display {preferred_display} (auto-selection would have used {display})", flush=True) display = preferred_display else: # A choice that has since disappeared -- an xrdp session that ended, say. Falling back # keeps the desktop reachable; silence here would look like the picker was ignored. print(f"[{time.strftime('%X')}] [!] Selected display {preferred_display} is not " f"present (found {_avail}); falling back to {display}", flush=True) x11vnc_path = None for cand in ["/usr/bin/x11vnc", "/usr/local/bin/x11vnc", "x11vnc"]: try: res = subprocess.run(["which", cand], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) if res.returncode == 0 and res.stdout.strip(): x11vnc_path = res.stdout.strip() break except Exception: pass if x11vnc_path: x11vnc_log_path = None try: vnc_rfb_port = target_port if target_port != 3389 else 5900 print(f"[{time.strftime('%X')}] [🖥️ DESKTOP] Spawning tuned on-demand x11vnc on {display} (port {vnc_rfb_port})...", flush=True) # Resource tuning: # -shared: allow browser reconnects # -nowf: eliminate unnecessary wireframe polling overhead # -ncache 10 -ncache_cr: client-side pixel cache to cut bandwidth by ~70% # -speeds 100: sets optimal high-throughput polling rates # -wait 40 -defer 30: throttles frame bursts, capping CPU < 5% during idle/typing # -noshm: this agent frequently runs as root while the display belongs to a # different, logged-in user -- the X server denies MIT-SHM attach across that UID # boundary (BadAccess), which crashes x11vnc immediately after it otherwise opens # the display fine. -noshm falls back to plain XGetImage polling, slower but the # only mode that actually works cross-UID; confirmed live (x11vnc 0.9.16) that # -shared alone still hits the same-UID SHM check even with a valid XAUTHORITY. cmd = [ x11vnc_path, "-display", display, "-rfbport", str(vnc_rfb_port), "-nopw", "-listen", "127.0.0.1", "-shared", "-nowf", "-noshm", "-speeds", "100", "-wait", "40", "-defer", "30", "-quiet" ] # This agent commonly runs as root (passwordless-sudo hosts), which has no # MIT-MAGIC-COOKIE of its own for a display that belongs to a logged-in, # non-root user -- x11vnc then fails XOpenDisplay and exits immediately, well # before the RFB port is ever bound. _find_xauthority_for_display locates that # user's cookie by finding the X server process actually serving this display and # reading its owner's .Xauthority, rather than assuming a fixed path. env = os.environ.copy() xauth_path = _find_xauthority_for_display(display) if xauth_path: env["XAUTHORITY"] = xauth_path x11vnc_log_path = os.path.join(tempfile.gettempdir(), f"applotus_x11vnc_{vnc_rfb_port}.log") with open(x11vnc_log_path, "wb") as log_f: vnc_proc = subprocess.Popen(cmd, stdout=log_f, stderr=subprocess.STDOUT, env=env) time.sleep(1.2) vnc_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) vnc_sock.settimeout(3.0) vnc_sock.connect(("127.0.0.1", vnc_rfb_port)) target_port = vnc_rfb_port except Exception as e: # x11vnc's own log is what actually explains a XOpenDisplay/auth failure -- the # "Connection refused" here is just the symptom (nothing ever bound the port). tail = "" if x11vnc_log_path: try: with open(x11vnc_log_path, "r", errors="replace") as log_f: tail = " | x11vnc: " + " / ".join(log_f.read().strip().splitlines()[-4:]) except Exception: pass print(f"[{time.strftime('%X')}] [-] Failed to start on-demand x11vnc: {e}{tail}", flush=True) if vnc_sock: vnc_sock.close() vnc_sock = None if vnc_sock is None: err_msg = f"No local VNC service found on port {target_port} and no display server helper available." print(f"[{time.strftime('%X')}] [-] {err_msg}", flush=True) try: ws.send_frame(json.dumps({"error": err_msg}), opcode=1) ws.close() except Exception: pass with active_desktop_lock: active_desktop_sessions.discard(session_id) return # Protocol check: Port 3389 is Microsoft RDP, not RFB/VNC. if target_port == 3389: err_msg = "Node desktop service is Microsoft RDP (Port 3389). In-browser HTML5 RFB viewer cannot negotiate directly with RDP. Please use the 'Download .rdp' launcher." print(f"[{time.strftime('%X')}] [!] {err_msg}", flush=True) try: ws.send_frame(json.dumps({"error": err_msg}), opcode=1) ws.close() vnc_sock.close() except Exception: pass with active_desktop_lock: active_desktop_sessions.discard(session_id) return # Socket buffer and low-latency tuning: try: vnc_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) vnc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) vnc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) except Exception: pass vnc_sock.setblocking(False) # Printed before the RFB handshake, so it cannot claim the session works -- node-antwin logged # "DESKTOP ACTIVE" immediately before failing authentication every time. print(f"[{time.strftime('%X')}] [🖥️ DESKTOP] Session {session_id} socket connected to " f"127.0.0.1:{target_port}; negotiating RFB", flush=True) # ── Darwin macOS: Intercept RFB auth negotiation ───────────────────────── # Apple Screen Sharing (ARD) presents auth-type 30 (Apple DH) and/or type 2 # (VNC password). noVNC only speaks types 1/2, and only type 2 is actually # implemented below -- type 30 is unofficial, reverse-engineered crypto with # no RFC, so a Mac offering *only* 30 (no VNC password set) gets a clear # error instead of a fake handshake. When 1 or 2 is available we do the # real RFB handshake here, then present the authenticated socket to the # browser as SecurityType None (1) — completely transparent to noVNC. if platform.system() in ("Darwin", "Windows"): # Pure-Python self-contained DES (RFC 6143 compliant) _IP = [58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, 62, 54, 46, 38, 30, 22, 14, 6, 64, 56, 48, 40, 32, 24, 16, 8, 57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7] _FP = [40, 8, 48, 16, 56, 24, 64, 32, 39, 7, 47, 15, 55, 23, 63, 31, 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25] _PC1 = [57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35, 27, 19, 11, 3, 60, 52, 44, 36, 63, 55, 47, 39, 31, 23, 15, 7, 62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4] _PC2 = [14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, 23, 19, 12, 4, 26, 8, 16, 7, 27, 20, 13, 2, 41, 52, 31, 37, 47, 55, 30, 40, 51, 45, 33, 48, 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32] _SHIFTS = [1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1] _E = [32, 1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9, 8, 9, 10, 11, 12, 13, 12, 13, 14, 15, 16, 17, 16, 17, 18, 19, 20, 21, 20, 21, 22, 23, 24, 25, 24, 25, 26, 27, 28, 29, 28, 29, 30, 31, 32, 1] _P = [16, 7, 20, 21, 29, 12, 28, 17, 1, 15, 23, 26, 5, 18, 31, 10, 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25] _S_BOX = [ [[14,4,13,1,2,15,11,8,3,10,6,12,5,9,0,7],[0,15,7,4,14,2,13,1,10,6,12,11,9,5,3,8],[4,1,14,8,13,6,2,11,15,12,9,7,3,10,5,0],[15,12,8,2,4,9,1,7,5,11,3,14,10,0,6,13]], [[15,1,8,14,6,11,3,4,9,7,2,13,12,0,5,10],[3,13,4,7,15,2,8,14,12,0,1,10,6,9,11,5],[0,14,7,11,10,4,13,1,5,8,12,6,9,3,2,15],[13,8,10,1,3,15,4,2,11,6,7,12,0,5,14,9]], [[10,0,9,14,6,3,15,5,1,13,12,7,11,4,2,8],[13,7,0,9,3,4,6,10,2,8,5,14,12,11,15,1],[13,6,4,9,8,15,3,0,11,1,2,12,5,10,14,7],[1,10,13,0,6,9,8,7,4,15,14,3,11,5,2,12]], [[7,13,14,3,0,6,9,10,1,2,8,5,11,12,4,15],[13,8,11,5,6,15,0,3,4,7,2,12,1,10,14,9],[10,6,9,0,12,11,7,13,15,1,3,14,5,2,8,4],[3,15,0,6,10,1,13,8,9,4,5,11,12,7,2,14]], [[2,12,4,1,7,10,11,6,8,5,3,15,13,0,14,9],[14,11,2,12,4,7,13,1,5,0,15,10,3,9,8,6],[4,2,1,11,10,13,7,8,15,9,12,5,6,3,0,14],[11,8,12,7,1,14,2,13,6,15,0,9,10,4,5,3]], [[12,1,10,15,9,2,6,8,0,13,3,4,14,7,5,11],[10,15,4,2,7,12,9,5,6,1,13,14,0,11,3,8],[9,14,15,5,2,8,12,3,7,0,4,10,1,13,11,6],[4,3,2,12,9,5,15,10,11,14,1,7,6,0,8,13]], [[4,11,2,14,15,0,8,13,3,12,9,7,5,10,6,1],[13,0,11,7,4,9,1,10,14,3,5,12,2,15,8,6],[1,4,11,13,12,3,7,14,10,15,6,8,0,5,9,2],[6,11,13,8,1,4,10,7,9,5,0,15,14,2,3,12]], [[13,2,8,4,6,15,11,1,10,9,3,14,5,0,12,7],[1,15,13,8,10,3,7,4,12,5,6,11,0,14,9,2],[7,11,4,1,9,12,14,2,0,6,10,13,15,3,5,8],[2,1,14,7,4,10,8,13,15,12,9,0,3,5,6,11]] ] def _perm(val, table, in_len): res = 0 for pos in table: res = (res << 1) | ((val >> (in_len - pos)) & 1) return res def _des_keys(key_bytes): k56 = _perm(int.from_bytes(key_bytes, "big"), _PC1, 64) c, d = (k56 >> 28) & 0x0FFFFFFF, k56 & 0x0FFFFFFF subkeys = [] for s in _SHIFTS: c = ((c << s) | (c >> (28 - s))) & 0x0FFFFFFF d = ((d << s) | (d >> (28 - s))) & 0x0FFFFFFF subkeys.append(_perm((c << 28) | d, _PC2, 56)) return subkeys def _des_block(block_bytes, subkeys): b = _perm(int.from_bytes(block_bytes, "big"), _IP, 64) l, r = (b >> 32) & 0xFFFFFFFF, b & 0xFFFFFFFF for k in subkeys: er = _perm(r, _E, 32) ^ k s_out = 0 for i in range(8): chunk = (er >> (42 - i * 6)) & 0x3F row = ((chunk & 0x20) >> 4) | (chunk & 1) col = (chunk >> 1) & 0x0F s_out = (s_out << 4) | _S_BOX[i][row][col] f = _perm(s_out, _P, 32) l, r = r, l ^ f res = _perm((r << 32) | l, _FP, 64) return res.to_bytes(8, "big") def _vnc_des_response(password, challenge): """Pure Python standard DES ECB encrypt for VNC challenge (zero external dependencies).""" pw_bytes = (password.encode("utf-8") + b"\x00" * 8)[:8] def _rev_bits(b): r = 0 for _ in range(8): r = (r << 1) | (b & 1) b >>= 1 return r vnc_key = bytes(_rev_bits(b) for b in pw_bytes) subkeys = _des_keys(vnc_key) return _des_block(challenge[:8], subkeys) + _des_block(challenge[8:16], subkeys) def _recv_exact(sock, n): buf = b"" while len(buf) < n: chunk = sock.recv(n - len(buf)) if not chunk: raise RuntimeError("VNC socket closed during handshake") buf += chunk return buf def _darwin_report_error(msg): """Tells the browser's noVNC client the connection failed, instead of leaving it to hang waiting on a handshake that will never complete. Reuses noVNC's own error-display path: a normal SecurityTypes message advertising 0 types is its documented signal for "connection failed, here's why" (RFC 6143 6.1.1).""" try: ws.send_frame(b"RFB 003.008\n", opcode=2) ws.send_frame(bytes([0]), opcode=2) # 0 security types = error err_bytes = msg.encode("utf-8") ws.send_frame(len(err_bytes).to_bytes(4, "big") + err_bytes, opcode=2) except Exception: pass try: vnc_sock.close() except Exception: pass # Set once the real server's ProtocolVersion has been read -- past that point the socket # is mid-negotiation and bytes already consumed can never be handed back to noVNC, so an # exception from here on can no longer be recovered by a "raw bridge" passthrough (see the # except block below). _handshake_started = False try: vnc_sock.setblocking(True) vnc_sock.settimeout(5.0) # Step 1: Read server ProtocolVersion (12 bytes: "RFB 003.008\n") proto = _recv_exact(vnc_sock, 12) _handshake_started = True # Reply with RFB 3.8 vnc_sock.sendall(b"RFB 003.008\n") # Step 2: Read SecurityTypes num_types = _recv_exact(vnc_sock, 1)[0] if num_types == 0: raise RuntimeError("VNC server sent 0 security types") types_data = _recv_exact(vnc_sock, num_types) supported = list(types_data) print(f"[{time.strftime('%X')}] [🖥️ RFB {platform.system()}] Offered auth types: " f"{_describe_auth_types(supported)}", flush=True) auth_ok = False if 1 in supported: # SecurityType None — no challenge needed vnc_sock.sendall(bytes([1])) result = _recv_exact(vnc_sock, 4) auth_ok = (result == b'\x00\x00\x00\x00') print(f"[{time.strftime('%X')}] [🖥️ RFB {platform.system()}] Auth None: {'OK' if auth_ok else 'FAIL'}", flush=True) elif 2 in supported: # SecurityType VNC Password vnc_sock.sendall(bytes([2])) challenge = _recv_exact(vnc_sock, 16) pw = vnc_password or "" response = _vnc_des_response(pw, challenge) vnc_sock.sendall(response) result = _recv_exact(vnc_sock, 4) auth_ok = (result == b'\x00\x00\x00\x00') print(f"[{time.strftime('%X')}] [🖥️ RFB {platform.system()}] VNC DES auth ({'blank' if not pw else 'stored'} pw): {'OK' if auth_ok else 'FAIL — check password'}", flush=True) if not auth_ok: try: reason_len = int.from_bytes(_recv_exact(vnc_sock, 4), 'big') reason = _recv_exact(vnc_sock, reason_len).decode('utf-8', errors='replace') print(f"[{time.strftime('%X')}] [!] RFB {platform.system()} auth failure reason: {reason}", flush=True) except Exception: pass # Close socket and notify browser so it doesn't hang in negotiation _darwin_report_error("VNC Authentication Failed. Please verify or re-set the VNC password.") return else: # No password-based auth on offer -- almost always type 30 (Apple Diffie-Hellman), # which macOS Screen Sharing falls back to when "VNC viewers may control screen # with password" is left unchecked and only login-based auth is configured. That # scheme isn't implemented here (unofficial, reverse-engineered by third parties, # no RFC), so continuing would mean picking a type we can't actually authenticate: # the real Mac sits waiting for a DH exchange that never comes, times out, and # closes -- which used to surface to the user as "connected, then immediately # disconnected" since this code told the browser auth had already succeeded. Fail # loudly and immediately instead, with the actual fix (enable a VNC password) named. print(f"[{time.strftime('%X')}] [!] RFB ({platform.system()}): no usable auth type " f"offered — {_describe_auth_types(supported)} (need 1=None or 2=VNC password)", flush=True) # Platform-specific, because the same condition has entirely different causes and # entirely different fixes on the two systems -- and a Windows user was being told # about "this Mac's Screen Sharing", which is how node-antwin stayed mysterious. if platform.system() == "Windows": # Name the server actually installed. Asserting TightVNC here was wrong on # node-antwin, which offers [17, 117, 118, 113] -- no type 16, so not TightVNC at # all -- and the TightVNC-specific advice that followed pointed at registry values # nothing on that host reads. _svc, _st = _windows_vnc_service() _which = ("The VNC service installed here is %s (%s)." % (_svc, _st)) if _svc \ else "No recognised VNC service (TightVNC/UltraVNC/RealVNC) was found by name." _darwin_report_error( "The VNC server on this Windows host offers only authentication types this " "agent cannot use: %s. %s Applotus can authenticate only type 1 (None) or " "type 2 (VNC password), so enable standard VNC authentication on that server, " "set a password, and store the same password against this node in Applotus." % (_describe_auth_types(supported), _which) ) else: _darwin_report_error( "This Mac's Screen Sharing doesn't have a VNC password set, so it only offers " "Apple's login-based authentication, which isn't supported here. Use the " "\"Enable Desktop\" action (or set a VNC password manually) and try again." ) return if auth_ok: # Synchronize RFB handshake with browser: # 1. Send ProtocolVersion to browser ws.send_frame(b"RFB 003.008\n", opcode=2) # 2. Wait for browser ProtocolVersion response (e.g. b"RFB 003.008\n") _op, _client_proto = ws.read_frame() # 3. Send SecurityTypes to browser: 1 type available, type 1 (None) ws.send_frame(bytes([1, 1]), opcode=2) # 4. Wait for browser SecurityType selection (b"\x01") _op, _client_sec = ws.read_frame() # 5. Send SecurityResult OK (0) to browser ws.send_frame(b'\x00\x00\x00\x00', opcode=2) # 6. Wait for browser ClientInit (1 byte shared flag) _op, _client_init = ws.read_frame() if _client_init: # Forward browser's ClientInit directly to macOS ARD socket vnc_sock.sendall(_client_init) print(f"[{time.strftime('%X')}] [🖥️ RFB {platform.system()}] Synthetic handshake complete & ClientInit bridged — streaming desktop", flush=True) vnc_sock.setblocking(False) except Exception as _darwin_err: if _handshake_started: # Bytes have already been read from and/or written to the real socket -- it is # mid-negotiation from the server's point of view, so there is no way to hand it to # noVNC for a fresh handshake of its own; that used to be attempted here ("raw # bridge fallback") and just produced a silently desynced, hanging connection. # Reporting the failure is the only option that leaves the user with an explanation # instead of a mysterious drop. print(f"[{time.strftime('%X')}] [!] RFB {platform.system()} handshake failed mid-negotiation: {_darwin_err}", flush=True) _darwin_report_error(f"Remote desktop connection failed: {_darwin_err}") return # Nothing has been read or written yet (e.g. the very first read timed out) -- the # socket is untouched, so letting noVNC negotiate directly against it from scratch is # still coherent. print(f"[{time.strftime('%X')}] [!] RFB {platform.system()} auth failed before handshake began: {_darwin_err} — raw bridge fallback", flush=True) try: vnc_sock.setblocking(False) except Exception: pass # ── End Darwin auth ─────────────────────────────────────────────────────── # An explicitly CHOSEN resolution is applied here, after the framebuffer is live -- not in the # pre-attach block near the top of this function. # # That block is gated off on Windows for a measured reason: with the gate open, every session on # winimac-ant attached normally and showed no picture, and neither reordering nor a connect retry # saved it. A mode switch immediately before the attach is what breaks it there. # # Changing the mode on an ALREADY-LIVE session is a different thing, and it demonstrably works on # that same node -- it is what the resolution dropdown does, via handle_set_display_mode, which # passes the force flag and bypasses the gate. A stored preference is the same kind of act as # that click (someone decided), so it takes the same path rather than the inference path. # # Threaded with a delay because the relay below has to be pumping for the client to receive the # DesktopSize update, and because handle_set_display_mode can take tens of seconds on a # session-0 agent where it has to go through a scheduled task in the console session. if desired_from_preference and desired_width and desired_height: def _apply_preferred_mode(): try: time.sleep(3.0) # No cap arguments: the server clamped this to its box before dispatching, so # re-capping here could only disagree with what it already decided. res = handle_set_display_mode(desired_width, desired_height) if res.get("applied"): print(f"[{time.strftime('%X')}] [🖥️ DISPLAY] Applied the stored preference " f"{desired_width}x{desired_height} to session {session_id}: " f"{res.get('reason')}", flush=True) else: # Reported, not swallowed. A preference that silently fails to apply is the bug # this whole change exists to fix, and repeating it quietly would be worse. print(f"[{time.strftime('%X')}] [!] Stored preference " f"{desired_width}x{desired_height} was not applied to session " f"{session_id}: {res.get('reason')}", flush=True) except Exception as _pref_err: print(f"[{time.strftime('%X')}] [!] Applying the stored preference raised: " f"{_pref_err!r}", flush=True) threading.Thread(target=_apply_preferred_mode, daemon=True).start() try: bridge_socket_over_ws(vnc_sock, ws, label="VNC", eof_hint="Check VNC server AllowLoopback setting or service status.") finally: if vnc_sock: try: vnc_sock.close() except Exception: pass if vnc_proc: try: vnc_proc.terminate() time.sleep(0.2) vnc_proc.kill() except Exception: pass if ws: ws.close() with active_desktop_lock: active_desktop_sessions.discard(session_id) print(f"[{time.strftime('%X')}] [🖥️ DESKTOP CLOSED] Session {session_id} ended", flush=True) # Active on-demand containers this agent process has provisioned: containerId -> {name, runtime, # hostPorts, expiresAt, persistent, ...}. Mirrored to disk (see _container_state_path) so a restart # keeps managing what it provisioned instead of abandoning it — without that, a container marked # persistent silently stopped being watched the moment the agent was upgraded or restarted, which # is precisely the window the guarantee is supposed to cover. active_containers = {} active_containers_lock = threading.Lock() # One lock per container id, created on demand (same pattern as _conn_pool_locks below). Lets two # dispatched tasks that target *different* containers run at once on the task pool while still # forcing tasks against the *same* container -- hibernate then wake, two env updates -- to apply # in the order the server queued them, rather than racing each other's podman/exec calls. _container_task_locks = {} _container_task_locks_master = threading.Lock() def _container_task_lock(container_id): with _container_task_locks_master: lock = _container_task_locks.get(container_id) if lock is None: lock = threading.Lock() _container_task_locks[container_id] = lock return lock # Where the manager mirrors its registry. Under the agent user's home rather than /var/lib so a # rootless agent can always write it. The filename carries the node id because more than one host # agent can run as the same user (different servers, different tenants) — a single shared file # would have each one loading and then deleting the other's containers on its first sweep. # Configuration environment variables are SENTINEL_* , aligned with the product name. The former # APPLOTUS_* names are still read as a fallback and still written when launching an in-container # agent, because both directions of a mixed-version fleet matter: a host agent that has not upgraded # yet passes only APPLOTUS_*, and a baked image may carry an older agent that only reads APPLOTUS_*. # Getting that wrong is not cosmetic -- an in-container agent that misses SENTINEL_NODE_KIND # registers as a standalone host node instead of a child of its host, which is silent and wrong. def _cfg(name, default=""): return os.environ.get("SENTINEL_" + name) or os.environ.get("APPLOTUS_" + name) or default CONTAINER_STATE_DIR = _cfg("STATE_DIR") or os.path.join(os.path.expanduser("~"), ".applotus") # Operating mode. "full" is every capability this agent has and is the default, so no existing # fleet node changes behaviour. "readonly" is for a host where the agent is a monitor rather than a # control plane -- specifically the agent running inside the Applotus server's own container, where # a shell would let anyone with the admin tenant read the session store on /data and mint a session # for any user in any tenant. AGENT_MODE = (_cfg("MODE", "full") or "full").strip().lower() # Allowlists, not denylists. A denylist grants every capability added after it is written, which is # the wrong default for a security boundary: a task type introduced next month should be refused # until someone decides it is safe, not permitted because nobody remembered to add it here. # # Both entries are pure reads that the fleet view needs to be useful at all -- the agent's own log # tail and its own metrics history. Everything else, including the "shell" default, is refused. READONLY_TASK_TYPES = frozenset(( "get_agent_logs", "get_metrics_history", # A TCP connect attempt against another node in the same workspace. Readonly-safe because it # changes nothing on this host, and it cannot be aimed anywhere interesting: the server only # ever dispatches targets that are another of this tenant's own nodes' reported LAN addresses # (see /api/sentinel/nodes/:nodeId/probe-lan), so this is not a general-purpose port scanner. "probe_lan_peer", )) # get_display_modes only reads; set_display_mode writes and is deliberately absent. list_files and # download_file are handled separately in _capability_allowed, because whether they are safe # depends on a file root being configured rather than on the mode alone. READONLY_CONTROL_TYPES = frozenset(( "get_display_modes", )) # Confines the file browser to a subtree. Unset means unconfined, which is the historical behaviour # and is right for a workstation whose owner already has a shell on it. In readonly mode a root is # *required* before any file read is served: unjailed access to this particular container's # filesystem is equivalent to the shell the mode exists to refuse. FILE_ROOT = (_cfg("FILE_ROOT", "") or "").strip() def _file_root_real(): """The configured file root with symlinks resolved. Three outcomes, deliberately distinct: None for unconfined, False for a root that is set but unresolvable, and the path otherwise. False must not collapse into None -- a misconfigured root that degraded to "no jail" would quietly hand over the whole filesystem, which is the opposite of what setting it asked for. """ if not FILE_ROOT: return None try: return os.path.realpath(os.path.expanduser(FILE_ROOT)) except Exception: return False def _jail_path(target_path): """(resolved, error) for a caller-supplied path under the file root. Returns (None, None) when no root is configured, meaning the caller should resolve the path itself exactly as it always has. realpath on both sides, not a prefix test on the raw string: a symlink inside the root pointing anywhere else would pass startswith() while reading from outside, which is the standard way this kind of check gets defeated. """ root = _file_root_real() if root is None: return (None, None) if root is False: return (None, "The configured file root could not be resolved") if not target_path or target_path in ("", "~", "."): return (root, None) try: p = os.path.realpath(os.path.abspath(os.path.expanduser(target_path))) except Exception as e: return (None, "Path could not be resolved: %s" % e) if p == root or p.startswith(root + os.sep): return (p, None) return (None, "Path is outside the permitted root for this node") def _capability_allowed(kind, name): """Whether a task type or control message may run under the current mode. kind is "task" or "control"; they are separate namespaces with overlapping names (start_pty arrives both ways), and collapsing them would let an allowance on one channel leak to the other. """ if AGENT_MODE != "readonly": return True if kind == "task": return name in READONLY_TASK_TYPES if name in READONLY_CONTROL_TYPES: return True if name in ("list_files", "download_file"): # Only with a usable root configured; _jail_path then enforces it. A root that is set but # unresolvable (False) is refused here too rather than treated as absent. return isinstance(_file_root_real(), str) return False def _capability_refusal(name): return ("Refused: this node's agent runs in read-only mode and will not perform '%s'. " "Set SENTINEL_MODE=full on the agent to allow it." % name) # Each control message's reply type, so a refusal reaches the caller in the shape it is waiting for # instead of leaving it to time out. Messages absent here carry no reqId and expect no reply. CONTROL_REPLY_TYPES = { "list_files": "list_files_response", "download_file": "download_file_response", "get_display_modes": "get_display_modes_result", "set_display_mode": "set_display_mode_result", # Writes, so deliberately absent from READONLY_CONTROL_TYPES above: a read-only node refuses # all four, and the refusal arrives in the shape the caller is waiting on rather than as a # timeout. "upload_file_begin": "upload_file_begin_response", "upload_file_chunk": "upload_file_chunk_response", "upload_file_end": "upload_file_end_response", "set_clipboard": "set_clipboard_response", } def _container_state_path(): safe = "".join(c for c in str(_manager_node_id or "default") if c.isalnum() or c in "-_")[:80] return os.path.join(CONTAINER_STATE_DIR, f"containers-{safe}.json") # How often the manager sweeps: expiry, liveness, and backup scheduling all ride this tick. 15s is # well under the server's 90s liveness threshold, so a restart is visible in the UI within one # heartbeat, and cheap enough to run on a small VPS. # Naming for everything this agent creates on a node. Constants rather than inline strings because # the prefix is the only handle for finding agent-spawned resources once the on-disk registry is # gone (agent reinstalled, host restored from an image, registry deleted) — which is precisely the # situation a cleanup exists to resolve. CONTAINER_NAME_PREFIX = "sntl-" CONTAINER_BACKUP_REPO = "sentinel-backup" CONTAINER_NETWORK_PREFIX = "sentinel-" # Agents before 1.11.0 named everything "applotus-*"; before this rename, "sentinel-*". Cleanup # matches all of them so a long-lived host still finds containers, snapshots and networks that # predate either rename. LEGACY_NAME_PREFIXES = ("applotus-", "sentinel-") MANAGED_NAME_PREFIXES = (CONTAINER_NAME_PREFIX,) + LEGACY_NAME_PREFIXES MANAGED_BACKUP_REPOS = (CONTAINER_BACKUP_REPO, "applotus-backup") MANAGED_NETWORK_PREFIXES = (CONTAINER_NETWORK_PREFIX,) + LEGACY_NAME_PREFIXES CONTAINER_MANAGER_INTERVAL = 15 # Snapshot cadence and retention for persistent containers. Snapshots are runtime image commits, # not volume copies — this agent never mounts host volumes into workloads by design, so the # container's own filesystem is the only thing there is to preserve. CONTAINER_BACKUP_INTERVAL = 900 # 15 min CONTAINER_BACKUP_RETAIN = 3 # Ceiling on a single extend, and on how far out any expiry may be pushed. A container that can be # extended without bound is just a persistent container with extra steps — that is what the # persistent flag is for, and it is the flag that turns on backups and self-healing. CONTAINER_MAX_EXTEND_SECONDS = 24 * 3600 CONTAINER_MAX_LIFETIME_SECONDS = 7 * 24 * 3600 # Set once by the main loop (and by the first provision) so the manager thread can report expiry, # restarts and backups back to the server. Deliberately module-level rather than stored per # container: it carries the agent's API key, and the on-disk registry must stay secret-free. _manager_report_ctx = {} _manager_node_id = "" _container_manager_started = False def _set_manager_context(report_ctx, node_id): """Records how the manager thread should talk to the server, and starts it on first call.""" global _manager_report_ctx, _manager_node_id, _container_manager_started if isinstance(report_ctx, dict) and report_ctx.get("server_url"): _manager_report_ctx = dict(report_ctx) if node_id: _manager_node_id = node_id if not _container_manager_started: _container_manager_started = True threading.Thread(target=_container_manager_loop, daemon=True).start() print(f"[{time.strftime('%X')}] [📦 CONTAINER MANAGER] Started " f"(sweep every {CONTAINER_MANAGER_INTERVAL}s)", flush=True) # Keys that are safe to mirror to disk. The creation argv (which carries database bootstrap # passwords in -e flags) and anything else secret-bearing is deliberately excluded: a restarted # agent can still watch, restart and back up a container, it just cannot rebuild one from scratch. _PERSISTED_CONTAINER_KEYS = ( "name", "runtime", "runtimePath", "hostPorts", "expiresAt", "persistent", "image", "serviceAlias", "createdAt", "ttlSeconds", "restarts", "lastBackupAt", "nextBackupAt", "backups", "agentNodeId", "restartPolicyApplied", "network", "withAgent", "containerPorts", "hibernated", "hibernatedAt", "recoverySpec", "cpus", "memoryMb", ) def _provisioned_key_path(): """Where a server-issued node key is cached. Sits beside the container registry so both share the same state dir semantics (STATE_DIR override, ~/.applotus default).""" return os.path.join(CONTAINER_STATE_DIR, "node-key.json") def load_provisioned_api_key(): """A node key previously issued by the server, if any. Returning None just means the agent falls back to whatever credential it was started with.""" try: with open(_provisioned_key_path()) as fh: data = json.load(fh) key = (data or {}).get("api_key") or None return key if isinstance(key, str) and key.startswith("al_live_") else None except Exception: return None def save_provisioned_api_key(api_key): """Persist a server-issued node key so a restart keeps authenticating with a real credential instead of falling back to the x-tenant-id header (which the server now rejects for everything except the heartbeat itself). Same atomic tmp+chmod+replace dance as the container registry -- this file holds a credential, so it is written 0600 and never left half-written.""" try: os.makedirs(CONTAINER_STATE_DIR, exist_ok=True) path = _provisioned_key_path() tmp_path = path + ".tmp" with open(tmp_path, "w") as fh: json.dump({"version": 1, "api_key": api_key}, fh) try: os.chmod(tmp_path, 0o600) except Exception: pass os.replace(tmp_path, path) return True except Exception as e: print(f"[{time.strftime('%X')}] [!] Could not persist issued node key: {e}", flush=True) return False def _save_container_state(): """Mirrors the registry to disk. Best-effort: losing the mirror degrades restart survival but must never take down provisioning, so every failure here is swallowed after one log line.""" try: with active_containers_lock: snapshot = { cid: {k: v for k, v in info.items() if k in _PERSISTED_CONTAINER_KEYS} for cid, info in active_containers.items() } os.makedirs(CONTAINER_STATE_DIR, exist_ok=True) state_path = _container_state_path() tmp_path = state_path + ".tmp" with open(tmp_path, "w") as fh: json.dump({"version": 1, "containers": snapshot}, fh) try: os.chmod(tmp_path, 0o600) except Exception: pass os.replace(tmp_path, state_path) except Exception as e: print(f"[{time.strftime('%X')}] [!] Could not persist container registry: {e}", flush=True) def _load_container_state(): """Restores the registry after an agent restart. Entries whose container no longer exists are dropped on the manager's first sweep, so a stale file cannot resurrect phantom nodes.""" try: state_path = _container_state_path() if not os.path.isfile(state_path): return with open(state_path) as fh: data = json.load(fh) restored = data.get("containers") or {} if not isinstance(restored, dict): return with active_containers_lock: for cid, info in restored.items(): if isinstance(info, dict) and cid not in active_containers: active_containers[cid] = info if restored: print(f"[{time.strftime('%X')}] [📦 CONTAINER MANAGER] Resumed management of " f"{len(restored)} container(s) from disk", flush=True) except Exception as e: print(f"[{time.strftime('%X')}] [!] Could not read container registry: {e}", flush=True) def _report_container_event(result_type, container_id, payload, ok=True, error=""): """Posts a manager-initiated container change to /agent/result so the server's node record tracks it. The manager acts on its own schedule, outside any dispatched task, so these results carry a synthetic taskId — the server keys container updates off containerId, not taskId.""" ctx = _manager_report_ctx if not ctx or not ctx.get("server_url"): return try: send_request( f"{ctx['server_url']}/api/sentinel/agent/result", data={ "nodeId": _manager_node_id, "taskId": f"mgr-{result_type}-{container_id[:12]}-{int(time.time())}", "type": result_type, "containerId": container_id, "exit_code": 0 if ok else 1, "stdout": json.dumps(payload) if ok else "", "stderr": "" if ok else str(error)[:2000], "duration": 0, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": ctx.get("tenant_id"), }, insecure=ctx.get("insecure", True), host_header=ctx.get("host_header"), tenant_id=ctx.get("tenant_id"), api_key=ctx.get("api_key"), ) except Exception as e: print(f"[{time.strftime('%X')}] [!] Failed to report {result_type} for " f"{container_id[:12]}: {e}", flush=True) def _runtime_path_for(info): return info.get("runtimePath") or info.get("runtime") def _node_stub_for_name(node_id): """The part of a node id worth putting in a container name -- strips a leading "node-" (the host node id convention elsewhere in this app), since embedding the whole thing produced names like "sntl-node-popos-col-..." where "node" says nothing a container name doesn't already imply. A container's own parent id (e.g. "container-8df4a07dd597") has no such prefix and passes through unchanged.""" s = str(node_id or "") if s.startswith("node-"): s = s[len("node-"):] return s[:24] def _node_name_prefixes(node_id): """Name prefixes that belong to *this* node specifically. Container names are f"{prefix}{stub}-{ts}-{hex}", so the node id is recoverable from the name. Scoping to it matters because more than one agent can run on one host (different servers, different tenants — this is normal, not exotic): a host-wide prefix scan would have one node's cleanup destroy another node's containers, which is the same class of mistake as reaching outside Applotus entirely. Checks both the stripped stub (current naming) and the full, unstripped one: every prior naming scheme embedded the whole node id including its own "node-" prefix, so a container named under any of them ("sentinel-node-popos-col-...", "applotus-node-popos-col-...") would otherwise stop being recognized as this node's own the moment this changed -- silently orphaning it from cleanup rather than just leaving a cosmetic old name in place.""" full_stub = str(node_id or "")[:24] if not full_stub: return () stubs = {full_stub, _node_stub_for_name(node_id)} return tuple(p + s for p in MANAGED_NAME_PREFIXES for s in stubs if s) def _managed_containers_on_host(runtime_path, node_id): """Containers on this host that *this node's* agent created, found by name rather than by the registry. The registry only knows what the current agent process provisioned; an agent that was reinstalled, upgraded, or restored from an image has an empty registry while its containers keep running. Those orphans are the whole reason cleanup exists, so it cannot rely on the registry.""" prefixes = _node_name_prefixes(node_id) if not runtime_path or not prefixes: return [] try: res = subprocess.run([runtime_path, "ps", "-a", "--format", "{{.ID}}\t{{.Names}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=30) if res.returncode != 0: return [] except Exception: return [] found = [] for line in (res.stdout or "").splitlines(): cid, _, name = line.partition("\t") cid, name = cid.strip(), name.strip() # Names can carry a leading '/' depending on runtime and format verb. bare = name.lstrip("/") if cid and bare.startswith(prefixes): found.append((cid, bare)) return found def cleanup_node_resources(runtime_info, node_id="", tenant_id=None, stop_agent=False): """Terminates everything this agent spawned on the node and reports what was reclaimed. Scoped three ways, and every one of them matters on a shared host: to Applotus' own name prefixes, to *this node id* within them, and — for networks — to this tenant. A container, image or network belonging to another agent, another tenant, or nothing to do with Applotus is never touched. A cleanup that reaches outside its own scope is worse than no cleanup at all. Persistent containers are included. This runs when a node is being removed from the fleet, so leaving a container alive to honour a guarantee scoped to a node that no longer exists is exactly the leak the action is meant to prevent.""" report = { "containers": [], "containersFailed": [], "images": [], "networks": [], "ptySessions": 0, "desktopSessions": 0, "agentStopped": False, } runtime_path = (runtime_info or {}).get("path") or (runtime_info or {}).get("runtime") # ── Containers: registry ∪ this node's prefix scan ──────────────────────── with active_containers_lock: registry = {cid: dict(info) for cid, info in active_containers.items()} targets = {} for cid, info in registry.items(): targets[cid] = info.get("name") or cid[:12] # `podman ps` reports 12-char ids while the registry holds full 64-char ones, so a plain # setdefault does not dedupe and the same container gets processed twice. The second pass then # "succeeds" because `rm -f` exits 0 on an already-gone container, inflating the reclaimed count. for cid, name in _managed_containers_on_host(runtime_path, node_id): if any(k == cid or k.startswith(cid) or cid.startswith(k) for k in targets): continue targets[cid] = name for cid, name in targets.items(): removed = False # Prefer remove_container so registry bookkeeping and TTL state unwind properly. if cid in registry: removed = remove_container(cid).get("success", False) if not removed and runtime_path: try: r = subprocess.run([runtime_path, "rm", "-f", cid], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, timeout=60) removed = r.returncode == 0 except Exception: removed = False (report["containers"] if removed else report["containersFailed"]).append( {"containerId": cid[:12], "name": name}) # ── Snapshot images for this node's containers ──────────────────────────── # Snapshot tags are f"{repo}/{container_name}:{ts}", and container_name carries the node id, so # the same node scoping applies. Another node's snapshots are left alone. if runtime_path: node_stubs = _node_name_prefixes(node_id) try: r = subprocess.run([runtime_path, "images", "--format", "{{.Repository}}:{{.Tag}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=45) for line in (r.stdout or "").splitlines() if r.returncode == 0 else []: ref = line.strip() if not ref: continue # Repository may be registry-qualified (localhost/sentinel-backup/foo), so match on # a path segment rather than the string start. repo_hit = next((repo for repo in MANAGED_BACKUP_REPOS if ("/" + repo + "/") in ("/" + ref)), None) if not repo_hit: continue tail = ref.split(repo_hit + "/", 1)[1] if (repo_hit + "/") in ref else "" if not tail.startswith(node_stubs): continue try: d = subprocess.run([runtime_path, "rmi", "-f", ref], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=90) if d.returncode == 0: report["images"].append(ref) except Exception: pass except Exception: pass # ── This tenant's network, once nothing is attached ─────────────────────── # Named exactly rather than matched by prefix: the network is shared by every node of a tenant, # and a prefix scan on a host serving two tenants would take out the other tenant's network. # `network rm` refuses while containers are attached, which is what protects a sibling node of # the same tenant on this host. if runtime_path and tenant_id: safe_tenant = "".join(c for c in str(tenant_id) if c.isalnum() or c in "_-")[:60] candidates = [p + safe_tenant for p in MANAGED_NETWORK_PREFIXES] for net in candidates: try: d = subprocess.run([runtime_path, "network", "rm", net], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=45) if d.returncode == 0: report["networks"].append(net) except Exception: pass # ── Live sessions ───────────────────────────────────────────────────────── # Counted, not individually torn down: the PTY children and WebSocket sockets are owned by their # own threads and there is no handle registry to close them through. Stopping the agent is what # actually ends them — every socket and child process dies with the process — so the count is # reported and the honest caveat is that leaving the agent up leaves them up. with active_pty_lock: report["ptySessions"] = len(active_pty_sessions) with active_desktop_lock: report["desktopSessions"] = len(active_desktop_sessions) _save_container_state() try: os.remove(_container_state_path()) except Exception: pass print(f"[{time.strftime('%X')}] [\U0001F9F9 NODE CLEANUP] " f"containers={len(report['containers'])} failed={len(report['containersFailed'])} " f"images={len(report['images'])} networks={len(report['networks'])} " f"pty={report['ptySessions']} desktop={report['desktopSessions']} " f"stop_agent={stop_agent}", flush=True) report["agentStopped"] = bool(stop_agent) return report def stop_agent_soon(delay=2.0): """Exits the process after the caller has had time to POST its result. A hard _exit rather than a graceful unwind: PTY children, WebSocket pumps and the manager thread are all daemon threads with sockets open, and the point of this path is that nothing survives it. Note a service manager (systemd, the nohup wrapper's supervisor) may start the agent back up — the server says so in the UI rather than pretending this is permanent.""" def _bye(): time.sleep(delay) print(f"[{time.strftime('%X')}] [\U0001F6D1 AGENT STOPPING] Requested by operator after node cleanup.", flush=True) sys.stdout.flush() os._exit(0) threading.Thread(target=_bye, daemon=True).start() def _container_liveness(runtime_path, container_id): """Returns 'running', 'stopped', 'missing', or None when the runtime could not be asked. Distinct from _container_state, which reports (status, exitCode) for the shell/exec paths and cannot tell a vanished container apart from a runtime that failed to answer. The manager has to make that distinction: one means rebuild, the other means try again next sweep. None and 'missing' are kept distinct on purpose: a transient CLI failure must not be read as 'the container is gone', which would trigger a pointless recreate on a container that is fine.""" if not runtime_path: return None try: res = subprocess.run([runtime_path, "inspect", "-f", "{{.State.Running}}", container_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=20) except Exception: return None if res.returncode != 0: err = (res.stderr or "").lower() if "no such" in err or "not found" in err or "no such object" in err: return "missing" return None return "running" if (res.stdout or "").strip().lower().startswith("true") else "stopped" def _container_restart_policy(runtime_path, container_id): """The restart policy the runtime actually has recorded, which is not always the one we asked for — `update --restart` is a podman 5 feature, and on 4.x the request is silently a no-op.""" if not runtime_path: return None try: res = subprocess.run([runtime_path, "inspect", "-f", "{{.HostConfig.RestartPolicy.Name}}", container_id], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=20) if res.returncode != 0: return None return (res.stdout or "").strip().lower() or None except Exception: return None def _apply_restart_policy(runtime_path, container_id, policy="always"): """Hands the restart guarantee to the runtime itself so it survives the agent being stopped and the host rebooting. `update --restart` is not available on every podman/docker build (podman added it in 5.0), so this reports whether the runtime really took it — the manager's own sweep is the fallback, and the UI says which of the two is holding the container up.""" if not runtime_path: return False try: res = subprocess.run([runtime_path, "update", "--restart", policy, container_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30) if res.returncode != 0: return False except Exception: return False return _container_restart_policy(runtime_path, container_id) == policy def backup_container(container_id, reason="scheduled"): """Snapshots a container's filesystem as a local image and prunes old snapshots. A commit rather than a volume copy because provision_container never mounts host volumes into a workload (that guardrail is what keeps a tenant container off the host filesystem), so the container's own layer is the entirety of its state. The snapshot is a normal local image, so recovery is a plain `run` against it.""" with active_containers_lock: info = active_containers.get(container_id) info = dict(info) if info else None if not info: return {"success": False, "error": "Unknown container — not managed by this agent."} runtime_path = _runtime_path_for(info) state = _container_liveness(runtime_path, container_id) if state == "missing": return {"success": False, "error": "Container no longer exists on this host."} safe_name = "".join(c for c in str(info.get("name") or container_id[:12]) if c.isalnum() or c in "-_").lower() tag = f"{CONTAINER_BACKUP_REPO}/{safe_name}:{int(time.time())}" try: # --pause is podman/docker default for commit and gives a consistent layer; it is a # sub-second freeze, not a stop, so a persistent service stays up across the snapshot. res = subprocess.run([runtime_path, "commit", container_id, tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=300) except subprocess.TimeoutExpired: return {"success": False, "error": "Snapshot timed out after 300s."} except Exception as e: return {"success": False, "error": str(e)} if res.returncode != 0: return {"success": False, "error": (res.stderr or res.stdout or "commit failed").strip()[:2000]} now = time.time() with active_containers_lock: live = active_containers.get(container_id) if live is not None: backups = list(live.get("backups") or []) backups.append({"tag": tag, "at": now, "reason": reason}) # Prune oldest beyond retention. Untagging is best-effort: a snapshot the runtime # refuses to drop (in use by a recovered container) just stays on disk. while len(backups) > CONTAINER_BACKUP_RETAIN: old = backups.pop(0) try: subprocess.run([runtime_path, "rmi", old.get("tag", "")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60) except Exception: pass live["backups"] = backups live["lastBackupAt"] = now live["nextBackupAt"] = now + CONTAINER_BACKUP_INTERVAL _save_container_state() print(f"[{time.strftime('%X')}] [📦 CONTAINER SNAPSHOT] {info.get('name')} " f"({container_id[:12]}) -> {tag} ({reason})", flush=True) return {"success": True, "tag": tag, "at": now, "reason": reason, "retain": CONTAINER_BACKUP_RETAIN} def extend_container_ttl(container_id, add_seconds): """Pushes a container's expiry out. Extending an already-expired-but-still-present container is allowed — the sweep may not have reaped it yet, and 'I got to it in time' is the whole point of the button. A persistent container has no expiry to extend and is reported as such rather than silently doing nothing.""" try: add_seconds = int(add_seconds) except (TypeError, ValueError): return {"success": False, "error": "addSeconds must be a whole number of seconds."} if add_seconds <= 0: return {"success": False, "error": "addSeconds must be positive."} add_seconds = min(add_seconds, CONTAINER_MAX_EXTEND_SECONDS) now = time.time() with active_containers_lock: info = active_containers.get(container_id) if not info: return {"success": False, "error": "Unknown container — not managed by this agent."} if info.get("persistent"): return {"success": False, "error": "Container is marked persistent and has no expiry to extend."} base = max(float(info.get("expiresAt") or now), now) new_expiry = min(base + add_seconds, now + CONTAINER_MAX_LIFETIME_SECONDS) info["expiresAt"] = new_expiry name = info.get("name") _save_container_state() print(f"[{time.strftime('%X')}] [📦 CONTAINER EXTENDED] {name} ({container_id[:12]}) " f"+{add_seconds}s -> {int(new_expiry - now)}s remaining", flush=True) return {"success": True, "expiresAt": new_expiry, "addedSeconds": add_seconds, "remainingSeconds": int(new_expiry - now), "persistent": False} def set_container_persistence(container_id, persistent, fallback_ttl_seconds=1800): """Flips a container between TTL-reaped and manager-managed. Turning it on hands the container to the manager: no expiry, runtime restart policy set, self-heal on exit, and a snapshot taken immediately so there is a recovery point from the moment the promise is made rather than only after the first scheduled backup.""" now = time.time() with active_containers_lock: info = active_containers.get(container_id) if not info: return {"success": False, "error": "Unknown container — not managed by this agent."} runtime_path = _runtime_path_for(info) name = info.get("name") if persistent: info["persistent"] = True info["expiresAt"] = None info["nextBackupAt"] = now # snapshot on this call, then on the normal cadence else: info["persistent"] = False try: ttl = max(60, min(int(fallback_ttl_seconds), CONTAINER_MAX_EXTEND_SECONDS)) except (TypeError, ValueError): ttl = 1800 info["ttlSeconds"] = ttl info["expiresAt"] = now + ttl policy_applied = False if persistent: policy_applied = _apply_restart_policy(runtime_path, container_id, "always") else: _apply_restart_policy(runtime_path, container_id, "no") with active_containers_lock: live = active_containers.get(container_id) if live is not None: live["restartPolicyApplied"] = policy_applied if persistent else False expires_at = live.get("expiresAt") _save_container_state() backup = None if persistent: backup = backup_container(container_id, reason="persistence-enabled") print(f"[{time.strftime('%X')}] [📦 CONTAINER {'PERSISTENT' if persistent else 'EPHEMERAL'}] " f"{name} ({container_id[:12]})" f"{'' if persistent else f' — expires in {int(expires_at - now)}s'}", flush=True) return { "success": True, "persistent": bool(persistent), "expiresAt": expires_at, # Surfaced so the UI can be honest about which layer is holding the guarantee: with the # runtime policy applied the container survives a host reboot, without it only the # agent's sweep brings it back. "restartPolicyApplied": policy_applied, "backup": backup, } def hibernate_container(container_id): """Stops a persistent container on purpose, freeing whatever CPU and memory the runtime had reserved for it for as long as it sits idle, while keeping it recoverable. Deliberately different from the container just dying: _container_manager_tick's stopped+persistent healing path checks the same "hibernated" flag this sets and leaves it alone, rather than starting it right back up on the very next sweep the way an actual crash would call for.""" with active_containers_lock: info = active_containers.get(container_id) if not info: return {"success": False, "error": "Unknown container — not managed by this agent."} if not info.get("persistent"): return {"success": False, "error": "Only persistent containers can be hibernated — a timed one already frees its resources when its TTL expires."} if info.get("hibernated"): return {"success": False, "error": "Already hibernated."} runtime_path = _runtime_path_for(info) name = info.get("name") # Fresh snapshot first: if the stopped container is ever lost outside this agent's own # tracking (a host reboot the runtime's restart policy does not survive, a manual `podman # rm`), this is the only way back -- a stale snapshot from whenever the last scheduled one # happened to land is a worse guarantee than the one persistent containers already get. backup = backup_container(container_id, reason="hibernate") try: res = subprocess.run([runtime_path, "stop", "-t", "30", container_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=60) except Exception as e: return {"success": False, "error": str(e)} if res.returncode != 0: err = (res.stderr or res.stdout or "stop failed").strip()[:2000] print(f"[{time.strftime('%X')}] [!] Could not hibernate {name} ({container_id[:12]}): {err}", flush=True) return {"success": False, "error": err} with active_containers_lock: live = active_containers.get(container_id) if live is not None: live["hibernated"] = True live["hibernatedAt"] = time.time() _save_container_state() print(f"[{time.strftime('%X')}] [💤 CONTAINER HIBERNATED] {name} ({container_id[:12]}) — " f"stopped, no longer holding its CPU/memory allocation", flush=True) return {"success": True, "backup": backup} def wake_container(container_id): """Starts a hibernated container back up. Deliberately does not try to relaunch its in-container agent or re-check its boot readiness itself -- both of those checks are already gated on state == "running" in _container_manager_tick, so clearing bootReady/nextNetworkCheckAt here is enough to make the very next sweep treat it exactly like any other container whose agent is not there yet, reusing that already-verified path instead of duplicating it.""" with active_containers_lock: info = active_containers.get(container_id) if not info: return {"success": False, "error": "Unknown container — not managed by this agent."} if not info.get("hibernated"): return {"success": False, "error": "This container is not hibernated."} runtime_path = _runtime_path_for(info) name = info.get("name") try: res = subprocess.run([runtime_path, "start", container_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=120) except Exception as e: return {"success": False, "error": str(e)} if res.returncode != 0: err = (res.stderr or res.stdout or "start failed").strip()[:2000] print(f"[{time.strftime('%X')}] [!] Could not wake {name} ({container_id[:12]}): {err}", flush=True) return {"success": False, "error": err} now = time.time() with active_containers_lock: live = active_containers.get(container_id) if live is not None: live["hibernated"] = False live["hibernatedAt"] = None has_ports = bool(live.get("containerPorts")) live["bootReady"] = None if has_ports else True live["bootDeadlineAt"] = (now + CONTAINER_BOOT_READY_TIMEOUT) if has_ports else None live["bootIssue"] = None live["bootRestarted"] = False live["nextNetworkCheckAt"] = now _save_container_state() print(f"[{time.strftime('%X')}] [⚡ CONTAINER WOKEN] {name} ({container_id[:12]})", flush=True) return {"success": True} NETWORK_CHECK_INTERVAL_OK = 300 # re-confirm a healthy container every 5 min NETWORK_CHECK_INTERVAL_REMEDIATING = 30 # right after an attempted fix, look again soon NETWORK_CHECK_INTERVAL_STUCK = 180 # already tried and failed; don't hammer sudo, but keep watching def _bridge_iface_for_network(runtime_path, network): """Host network interface backing a Podman/Docker user-defined network, so a firewall fix can be scoped to it rather than opened up host-wide.""" if not runtime_path or not network: return None try: res = subprocess.run([runtime_path, "network", "inspect", network, "--format", "{{.NetworkInterface}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=15) if res.returncode == 0: iface = (res.stdout or "").strip() if iface: return iface except Exception: pass return None def _probe_container_egress(runtime_path, container_id, server_url, insecure=False, timeout=10): """Execs into a running container and checks whether it can actually reach the control plane: resolve the server's hostname, then open a TCP connection to it. This is exactly the path a baked-in agent needs for its own heartbeat/control-channel, so a failure here predicts 'agentConnected: false' before waiting for a heartbeat that will never arrive to say so. Runs from inside the container rather than the host because the failure this looks for is specific to *that* network namespace -- the host can often reach the same server fine while the container's bridge cannot, which is the whole reason this check exists.""" if not runtime_path or not container_id or not server_url: return {"ok": False, "stage": None, "error": "missing runtime, container id, or server url"} host_part = server_url.split("://", 1)[-1].split("/", 1)[0].split(":", 1)[0] port = 443 if server_url.startswith("https") else 80 unverified = "True" if insecure else "False" code = ( "import socket,sys;" "h=%r;" "\ntry:\n" " ip=socket.gethostbyname(h)\n" "except Exception as e:\n" " print('DNS_FAIL:'+str(e)); sys.exit(1)\n" "s=socket.socket(socket.AF_INET,socket.SOCK_STREAM); s.settimeout(6)\n" "try:\n" " s.connect((ip,%d)); print('OK')\n" "except Exception as e:\n" " print('TCP_FAIL:'+str(e)); sys.exit(1)\n" ) % (host_part, port) try: res = subprocess.run([runtime_path, "exec", container_id, "python3", "-c", code], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=timeout) except subprocess.TimeoutExpired: return {"ok": False, "stage": None, "error": f"probe timed out after {timeout}s"} except Exception as e: return {"ok": False, "stage": None, "error": str(e)} out = (res.stdout or "").strip() if res.returncode == 0 and out == "OK": return {"ok": True, "stage": None, "error": None} if out.startswith("DNS_FAIL:"): return {"ok": False, "stage": "dns", "error": out[len("DNS_FAIL:"):][:300]} if out.startswith("TCP_FAIL:"): return {"ok": False, "stage": "tcp", "error": out[len("TCP_FAIL:"):][:300]} return {"ok": False, "stage": None, "error": (res.stderr or out or "unknown probe failure").strip()[:300]} def _container_agent_alive(runtime_path, container_id, timeout=10): """Whether the Applotus agent is actually still a running process inside this container -- not whether the server last heard from it. Needed because agentConnected on the server is a one-way flag (set true on heartbeat, never cleared), so a container whose agent has died looks identical to a healthy one until something actually looks inside. Scanning /proc rather than shelling out to `pgrep`/`ps` because neither is guaranteed to exist in a minimal image -- python3 is the one thing this agent can already assume is present. Returns True/False, or None when the container couldn't be asked at all (exec failed outright, e.g. it is mid-restart) -- None means "don't know", not "dead", so a transient exec failure does not itself trigger a healing attempt against a container that is simply still coming up.""" code = ( "import os\n" "self_pid = str(os.getpid())\n" "found = False\n" "for p in os.listdir('/proc'):\n" " if not p.isdigit() or p == self_pid:\n" " continue\n" " try:\n" " with open('/proc/' + p + '/cmdline', 'rb') as fh:\n" " cmd = fh.read()\n" " except Exception:\n" " continue\n" " if b'sentinel-agent.py' in cmd or b'sentinel_agent.py' in cmd:\n" " found = True\n" " break\n" "print('YES' if found else 'NO')\n" ) try: res = subprocess.run([runtime_path, "exec", container_id, "python3", "-c", code], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=timeout) except Exception: return None if res.returncode != 0: return None out = (res.stdout or "").strip() if out == "YES": return True if out == "NO": return False return None CONTAINER_BOOT_READY_TIMEOUT = 90 # generous margin above the ~15-45s a normal mysql-style init takes def _container_ports_listening(runtime_path, container_id, ports, timeout=10): """Whether any of the container's expected ports are actually in LISTEN state inside it. Reads /proc/net/tcp[6] directly rather than asking the workload to prove itself (no assumption that ss/netstat/nc exist in the image) and rather than probing from the host (loopback-only services and network-namespace quirks would give a false negative there). Returns True/False, or None when the container couldn't be asked at all (exec failed outright).""" if not ports: return None wanted = ",".join(str(int(p)) for p in ports) code = ( "wanted = {" + wanted + "}\n" "found = False\n" "for fname in ('/proc/net/tcp', '/proc/net/tcp6'):\n" " try:\n" " with open(fname) as fh:\n" " lines = fh.readlines()[1:]\n" " except Exception:\n" " continue\n" " for line in lines:\n" " parts = line.split()\n" " if len(parts) < 4:\n" " continue\n" " local, state = parts[1], parts[3]\n" " if state != '0A':\n" " continue\n" " try:\n" " port = int(local.split(':')[-1], 16)\n" " except Exception:\n" " continue\n" " if port in wanted:\n" " found = True\n" "print('YES' if found else 'NO')\n" ) try: res = subprocess.run([runtime_path, "exec", container_id, "python3", "-c", code], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=timeout) except Exception: return None if res.returncode != 0: return None out = (res.stdout or "").strip() if out == "YES": return True if out == "NO": return False return None def _container_pid1_summary(runtime_path, container_id, timeout=10): """Best-effort description of what PID 1 is actually doing, for the diagnostic message a stuck boot leaves on the node card -- 'PID 1 is still gosu' says a lot more than 'not ready yet'.""" code = ( "try:\n" " with open('/proc/1/cmdline', 'rb') as fh:\n" " parts = [p.decode(errors='replace') for p in fh.read().split(b'\\x00') if p]\n" " print(' '.join(parts)[:200])\n" "except Exception as e:\n" " print('(unreadable: %s)' % e)\n" ) try: res = subprocess.run([runtime_path, "exec", container_id, "python3", "-c", code], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=timeout) except Exception: return None if res.returncode != 0: return None return (res.stdout or "").strip() or None def _attempt_bridge_egress_remediation(runtime_path, network): """Best-effort fix for the failure mode this has actually been seen to have: a host firewall (ufw and/or a separate iptables FORWARD policy) that never learned to trust the container bridge, so DNS and control-plane traffic from a container gets dropped before it leaves the host. Both edits are additive and idempotent -- checked before insertion so repeated sweeps don't pile up duplicate rules -- and scoped to this network's own bridge interface rather than opening the host up generally. Deliberately does not touch --dns flags or per-network DNS config: that class of failure (upstream resolver unreachable, e.g. a locked-down corporate network) is handled by never forcing a specific resolver in the first place, not by patching it after the fact here. Returns (changed: bool, detail: str). changed=False with a populated detail means "diagnosed but not fixable from here" -- e.g. no root, no iptables/ufw -- which is exactly what should end up on the node card when this can't be resolved automatically.""" if platform.system() != "Linux": return False, "not attempted: remediation only implemented for Linux hosts" iface = _bridge_iface_for_network(runtime_path, network) if not iface: return False, f"could not determine the bridge interface for network {network!r}" notes = [] changed = False tool_found = False sudo_prefix = [] if os.geteuid() == 0 else ["sudo", "-n"] if shutil.which("iptables"): tool_found = True for direction, args in (("out of", ["-i", iface]), ("into", ["-o", iface])): # -C requires an exact match on every match module a candidate rule carries, so the # comment has to be part of *both* the check and the insert -- checking without it # never matches a rule that has one, and every sweep would insert another duplicate. comment = f"sentinel: allow {direction} {iface} bridge (auto-remediated DNS/egress)" rule_spec = args + ["-j", "ACCEPT", "-m", "comment", "--comment", comment] check_cmd = sudo_prefix + ["iptables", "-C", "FORWARD"] + rule_spec try: chk = subprocess.run(check_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10) except Exception as e: notes.append(f"iptables check failed: {e}") continue if chk.returncode == 0: continue # already present -- nothing to note, this direction is already fine insert_cmd = sudo_prefix + ["iptables", "-I", "FORWARD", "1"] + rule_spec try: r = subprocess.run(insert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=15) except Exception as e: notes.append(f"iptables insert failed: {e}") continue if r.returncode == 0: changed = True notes.append(f"inserted FORWARD accept ({direction} {iface})") else: notes.append(f"iptables insert failed: {(r.stderr or r.stdout or '').strip()[:150]}") if shutil.which("ufw"): tool_found = True try: st = subprocess.run(sudo_prefix + ["ufw", "status"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=10) active = "Status: active" in (st.stdout or "") already = f"on {iface}" in (st.stdout or "") except Exception as e: active, already = False, False notes.append(f"ufw status failed: {e}") if active and not already: try: r = subprocess.run(sudo_prefix + ["ufw", "allow", "in", "on", iface, "comment", "sentinel: trust podman bridge (auto-remediated)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=15) except Exception as e: r = None notes.append(f"ufw allow failed: {e}") if r is not None: if r.returncode == 0: changed = True notes.append(f"ufw allow in on {iface}") else: notes.append(f"ufw allow failed: {(r.stderr or r.stdout or '').strip()[:150]}") if not tool_found: return False, "no firewall tooling found (iptables/ufw) to remediate with" if not notes: return False, f"nothing to fix: {iface} already allowed in both iptables FORWARD and ufw" return changed, "; ".join(notes) def _container_manager_tick(): """One sweep: reap what expired, heal what died, snapshot what is due.""" with active_containers_lock: tracked = {cid: dict(info) for cid, info in active_containers.items()} if not tracked: return now = time.time() for cid, info in tracked.items(): runtime_path = _runtime_path_for(info) persistent = bool(info.get("persistent")) expires_at = info.get("expiresAt") # ── Expiry ──────────────────────────────────────────────────────────── if not persistent and expires_at and now >= float(expires_at): rm_res = remove_container(cid) if rm_res.get("success"): _save_container_state() _report_container_event("remove_container_result", cid, rm_res) continue state = _container_liveness(runtime_path, cid) if state is None: continue # couldn't ask; try again next sweep rather than guessing if state == "missing": # Gone from the runtime. For an ephemeral container that is just cleanup; for a # persistent one it is the failure the guarantee exists for. Prefer the full in-memory # spec (still holds this process's own agent-launch credentials), but fall back to the # redacted, on-disk recoverySpec -- secret-free, so it survives an agent restart -- # rather than reporting the loss when a perfectly good snapshot still exists. spec = None with active_containers_lock: live = active_containers.get(cid) if live: spec = live.get("spec") or live.get("recoverySpec") if persistent and spec: if _recover_container(cid, spec): continue with active_containers_lock: active_containers.pop(cid, None) _save_container_state() print(f"[{time.strftime('%X')}] [📦 CONTAINER LOST] {info.get('name')} ({cid[:12]}) " f"no longer exists on this host{' and could not be rebuilt' if persistent else ''}", flush=True) continue # A hibernated container is stopped on purpose (see hibernate_container) -- without this # guard, this exact healing path would restart it right back up on the very next sweep, # since "persistent and stopped" otherwise means "died and needs restarting", not "was # deliberately put to sleep to stop paying for its CPU/memory the whole time it sits idle". if state == "stopped" and persistent and not info.get("hibernated"): try: res = subprocess.run([runtime_path, "start", cid], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=120) ok = res.returncode == 0 except Exception as e: ok, res = False, None err = str(e) if ok: with active_containers_lock: live = active_containers.get(cid) if live is not None: live["restarts"] = int(live.get("restarts") or 0) + 1 live["lastRestartAt"] = now restarts = live["restarts"] _save_container_state() print(f"[{time.strftime('%X')}] [📦 CONTAINER HEALED] {info.get('name')} " f"({cid[:12]}) was stopped — restarted (restart #{restarts})", flush=True) _report_container_event("container_healed_result", cid, {"action": "restart", "restarts": restarts}) else: err = (res.stderr or res.stdout or "start failed").strip()[:500] if res is not None else err print(f"[{time.strftime('%X')}] [!] Could not restart persistent container " f"{info.get('name')} ({cid[:12]}): {err}", flush=True) continue # ── Scheduled snapshot ──────────────────────────────────────────────── if persistent and state == "running": next_backup = info.get("nextBackupAt") if next_backup is None or now >= float(next_backup): res = backup_container(cid, reason="scheduled") if res.get("success"): _report_container_event("backup_container_result", cid, res) else: # Push the next attempt out a full interval so a container whose runtime # refuses commits does not retry every 15 seconds forever. with active_containers_lock: live = active_containers.get(cid) if live is not None: live["nextBackupAt"] = now + CONTAINER_BACKUP_INTERVAL # ── Workload boot readiness ───────────────────────────────────────────── # "running" only means PID 1 is alive, not that the thing inside it works -- the # crun+no-new-privileges+gosu incident this was built for left PID 1 spinning forever with # the data directory still empty, "running" the whole time. Give it a generous grace # window (first boots are legitimately slow), then take one bounded shot at a plain # restart before giving up and reporting it plainly with whatever PID 1 is stuck on. if info.get("bootReady") is None and state == "running": listening = _container_ports_listening(runtime_path, cid, info.get("containerPorts") or []) if listening is True: with active_containers_lock: live = active_containers.get(cid) if live is not None: live["bootReady"] = True live["bootIssue"] = None print(f"[{time.strftime('%X')}] [🚦 BOOT READY] {info.get('name')} ({cid[:12]}) " f"is listening on its expected port(s)", flush=True) elif listening is False: deadline = info.get("bootDeadlineAt") if deadline is not None and now >= float(deadline): pid1 = _container_pid1_summary(runtime_path, cid) or "unknown" if not info.get("bootRestarted"): try: r = subprocess.run([runtime_path, "restart", cid], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=60) restarted_ok = r.returncode == 0 except Exception: restarted_ok = False with active_containers_lock: live = active_containers.get(cid) if live is not None: live["bootRestarted"] = True # Fresh PID 1 gets a fresh full window, not whatever was left of the old one. live["bootDeadlineAt"] = now + CONTAINER_BOOT_READY_TIMEOUT live["bootIssue"] = { "stage": "restarted" if restarted_ok else "restart_failed", "pid1": pid1, } print(f"[{time.strftime('%X')}] [🚦 BOOT STUCK] {info.get('name')} ({cid[:12]}) " f"never opened its expected port after {CONTAINER_BOOT_READY_TIMEOUT}s " f"(PID 1: {pid1}) -- {'restarted once' if restarted_ok else 'restart itself failed'}", flush=True) _report_container_event("container_boot_restarted_result", cid, {"pid1": pid1, "restarted": restarted_ok}) else: elapsed = int(now - (info.get("createdAt") or now)) with active_containers_lock: live = active_containers.get(cid) if live is not None: live["bootIssue"] = {"stage": "stuck", "pid1": pid1, "elapsedSeconds": elapsed} print(f"[{time.strftime('%X')}] [!] {info.get('name')} ({cid[:12]}) still not " f"ready {elapsed}s in, even after a restart (PID 1: {pid1}) -- needs a look", flush=True) # ── In-container agent liveness ───────────────────────────────────────── # The workload container being "running" says nothing about whether the agent exec'd into # it is still alive. It reliably is not after a mysql-style image's first-boot sequence: # the official entrypoint inits into a throwaway mysqld, shuts it down, and re-execs the # real one -- and under --restart always that shutdown is a full container restart, not an # in-place re-exec, which takes the exec'd agent down with the old instance. Nothing else # would ever notice, because agentConnected on the server is a one-way flag: it never goes # back to false on its own, so a dead-agent container looks identical to a healthy one # everywhere except here. if info.get("withAgent") and state == "running": alive = _container_agent_alive(runtime_path, cid) if alive is False: # Not info["spec"]["agentCtx"]: that dict is deliberately in-memory only (it holds # the container's scoped API key, and _PERSISTED_CONTAINER_KEYS excludes "spec" on # purpose so no credential is ever mirrored to disk). That makes it the *first* # casualty of anything that restarts this process -- a self-upgrade, a crash, a # host reboot -- which is exactly when healing is needed most. The host's own # report-context credential is always available without persisting anything new, # and it is no wider a grant than this agent already has: it can already exec # arbitrary commands inside this container, so injecting its own key into it adds # no privilege the host agent didn't already hold. agent_ctx = { "server_url": _manager_report_ctx.get("server_url"), "api_key": _manager_report_ctx.get("api_key"), "insecure": _manager_report_ctx.get("insecure"), "host_header": _manager_report_ctx.get("host_header"), "tenant_id": _manager_report_ctx.get("tenant_id"), } relaunch = _start_in_container_agent( runtime_path, cid, info.get("name"), _manager_node_id, info.get("containerPorts") or [], agent_ctx) if relaunch.get("success"): print(f"[{time.strftime('%X')}] [🩺 AGENT HEALED] {info.get('name')} ({cid[:12]}) " f"had no live agent process -- relaunched", flush=True) with active_containers_lock: live = active_containers.get(cid) if live is not None: live["agentNodeId"] = relaunch.get("nodeId") or live.get("agentNodeId") # Force an egress recheck on the next sweep instead of waiting out # whatever interval the last (possibly stale) result set. live["nextNetworkCheckAt"] = now _report_container_event("container_agent_healed_result", cid, { "action": "relaunch_agent", "reason": "agent process not found in container"}) else: print(f"[{time.strftime('%X')}] [!] Could not relaunch agent for " f"{info.get('name')} ({cid[:12]}): {relaunch.get('error')}", flush=True) # ── Network egress health (DNS + reach to the control plane) ─────────── # Only meaningful for a container that carries its own agent: that agent needs this exact # path for its heartbeat and control channel, so a failure here is the same failure that # would otherwise only show up later as a silent "agentConnected: false" with no # explanation. Checked on its own schedule (not every sweep) because the probe execs into # the container and a failed remediation attempt shells out to iptables/ufw -- neither is # free, and this needs to run often enough to catch a regression, not on every 15s tick. if info.get("withAgent") and state == "running": next_check = info.get("nextNetworkCheckAt") if next_check is None or now >= float(next_check): server_url = _manager_report_ctx.get("server_url") insecure = bool(_manager_report_ctx.get("insecure")) probe = _probe_container_egress(runtime_path, cid, server_url, insecure=insecure) was_ok = info.get("networkOk") remediated_detail = info.get("networkRemediationDetail") remediation_attempted = bool(info.get("networkRemediationAttempted")) if probe["ok"]: interval = NETWORK_CHECK_INTERVAL_OK if was_ok is False: print(f"[{time.strftime('%X')}] [🌐 NETWORK RECOVERED] {info.get('name')} " f"({cid[:12]}) can reach the control plane again", flush=True) elif not remediation_attempted: # First time seeing this fail for this container: try the known fix once # before reporting anything as unrecoverable. changed, detail = _attempt_bridge_egress_remediation(runtime_path, info.get("network")) remediation_attempted = True remediated_detail = detail print(f"[{time.strftime('%X')}] [🌐 NETWORK ISSUE] {info.get('name')} ({cid[:12]}) " f"cannot reach the control plane ({probe.get('stage') or 'connect'}: " f"{probe.get('error')}) — remediation: {detail}", flush=True) if changed: # Re-probe immediately so a real fix is reflected without waiting a full # interval, rather than reporting "issue" for one extra cycle after it's # actually resolved. probe = _probe_container_egress(runtime_path, cid, server_url, insecure=insecure) interval = NETWORK_CHECK_INTERVAL_REMEDIATING else: interval = NETWORK_CHECK_INTERVAL_STUCK with active_containers_lock: live = active_containers.get(cid) if live is not None: live["networkOk"] = probe["ok"] live["networkIssue"] = None if probe["ok"] else { "stage": probe.get("stage"), "error": probe.get("error")} live["networkRemediationAttempted"] = remediation_attempted live["networkRemediationDetail"] = remediated_detail live["nextNetworkCheckAt"] = now + interval _save_container_state() def _redact_argv_for_persistence(argv, image_index): """Strips every -e KEY=VALUE pair from a container's creation argv, producing a copy that is safe to write to disk -- unlike the full argv (see _PERSISTED_CONTAINER_KEYS), which carries whatever secrets were passed at creation (the container's own API key, a DB bootstrap password) and is deliberately kept in memory only. Recovering from this redacted copy does not need those values back: a rebuilt container gets its in-container agent relaunched by the very next manager sweep using the host agent's own key (see the "in-container agent liveness" check), and a DB bootstrap password in argv only ever affects a truly empty data directory -- irrelevant once restoring from a snapshot that already has one. This is what lets recovery and env updates survive an agent restart, not just the runtime process crashing within the same agent lifetime.""" redacted = [] new_image_index = image_index i = 0 while i < len(argv): if argv[i] in ("-e", "--env") and i + 1 < len(argv): if i < image_index: new_image_index -= 2 i += 2 continue redacted.append(argv[i]) i += 1 return redacted, new_image_index def _recover_container(container_id, spec): """Rebuilds a vanished persistent container from its newest snapshot, preserving its identity in the registry so the server's node record follows it rather than a new one appearing. Reachable with either the full in-memory spec or the redacted, disk-persisted recoverySpec (see _container_manager_tick and _redact_argv_for_persistence) -- the latter is what lets this survive an agent restart, not just a runtime-level container loss within the same process.""" argv = list(spec.get("argv") or []) image_index = spec.get("imageIndex") if not argv or not isinstance(image_index, int) or image_index >= len(argv): return False with active_containers_lock: info = active_containers.get(container_id) info = dict(info) if info else None if not info: return False runtime_path = _runtime_path_for(info) backups = info.get("backups") or [] # Newest snapshot if there is one, otherwise the original image — a container that died before # its first backup still comes back, just empty rather than restored. restore_image = backups[-1].get("tag") if backups else argv[image_index] rebuilt = list(argv) rebuilt[image_index] = restore_image # The old name is still reserved if the container was merely removed from the runtime's view; # clear it so the rebuild cannot fail on a name conflict. try: subprocess.run([runtime_path, "rm", "-f", info.get("name") or container_id], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) except Exception: pass try: res = subprocess.run(rebuilt, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=180) except Exception as e: print(f"[{time.strftime('%X')}] [!] Recovery of {info.get('name')} failed: {e}", flush=True) return False if res.returncode != 0: print(f"[{time.strftime('%X')}] [!] Recovery of {info.get('name')} failed: " f"{(res.stderr or res.stdout or '').strip()[:500]}", flush=True) return False new_id = (res.stdout or "").strip() if not new_id: return False with active_containers_lock: record = active_containers.pop(container_id, None) or info record["restarts"] = int(record.get("restarts") or 0) + 1 record["lastRestartAt"] = time.time() record["recoveredFrom"] = restore_image active_containers[new_id] = record restarts = record["restarts"] _save_container_state() _apply_restart_policy(runtime_path, new_id, "always") print(f"[{time.strftime('%X')}] [📦 CONTAINER RECOVERED] {info.get('name')} " f"({container_id[:12]} -> {new_id[:12]}) rebuilt from {restore_image} " f"(restart #{restarts})", flush=True) # The container id is the server's key for the child node, so the swap has to be reported # explicitly or the UI keeps polling an id that no longer exists. _report_container_event("container_recovered_result", container_id, { "action": "recover", "previousContainerId": container_id, "containerId": new_id, "restoredFrom": restore_image, "restarts": restarts, }) return True def _container_manager_loop(): while True: try: _container_manager_tick() except Exception as e: print(f"[{time.strftime('%X')}] [!] Container manager sweep failed: {e}", flush=True) time.sleep(CONTAINER_MANAGER_INTERVAL) _CONTAINER_NAME_RE = None # compiled lazily to avoid importing re at module load if unused def _valid_image_ref(image): import re global _CONTAINER_NAME_RE if _CONTAINER_NAME_RE is None: _CONTAINER_NAME_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9._/:-]{0,255}$') # Type-check before matching: a non-string `image` (a JSON number or object posted straight # to the dispatch endpoint) would make re.match raise TypeError, which would escape the task # loop and strand every sibling task already flipped to 'dispatched' in the same batch. if not isinstance(image, str): return False return bool(image) and bool(_CONTAINER_NAME_RE.match(image)) def _qualify_image_ref(image): """Expands a short image name to a fully-qualified one. Docker silently resolves "nginx:alpine" against Docker Hub; Podman refuses it outright ('short-name did not resolve to an alias and no unqualified-search registries are defined') unless the host happens to configure unqualified-search registries. Normalizing here means the same image string works on both runtimes instead of failing on every Podman host.""" if not isinstance(image, str) or not image: return image first = image.split("/")[0] if "/" not in image: # Bare repo name -> Docker Hub official library. return "docker.io/library/" + image if "." in first or ":" in first or first == "localhost": return image # already carries a registry host (and optional port) # user/repo form -> Docker Hub, but not the official library namespace. return "docker.io/" + image AGENT_IN_CONTAINER_PATH = "/opt/applotus/sentinel-agent.py" # The read-only mount goes to a separate path, and the container runs a writable copy at # AGENT_IN_CONTAINER_PATH. Two reasons this matters: # - upgrade_agent_self replaces the script at its own path. Mounting directly onto the run path made # that path read-only, so an in-container agent could never self-upgrade -- it stayed frozen at # whatever version existed when its container was created, forever, and only a redeploy fixed it. # - the container fetches the server's current agent at boot, so a container started long after the # host upgraded still begins on the current version rather than the host's on-disk copy. # The mount is the fallback for when that fetch cannot happen (no egress yet, server down). AGENT_BUNDLED_IN_CONTAINER_PATH = "/opt/applotus/sentinel-agent.bundled.py" AGENT_BUNDLED_LABEL = "org.applotus.agent" def _image_bundles_agent(runtime_path, image): """True when the image declares it ships the Applotus agent at AGENT_IN_CONTAINER_PATH. Read from the image's labels, so it works for any image built that way rather than only ones named applotus/*. Answered before the container exists, which is why it inspects the image and not a running container: the bind mount it decides can only be set at creation time. Every failure answers False — image not pulled yet, older runtime, malformed label. False means 'mount the host copy', which is the behaviour that always works: mounting over a baked agent is harmless, while wrongly skipping the mount would leave a container with no agent at all.""" if not runtime_path or not image: return False try: res = subprocess.run( [runtime_path, "image", "inspect", "--format", '{{index .Config.Labels "' + AGENT_BUNDLED_LABEL + '"}}', image], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=25) except Exception: return False if res.returncode != 0: return False val = (res.stdout or "").strip().lower() # Podman renders a missing label as ""; Docker as an empty line. return val == "bundled" def _agent_boot_script(py, server_url, child_node_id, workload_name, insecure=False, tenant_id=None, api_key=None, parent_node_id=None): """Shell run inside the container: fetch the server's current agent, else fall back to the mounted copy, then exec it from a writable path. Fetched with python rather than curl because python is already a hard requirement for the agent while curl is absent from most minimal images. Compiled before use so a truncated or error-page response cannot replace a working agent with something that will not start -- the same check upgrade_agent_self makes before replacing itself.""" run = AGENT_IN_CONTAINER_PATH bundled = AGENT_BUNDLED_IN_CONTAINER_PATH url = server_url.rstrip("/") + "/download/sentinel-agent.py" unverified = "True" if insecure else "False" # Only double quotes inside, so the whole program can be wrapped in single quotes for sh. fetch = ( "import urllib.request,ssl;" "ctx=ssl._create_unverified_context() if %s else None;" "d=urllib.request.urlopen(\"%s\",timeout=25,context=ctx).read();" "open(\"%s\",\"wb\").write(d)" ) % (unverified, url, run) args = ["--server", server_url, "--id", child_node_id, "--name", workload_name, "--interval", "10", "--kind", "container"] if parent_node_id: args += ["--parent-node-id", parent_node_id] if api_key: args += ["--api-key", api_key] if insecure: args.append("--insecure") if tenant_id: args += ["--tenant", tenant_id] arg_str = " ".join(args) return ( "mkdir -p /opt/applotus; " "if {py} -c '{fetch}' 2>/dev/null && {py} -m py_compile {run} 2>/dev/null; then " " echo \"[sentinel] agent fetched from server\"; " "else " " echo \"[sentinel] fetch failed, using the copy mounted from the host\"; " " cp {bundled} {run} 2>/dev/null || true; " "fi; " "rm -rf /opt/applotus/__pycache__ 2>/dev/null || true; " "exec {py} {run} {args}" ).format(py=py, fetch=fetch, run=run, bundled=bundled, args=arg_str) def _start_in_container_agent(runtime_path, workload_id, workload_name, parent_node_id, container_ports, agent_ctx): """Starts the Applotus agent *inside* the workload container, as an extra process. Deliberately uses `exec -d` rather than putting the agent in the container's run arguments: the run command replaces the image's entrypoint, so baking the agent in would either stop the real application from starting or require a wrapper that knows the image's original CMD. Exec'ing in afterwards leaves the workload running exactly as its image intends (PID 1 stays the app) and adds the agent alongside it. Running in-container rather than in a network-namespace sidecar is what gives the agent the workload's filesystem and process table, not just its ports -- so the agent's own remote access features operate on the container itself. The cost is that the image must ship a python3, which is checked here rather than assumed.""" server_url = agent_ctx.get("server_url") api_key = agent_ctx.get("api_key") if not server_url: return {"success": False, "error": "No server URL available for the in-container agent."} if not api_key: return {"success": False, "error": "No API key supplied for the in-container agent."} # Loopback is the *container's* loopback here, so the control plane would be unreachable and # the agent would fail silently in a retry loop nobody is watching. _host_part = server_url.split("://", 1)[-1].split("/", 1)[0].split(":", 1)[0].lower() if _host_part in ("127.0.0.1", "localhost", "::1", "0.0.0.0"): return {"success": False, "error": ("Server URL %s is loopback, which is unreachable from inside the " "container. Point the agent at an address the container can reach " "(LAN IP or public hostname)." % server_url)} # Confirm the image actually ships an interpreter before trying to run the agent, so the # failure names the real cause instead of surfacing a bare exec error. py = None for cand in ("python3", "python"): try: probe = subprocess.run([runtime_path, "exec", workload_id, "sh", "-c", "command -v " + cand], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=20) if probe.returncode == 0 and (probe.stdout or "").strip(): py = (probe.stdout or "").strip().splitlines()[0] break except Exception: pass is_sidecar = False sidecar_id = None if not py: # Fall back to launching a lightweight python sidecar sharing network & pid namespace with workload sidecar_name = workload_name + "-sidecar" agent_src = os.path.abspath(sys.argv[0]) sidecar_cmd = [ runtime_path, "run", "-d", "--name", sidecar_name, f"--net=container:{workload_id}", f"--pid=container:{workload_id}", "-v", agent_src + ":" + AGENT_BUNDLED_IN_CONTAINER_PATH + ":ro", "docker.io/library/python:3.11-alpine", "sh", "-c", f"python3 {AGENT_IN_CONTAINER_PATH} --server {server_url} --id container-{workload_id[:12]} --name {workload_name} --interval 10 --kind container --parent-node-id {parent_node_id or ''} --api-key {api_key}" + (" --insecure" if agent_ctx.get("insecure") else "") + (f" --tenant {agent_ctx['tenant_id']}" if agent_ctx.get("tenant_id") else "") ] try: sc_res = subprocess.run(sidecar_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=60) if sc_res.returncode == 0: sidecar_id = (sc_res.stdout or "").strip() is_sidecar = True child_node_id = "container-" + workload_id[:12] print(f"[{time.strftime('%X')}] [\U0001F916 SIDECAR AGENT UP] {sidecar_name} ({sidecar_id[:12]}) " f"paired with workload {workload_id[:12]}", flush=True) return {"success": True, "nodeId": child_node_id, "sidecarId": sidecar_id, "ports": container_ports} except Exception as sc_err: pass return {"success": False, "error": ("Image has no python3 and failed to start sidecar agent. Deploy " "without an in-container agent, or choose a python-bearing image.")} child_node_id = "container-" + workload_id[:12] cmd = [ runtime_path, "exec", "-d", # Both namings on purpose: a baked image may carry an agent that predates SENTINEL_*, and an # in-container agent that misses NODE_KIND registers as a host node rather than a child. "-e", "SENTINEL_API_KEY=" + api_key, "-e", "SENTINEL_NODE_KIND=container", "-e", "SENTINEL_PARENT_NODE_ID=" + (parent_node_id or ""), "-e", "SENTINEL_CONTAINER_ID=" + workload_id, "-e", "APPLOTUS_API_KEY=" + api_key, "-e", "APPLOTUS_NODE_KIND=container", "-e", "APPLOTUS_PARENT_NODE_ID=" + (parent_node_id or ""), "-e", "APPLOTUS_CONTAINER_ID=" + workload_id, workload_id, # A boot script rather than the interpreter directly: it fetches the server's current agent to # a writable path, falls back to the mounted copy, then execs it. Passed as one argv element, # so sh receives it verbatim and nothing here goes through a second round of shell quoting. "sh", "-c", _agent_boot_script( py, server_url, child_node_id, workload_name, insecure=bool(agent_ctx.get("insecure")), tenant_id=agent_ctx.get("tenant_id"), api_key=api_key, parent_node_id=parent_node_id, ), ] try: res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=60) except subprocess.TimeoutExpired: return {"success": False, "error": "Starting the in-container agent timed out after 60s."} except Exception as e: return {"success": False, "error": str(e)} if res.returncode != 0: return {"success": False, "error": (res.stderr or res.stdout or "").strip()[:2000]} print(f"[{time.strftime('%X')}] [\U0001F916 IN-CONTAINER AGENT UP] {workload_name} " f"({workload_id[:12]}) via {py} — node {child_node_id}, no host ports published", flush=True) return {"success": True, "nodeId": child_node_id, "python": py, "ports": container_ports} # A desktop session needs real headroom of its own on top of whatever the workload uses -- below # this a container's fractional CPU/memory share would be split between the app it's meant to run # and a full Xvfb+XFCE+VNC stack, starving both. 2GB rather than 1GB: confirmed live that 1GB was # not enough -- installing the stack alongside an already-running mysqld got OOM-killed by the # container's own memory cgroup partway through dependency resolution. CONTAINER_DESKTOP_MIN_CPUS = 1.0 CONTAINER_DESKTOP_MIN_MEMORY_MB = 2048 # Installs are attempted only if the stack is not already present, so re-running this against an # image that already ships one (applotus-vps) is a fast no-op rather than a redundant apt-get. _CONTAINER_DESKTOP_INSTALL_SCRIPT = ( "set -e; " "if command -v Xvfb >/dev/null 2>&1 && command -v x11vnc >/dev/null 2>&1; then " "echo already-installed; exit 0; " "fi; " "if command -v apt-get >/dev/null 2>&1; then " "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends " "xvfb x11vnc dbus-x11 xfce4-session xfwm4 xfce4-panel xfdesktop4; " "elif command -v apk >/dev/null 2>&1; then " "apk add --no-cache xvfb x11vnc dbus-x11 xfce4 xfce4-panel; " "else " # Confirmed live against docker.io/library/mysql:8 (Oracle Linux 9): none of the RPM-based # catalog images carry a desktop package set in their default repos. Getting one would mean # enabling EPEL (or equivalent) inside a running production container -- also confirmed live, # and it pulled in a ~100-package systemd/GNOME-stack dependency chain that then failed # partway through anyway. Not worth the size or fragility for what this action buys; report # the real limitation instead of half-installing a desktop that will not come up. "echo 'this image has no desktop packages in its default repositories (RPM-based images would need EPEL or similar enabled first, which this action does not do automatically)' >&2; exit 3; " "fi" ) # Mirrors vps-entrypoint.sh's sequence (Xvfb -> wait for the socket -> XFCE -> x11vnc), loopback # only and no password: the only route in is this container's own agent, already authenticated to # Applotus, so a VNC password would be a second secret sitting in the container for no benefit. _CONTAINER_DESKTOP_LAUNCH_SCRIPT = ( "mkdir -p /var/log/applotus /tmp/.X11-unix; " "rm -f /tmp/.X1-lock /tmp/.X11-unix/X1 2>/dev/null || true; " "DISPLAY=:1 Xvfb :1 -screen 0 1024x768x16 -nolisten tcp >/var/log/applotus/xvfb.log 2>&1 & " "i=0; while [ ! -S /tmp/.X11-unix/X1 ]; do i=$((i+1)); [ \"$i\" -gt 100 ] && exit 1; sleep 0.1; done; " "DISPLAY=:1 dbus-launch --exit-with-session xfce4-session >/var/log/applotus/xfce.log 2>&1 & " "DISPLAY=:1 x11vnc -display :1 -rfbport 5900 -listen 127.0.0.1 -nopw -shared -forever " "-noxdamage -wait 40 -defer 30 -quiet >/var/log/applotus/x11vnc.log 2>&1 &" ) def _container_resource_limits(runtime_path, container_id): """Live CPU/memory limits straight from the runtime, in the same units provision_container was given (whole CPUs, MB) -- the source of truth, unlike the cached copy on active_containers which a container created before cpus/memoryMb joined _PERSISTED_CONTAINER_KEYS may be missing. Returns None on any failure so the caller can fall back to whatever it already had.""" try: res = subprocess.run( [runtime_path, "inspect", container_id, "--format", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=15) if res.returncode != 0: return None parts = (res.stdout or "").strip().split() if len(parts) != 2: return None nano_cpus, mem_bytes = int(parts[0]), int(parts[1]) return (nano_cpus / 1e9, int(mem_bytes / (1024 * 1024))) except Exception: return None def install_container_desktop(container_id): """Installs and starts a lightweight virtual desktop (Xvfb + XFCE + x11vnc bound to the container's own loopback) inside an already-running container -- the same stack applotus-vps ships baked in. No new detection code is needed: the in-container agent already probes 127.0.0.1:5900-5905 on its own heartbeat (check_desktop_env), so the existing Remote Desktop bridge lights up on its own the moment x11vnc starts listening.""" with active_containers_lock: info = active_containers.get(container_id) if not info: return {"success": False, "error": "Unknown container — not managed by this agent."} cpus = info.get("cpus") or 0 memory_mb = info.get("memoryMb") or 0 runtime_path = _runtime_path_for(info) with_agent = info.get("withAgent") name = info.get("name") if not with_agent: return {"success": False, "error": "This container has no in-container agent, so a desktop session inside it could not be detected or reached."} if not cpus or not memory_mb: # Neither survives an agent restart on its own (see _PERSISTED_CONTAINER_KEYS) for a # container created before that field was added to it -- ask the runtime directly rather # than reject a container that may well qualify. cpus, memory_mb = _container_resource_limits(runtime_path, container_id) or (cpus, memory_mb) if cpus < CONTAINER_DESKTOP_MIN_CPUS or memory_mb < CONTAINER_DESKTOP_MIN_MEMORY_MB: return {"success": False, "error": f"Needs at least {CONTAINER_DESKTOP_MIN_CPUS:g} CPU / {CONTAINER_DESKTOP_MIN_MEMORY_MB}MB memory for " f"a desktop session; this container has {cpus:g} CPU / {memory_mb}MB."} try: install_res = subprocess.run( [runtime_path, "exec", container_id, "sh", "-c", _CONTAINER_DESKTOP_INSTALL_SCRIPT], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240) except subprocess.TimeoutExpired: return {"success": False, "error": "Installing desktop packages timed out after 240s."} except Exception as e: return {"success": False, "error": str(e)} if install_res.returncode != 0: if install_res.returncode == 137: # 137 = 128 + SIGKILL, the exit code the shell reports when a child inside it is # killed -- inside a container's own memory cgroup that reliably means the OOM killer, # not a hang. Confirmed against dmesg during testing: microdnf resolving a full XFCE # dependency set pushed a mysqld-plus-install container over its 1GB limit. err = ("Ran out of memory installing the desktop stack (the workload plus the " "package manager exceeded this container's memory limit). It needs more " "headroom than the floor for this action currently requires.") else: err = ((install_res.stderr or "") + (install_res.stdout or "")).strip()[-2000:] or "install failed" print(f"[{time.strftime('%X')}] [!] Could not install desktop stack in {name} " f"({container_id[:12]}): {err}", flush=True) return {"success": False, "error": err} try: launch_res = subprocess.run( [runtime_path, "exec", "-d", container_id, "sh", "-c", _CONTAINER_DESKTOP_LAUNCH_SCRIPT], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30) except Exception as e: return {"success": False, "error": "Packages installed but starting the desktop failed: " + str(e)} if launch_res.returncode != 0: err = (launch_res.stderr or launch_res.stdout or "launch failed").strip()[-2000:] return {"success": False, "error": "Packages installed but starting the desktop failed: " + err} print(f"[{time.strftime('%X')}] [\U0001F5A5️ CONTAINER DESKTOP INSTALLED] {name} " f"({container_id[:12]}) — x11vnc on its own 127.0.0.1:5900", flush=True) return {"success": True} KEEPALIVE_CMD = ["sh", "-c", "while :; do sleep 3600; done"] def _container_state(runtime_path, cid): try: r = subprocess.run([runtime_path, "inspect", cid, "--format", "{{.State.Status}}|{{.State.ExitCode}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=15) if r.returncode == 0 and (r.stdout or "").strip(): status, _, code = (r.stdout or "").strip().partition("|") return status.strip(), code.strip() except Exception: pass return None, None def _ensure_container_network(runtime_path, network): """Creates a user-defined container network if absent, returning True when it is usable. Containers must share a *user-defined* network to reach each other: on podman's default "podman" bridge there is no DNS at all, and rootless containers there could not even reach one another by IP -- verified on a real host, both name resolution and ping failed. A user-defined network runs aardvark-dns, so containers resolve each other by name and by --network-alias. Using one network per tenant also isolates tenants from each other for free: a container on another tenant's network cannot resolve or route to these, which was confirmed the same way. """ if not network: return False try: res = subprocess.run([runtime_path, "network", "inspect", network], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20) if res.returncode == 0: return True except Exception: return False try: # Racing agents on the same host can both try to create it; "already exists" is success. res = subprocess.run([runtime_path, "network", "create", network], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=45) if res.returncode == 0: print(f"[{time.strftime('%X')}] [\U0001F310 NETWORK CREATED] {network}") return True err = ((res.stderr or "") + (res.stdout or "")).lower() if "already exists" in err or "already used" in err: return True print(f"[{time.strftime('%X')}] [\U0001F310 NETWORK UNAVAILABLE] {network}: " f"{(res.stderr or res.stdout or '').strip()[:160]}") except Exception as exc: print(f"[{time.strftime('%X')}] [\U0001F310 NETWORK UNAVAILABLE] {network}: {exc}") return False def provision_container(runtime_info, image, cpus=1, memory_mb=256, disk_mb=1024, ports=None, ttl_seconds=1800, node_id="", report_ctx=None, with_agent=False, agent_ctx=None, command=None, network=None, service_alias=None, env=None, persistent=False, image_args=None): """Runs a resource-capped, network-isolated container via Podman (preferred, rootless) or Docker (fallback), binding requested container ports to loopback-only host ports so they can be streamed to the browser over run_container_port_session. Guardrails are structural, not flag-based: we build the argv ourselves and never accept --privileged/--net=host/--pid=host, volume mounts, or shell interpolation of user input — image/cpu/memory/port values are validated before ever reaching subprocess. report_ctx, when provided, is a dict of {server_url, insecure, host_header, tenant_id, api_key} used solely so the TTL auto-reaper can report the expiry back to the server as a remove_container_result — without it, the server's DB record for the child node would never learn the container was torn down and would linger as a stale 'online' node forever.""" runtime = runtime_info.get("runtime") runtime_path = runtime_info.get("path") or runtime if not runtime_info.get("available") or not runtime_path: return {"success": False, "error": "No usable container runtime on this node."} if not _valid_image_ref(image): return {"success": False, "error": f"Rejected image reference: {image!r}"} # Normalize before use so a short name works on Podman as well as Docker. image = _qualify_image_ref(image) try: cpus = max(0.1, min(float(cpus), 4.0)) except (TypeError, ValueError): cpus = 0.5 try: memory_mb = max(64, min(int(memory_mb), 4096)) except (TypeError, ValueError): memory_mb = 256 try: disk_mb = max(256, min(int(disk_mb), 20480)) except (TypeError, ValueError): disk_mb = 1024 try: ttl_seconds = max(60, min(int(ttl_seconds), 24 * 3600)) except (TypeError, ValueError): ttl_seconds = 1800 container_ports = [] for p in (ports or []): try: p_int = int(p) if 1 <= p_int <= 65535: container_ports.append(p_int) except (TypeError, ValueError): continue container_ports = container_ports[:10] # cap: this is a lightweight demo container, not a router container_name = f"{CONTAINER_NAME_PREFIX}{_node_stub_for_name(node_id)}-{int(time.time())}-{os.urandom(3).hex()}" container_name = "".join(c for c in container_name if c.isalnum() or c in "-_") # Bind an ephemeral loopback host port for each requested container port. # # In sidecar mode we publish nothing at all: the in-container agent shares the workload's # network namespace, so it reaches those ports on its own localhost and tunnels them out # over its WebSocket. That consumes no host port and leaves nothing listening on the host, # which is both tighter than loopback publishing and works on hosts where binding is # restricted or ports are scarce. host_port_map = {} if not with_agent: for cport in container_ports: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("127.0.0.1", 0)) hport = s.getsockname()[1] s.close() host_port_map[cport] = hport # Attach to the tenant network so sibling containers can reach this one by name. Falls back # to the runtime default when the network cannot be created rather than failing the deploy -- # a container with no peers is still useful, and the reason is logged. # Environment variables, mostly database bootstrap credentials. Values are never logged: # MYSQL_ROOT_PASSWORD and friends are secrets, and the container log line below is printed to # stdout where the agent's output is commonly captured to a file. env_args = [] if isinstance(env, dict): for k, v in list(env.items())[:40]: key = "".join(c for c in str(k) if c.isalnum() or c == "_")[:64] if not key or v is None: continue val = str(v) # Reject embedded newlines rather than truncating: a newline in an env value can split # it into what looks like a second assignment to whatever reads the environment. if "\n" in val or "\r" in val: continue env_args += ["-e", f"{key}={val[:512]}"] # Inject the agent API key from agent_ctx into the container environment so the baked # sentinel-agent inside the container receives SENTINEL_API_KEY at startup. Without this, # the in-container agent exits immediately (entrypoint checks for the key and bails with # code 64 if missing), and mesh registration never happens — services stays empty. api_key_for_env = (agent_ctx or {}).get("api_key") if api_key_for_env: env_args += ["-e", f"SENTINEL_API_KEY={api_key_for_env}"] # Dual naming: older baked images may only read APPLOTUS_* prefix env_args += ["-e", f"APPLOTUS_API_KEY={api_key_for_env}"] net_args = [] if network and _ensure_container_network(runtime_path, network): net_args = ["--network", network] alias = "".join(c for c in str(service_alias or "").lower() if c.isalnum() or c in "-_")[:40] if alias: net_args += ["--network-alias", alias] # Deploying straight into persistent mode sets the restart policy at creation. `update # --restart` only landed in podman 5, so on a 4.x host that is the difference between the # runtime bringing the container back after a reboot and only the agent's sweep doing it. restart_args = ["--restart", "always"] if persistent else [] cmd = [ runtime_path, "run", "-d", "--name", container_name, "--cpus", str(cpus), "--memory", f"{memory_mb}m", "--pids-limit", "512", # Deliberately no --dns override: forcing 8.8.8.8/1.1.1.1 here breaks resolution outright # on any host where the network only allows DNS through its own resolver (corporate # networks, locked-down cloud VPCs) -- aardvark-dns still answers container-name lookups # fine and forwards everything else to whatever the host's own resolv.conf already points # at, which is the resolver actually allowed to leave the host. ] + restart_args + net_args + env_args + [ "--security-opt", "no-new-privileges", # Drop all capabilities, then re-add only what common images' root-startup entrypoints # need (chown'ing data dirs, dropping to a service uid, binding <1024 ports) — verified # against nginx:latest and postgres:15-alpine, which both fail to start under a bare # --cap-drop ALL. Still far short of the full default Docker capability set. "--cap-drop", "ALL", "--cap-add", "CHOWN", "--cap-add", "SETUID", "--cap-add", "SETGID", "--cap-add", "DAC_OVERRIDE", "--cap-add", "NET_BIND_SERVICE", # WordPress's entrypoint tar-extracts its own bundled files into /var/www/html on first # boot, then chmods/utimes them -- CHOWN alone does not cover that; chmod/utime on a file # not owned by the calling uid inside the container needs FOWNER too, even as uid 0. # Verified live: without it, that extraction failed outright (WordPress never finished # installing) with a wall of "Cannot change mode: Operation not permitted"; with it, zero. "--cap-add", "FOWNER", ] # Best-effort disk quota — only meaningful with overlay2 + a backing filesystem that supports # project quotas. Not a hard guarantee; we don't fail provisioning if the runtime rejects it. disk_opt_supported = True for cport, hport in host_port_map.items(): cmd += ["-p", f"127.0.0.1:{hport}:{cport}"] if with_agent: # An image that bakes the agent in declares it with the org.applotus.agent label, and the # mount is skipped for it. That is not just tidiness: the mount is read-only, and # upgrade_agent_self replaces the script at its own path — so a mounted agent can never # self-upgrade, while a baked one repairs itself on its first heartbeat. # # Keyed on the label rather than on the image name, so any image built with the agent in it # qualifies without having to be called applotus/*. if _image_bundles_agent(runtime_path, image): print(f"[{time.strftime('%X')}] [\U0001F4E6 AGENT BUNDLED] {image} declares " f"org.applotus.agent — using the image's own copy, no mount", flush=True) else: # Mounts can only be set at creation, so the agent script has to be in place before the # container starts even though it is exec'd in afterwards. Read-only: the workload has # no business modifying the agent it hosts. agent_src = os.path.abspath(sys.argv[0]) if not os.path.isfile(agent_src): return {"success": False, "error": f"Cannot locate agent script to mount ({agent_src})."} cmd += ["-v", agent_src + ":" + AGENT_BUNDLED_IN_CONTAINER_PATH + ":ro"] # Remember where the image sits. Storage flags must be inserted *before* it, and inserting # relative to the end of argv breaks the moment anything follows the image -- which is exactly # how --storage-opt previously ended up inside a user command ("sh: bad option"). image_index = len(cmd) cmd += [image] user_cmd = None if isinstance(command, str) and command.strip(): # Deliberately not shell-split: pass it to the image's shell so quoting and pipes behave # the way the person typing it expects, while still never being interpolated on our side. user_cmd = ["sh", "-c", command.strip()] cmd += user_cmd elif image_args: # Passed straight through as discrete argv items, not wrapped in sh -c like `command` is: # these are meant to reach the image's own entrypoint as "$@" (e.g. mysqld's --port), which # a single shell-string argument would not do -- the entrypoint's own arg parsing (e.g. # "does $1 start with a dash") expects them separate, exactly as `docker run image --flag # value` would deliver them. Mutually exclusive with `command`: both are ways of extending # the image's default CMD, and combining them would just be confusing. user_cmd = [str(a) for a in image_args] cmd += user_cmd def _run_with_optional_storage_opt(base_cmd): # Insert at the recorded image index: never by a fixed offset (which can land between a # flag and its value) and never relative to the end (which lands inside a trailing user # command). Everything before image_index is runtime flags; everything after is the # image and its arguments. # # Also returns the argv that actually worked, and where the image sits inside it, so the # manager can rebuild this exact container later against a snapshot image. Recomputing # either from the caller's side would have to re-derive whether the quota flag survived. trial_cmd = base_cmd[:image_index] + ["--storage-opt", f"size={disk_mb}m"] + base_cmd[image_index:] res = subprocess.run(trial_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=60) if res.returncode == 0: return res, True, trial_cmd, image_index + 2 # Storage driver likely doesn't support per-container size quotas (e.g. overlay without # pquota) — retry without the quota flag rather than failing the whole provision. Clear # the name first: some runtime/version combinations reserve the container name before # failing on an unsupported flag, which would make the retry die on a name conflict. subprocess.run([runtime_path, "rm", "-f", container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) res2 = subprocess.run(base_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=60) return res2, False, list(base_cmd), image_index try: result, disk_opt_supported, created_argv, created_image_index = _run_with_optional_storage_opt(cmd) except subprocess.TimeoutExpired: return {"success": False, "error": "Container start timed out after 60s (image pull may be slow)."} except Exception as e: return {"success": False, "error": str(e)} if result.returncode != 0: return {"success": False, "error": (result.stderr or result.stdout or "Unknown error").strip()[:2000]} container_id = (result.stdout or "").strip() expires_at = time.time() + ttl_seconds # Many base and interpreter images (alpine, ubuntu, python:*) run their default CMD and exit # straight away. There is then no live container to exec the agent into -- and no useful # container either. If the caller gave no command of their own, retry once with a keep-alive # so the container persists as a sandbox; otherwise report the exit plainly instead of the # confusing downstream "container is stopped" error. time.sleep(1.2) status, exit_code = _container_state(runtime_path, container_id) if status and status != "running": if user_cmd is None: try: subprocess.run([runtime_path, "rm", "-f", container_id], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) except Exception: pass retry_cmd = cmd + KEEPALIVE_CMD try: result = subprocess.run(retry_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=60) except Exception as e: return {"success": False, "error": str(e)} if result.returncode != 0: return {"success": False, "error": (result.stderr or result.stdout or "").strip()[:2000]} container_id = (result.stdout or "").strip() time.sleep(1.2) status, exit_code = _container_state(runtime_path, container_id) if status and status != "running": return {"success": False, "error": (f"Image {image} exits immediately (status {status}, exit {exit_code}) " "even with a keep-alive. Provide a command that stays running.")} print(f"[{time.strftime('%X')}] [\U0001F4E6 KEEP-ALIVE] {image} exits on its own; " f"started it with a keep-alive so it can host an agent", flush=True) else: return {"success": False, "error": (f"Container exited immediately (status {status}, exit {exit_code}). " f"The command {command!r} did not stay running.")} agent_info = None if with_agent: agent_info = _start_in_container_agent(runtime_path, container_id, container_name, node_id, container_ports, agent_ctx or {}) if not agent_info.get("success"): # The workload is up but unreachable without its agent (nothing is published on the # host in this mode), so a half-provisioned pod is not a useful outcome — tear the # workload down and report the real reason rather than leaving a black box running. try: subprocess.run([runtime_path, "rm", "-f", container_id], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) except Exception: pass return {"success": False, "error": "Container started but its in-container agent failed: " + str(agent_info.get("error"))} _redacted_argv, _redacted_image_index = _redact_argv_for_persistence(created_argv, created_image_index) with active_containers_lock: active_containers[container_id] = { "name": container_name, "runtime": runtime, "runtimePath": runtime_path, "hostPorts": host_port_map, "expiresAt": None if persistent else expires_at, "agentNodeId": (agent_info or {}).get("nodeId"), "sidecarId": (agent_info or {}).get("sidecarId"), "persistent": bool(persistent), "image": image, "serviceAlias": service_alias or None, "createdAt": time.time(), "ttlSeconds": ttl_seconds, "restarts": 0, "lastBackupAt": None, # Persistent containers snapshot immediately so a recovery point exists from the # moment the guarantee starts, not one interval later. "nextBackupAt": time.time() if persistent else None, "backups": [], "restartPolicyApplied": False, "network": network or None, "withAgent": bool(with_agent), "containerPorts": list(container_ports or []), "hibernated": False, "hibernatedAt": None, # Network egress health (DNS + reach to the control plane), checked and, where # possible, auto-remediated by the manager loop. None until the first check runs. "networkOk": None, "networkIssue": None, "networkRemediationAttempted": False, "networkRemediationDetail": None, "nextNetworkCheckAt": time.time() if with_agent else None, # Boot readiness: did the workload's own expected ports actually come up, not just # "the container is running" (PID 1 alive says nothing about a hung entrypoint -- see # the crun/no-new-privileges/gosu incident this was built for). None until confirmed # either way; bootDeadlineAt is when a still-unconfirmed container is worth worrying # about rather than just "still booting". "bootReady": None if container_ports else True, "bootIssue": None, "bootDeadlineAt": (time.time() + CONTAINER_BOOT_READY_TIMEOUT) if container_ports else None, "bootRestarted": False, # In-memory only — never mirrored to disk, because the argv carries the -e bootstrap # credentials. Lets the manager rebuild the container from a snapshot *within this # process's lifetime*; recoverySpec below is what carries that across a restart of it. "spec": {"argv": created_argv, "imageIndex": created_image_index, "image": image, "withAgent": bool(with_agent), "agentCtx": dict(agent_ctx or {})}, # Secret-free counterpart of spec, safe to persist (see _redact_argv_for_persistence). "recoverySpec": {"argv": _redacted_argv, "imageIndex": _redacted_image_index}, } # A single manager thread owns expiry, self-healing and snapshots for every container on this # node. It replaces the per-container sleep-then-reap thread, which could not be told to wait # longer: extending a TTL meant racing a thread that had already committed to its deadline. _set_manager_context(report_ctx, node_id) if persistent: # Already set at creation via --restart; confirm rather than assume, so the UI does not # claim reboot durability the runtime never accepted. restart_ok = _container_restart_policy(runtime_path, container_id) == "always" with active_containers_lock: live = active_containers.get(container_id) if live is not None: live["restartPolicyApplied"] = restart_ok _save_container_state() print(f"[{time.strftime('%X')}] [📦 CONTAINER UP] {container_name} ({container_id[:12]}) image={image} " f"cpus={cpus} mem={memory_mb}m disk_quota_applied={disk_opt_supported} ports={host_port_map} " f"{'persistent (managed)' if persistent else f'ttl={ttl_seconds}s'}", flush=True) return { "success": True, "containerId": container_id, "name": container_name, "runtime": runtime, "cpus": cpus, "memoryMb": memory_mb, "diskMb": disk_mb, "diskQuotaApplied": disk_opt_supported, "hostPorts": host_port_map, "ttlSeconds": ttl_seconds, "expiresAt": None if persistent else expires_at, "persistent": bool(persistent), "serviceAlias": service_alias or None, "withAgent": bool(with_agent), # The ports the workload actually listens on, independent of whether anything was # published on the host. In agent mode nothing is published, so hostPorts is empty and # this is the only record of what is reachable through the tunnel. "containerPorts": container_ports, "agent": ({"nodeId": agent_info.get("nodeId"), "python": agent_info.get("python")} if agent_info else None), } def remove_container(container_id): with active_containers_lock: info = active_containers.pop(container_id, None) if not info: # Not tracked in memory -- an agent restart before the on-disk mirror was written, a # reconciliation sweep dropping it, or a container created by an older agent build. Every # other per-container operation already tolerates this by falling back to a bare runtime # lookup (see get_container_env/get_container_logs); this one did not, so a container that # outlived its in-memory record could never be removed through here again -- only by hand # on the host. Mirror that same fallback: ask the runtime directly whether it still knows # about this ID before giving up. runtime_path = None for cand in ("podman", "docker"): p = shutil.which(cand) if p: runtime_path = p break if not runtime_path: return {"success": False, "error": "Unknown or already-removed container."} try: check = subprocess.run([runtime_path, "inspect", container_id], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15) except Exception: return {"success": False, "error": "Unknown or already-removed container."} if check.returncode != 0: # The runtime itself -- not just our own bookkeeping -- confirms this container does # not exist. The caller's goal ("this should not exist") is already satisfied, so this # is a success, not a failure: the server only clears the node record on a successful # remove_container_result, so reporting failure here would leave a stale, permanently # unremovable ghost record behind for something that is verifiably already gone. print(f"[{time.strftime('%X')}] [📦 CONTAINER REAPED] (untracked, confirmed already gone) ({container_id[:12]})", flush=True) return {"success": True} try: res = subprocess.run([runtime_path, "rm", "-f", container_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30) except Exception as e: return {"success": False, "error": str(e)} if res.returncode != 0: err = (res.stderr or res.stdout or f"exit code {res.returncode}").strip()[:2000] print(f"[{time.strftime('%X')}] [!] CONTAINER REMOVE FAILED (untracked) ({container_id[:12]}): {err}", flush=True) return {"success": False, "error": err} print(f"[{time.strftime('%X')}] [📦 CONTAINER REAPED] (untracked, found via runtime lookup) ({container_id[:12]})", flush=True) return {"success": True} # Clean up sidecar container if running in sidecar mode if info.get("sidecarId"): try: subprocess.run([info["runtimePath"], "rm", "-f", info["sidecarId"]], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20) except Exception: pass try: res = subprocess.run([info["runtimePath"], "rm", "-f", container_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=30) except Exception as e: # Put it back so a later attempt (manual remove, or a future reconciliation pass) can # still find it — dropping it here would strand a running container with no tracking. with active_containers_lock: active_containers.setdefault(container_id, info) return {"success": False, "error": str(e)} if res.returncode != 0: # The CLI refused (daemon lock, storage driver busy, permissions). The container may well # still be running, so do NOT report success — the server would drop the node record and # leave a real orphan behind with nothing tracking it. with active_containers_lock: active_containers.setdefault(container_id, info) err = (res.stderr or res.stdout or f"exit code {res.returncode}").strip()[:2000] print(f"[{time.strftime('%X')}] [!] CONTAINER REMOVE FAILED {info['name']} ({container_id[:12]}): {err}", flush=True) return {"success": False, "error": err} print(f"[{time.strftime('%X')}] [📦 CONTAINER REAPED] {info['name']} ({container_id[:12]})", flush=True) _save_container_state() return {"success": True} def get_container_env(container_id): """Inspects a container and returns its user/deployment-defined environment variables. System and runtime variables (PATH, HOSTNAME, APPLOTUS_*, SENTINEL_*, etc.) are filtered out.""" with active_containers_lock: info = active_containers.get(container_id) info = dict(info) if info else None runtime_path = _runtime_path_for(info) if info else None if not runtime_path: for cand in ("podman", "docker"): p = shutil.which(cand) if p: runtime_path = p break if not runtime_path: return {"success": False, "error": "No container runtime found on this node."} # Inspect the container config environment try: res = subprocess.run([runtime_path, "inspect", "--format", "{{json .Config.Env}}", container_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=20) except Exception as e: return {"success": False, "error": str(e)} if res.returncode != 0: err = (res.stderr or res.stdout or "Inspect failed").strip() return {"success": False, "error": err} raw_out = (res.stdout or "").strip() env_list = [] try: env_list = json.loads(raw_out) if raw_out else [] except Exception: # Fallback if json parsing failed: comma or newline separated pass # Filter out system and agent-injected internal variables ignored_prefixes = ( "PATH=", "HOSTNAME=", "TERM=", "HOME=", "LANG=", "GPG_KEY=", "PYTHON_", "APPLOTUS_", "SENTINEL_", "PWD=", "SHLVL=" ) user_env = {} if isinstance(env_list, list): for item in env_list: if not isinstance(item, str) or not item or "=" not in item: continue k, v = item.split("=", 1) k_upper = k.upper() if any(item.upper().startswith(p) for p in ignored_prefixes): continue if k_upper in ("PATH", "HOSTNAME", "TERM", "HOME", "LANG", "GPG_KEY", "PWD", "SHLVL"): continue user_env[k] = v return {"success": True, "containerId": container_id, "env": user_env} # Bounded well below the 8MB container-proxy limit and the general result-payload size: this is # for a quick "is it actually running/erroring" look, not a log aggregator. CONTAINER_LOGS_MAX_LINES = 500 CONTAINER_LOGS_MAX_BYTES = 200 * 1024 def get_container_logs(container_id, tail=200): """Tails a container's stdout/stderr via the runtime's own log store -- no in-container agent needed, same reasoning as get_container_env: podman/docker already capture this outside the container, so there is nothing to reach into the workload itself for.""" with active_containers_lock: info = active_containers.get(container_id) info = dict(info) if info else None runtime_path = _runtime_path_for(info) if info else None if not runtime_path: for cand in ("podman", "docker"): p = shutil.which(cand) if p: runtime_path = p break if not runtime_path: return {"success": False, "error": "No container runtime found on this node."} try: tail_n = max(1, min(int(tail or 200), CONTAINER_LOGS_MAX_LINES)) except (TypeError, ValueError): tail_n = 200 try: res = subprocess.run( [runtime_path, "logs", "--tail", str(tail_n), "--timestamps", container_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=20) except subprocess.TimeoutExpired: return {"success": False, "error": "Reading logs timed out after 20s."} except Exception as e: return {"success": False, "error": str(e)} # podman/docker logs writes the container's stdout and stderr streams to the CLI's own stdout # and stderr respectively -- both are real log output, not an error about the command itself, # so a non-zero exit (the container has already exited) still returns whatever was captured. combined = (res.stdout or "") + (res.stderr or "") if len(combined) > CONTAINER_LOGS_MAX_BYTES: combined = combined[-CONTAINER_LOGS_MAX_BYTES:] if res.returncode != 0 and not combined.strip(): return {"success": False, "error": (res.stderr or res.stdout or "logs failed").strip()[:2000]} return {"success": True, "containerId": container_id, "logs": combined, "tail": tail_n} def update_container_env(container_id, new_env, timeout=120): """Recreates a container with updated environment variables, preserving its configuration, volumes, network, and in-container agent. Option 1: clean recreation via spec argv.""" if not isinstance(new_env, dict): return {"success": False, "error": "new_env must be a key-value dictionary."} with active_containers_lock: info = active_containers.get(container_id) info = dict(info) if info else None if not info: return {"success": False, "error": f"Container {container_id[:12]} is not tracked by this agent."} spec = info.get("spec") or info.get("recoverySpec") if not spec or not spec.get("argv"): return {"success": False, "error": f"Container {container_id[:12]} has no recorded recreation spec."} runtime_path = _runtime_path_for(info) old_argv = list(spec.get("argv") or []) image_index = spec.get("imageIndex") if not isinstance(image_index, int) or image_index >= len(old_argv): return {"success": False, "error": "Invalid container launch spec."} # Split old_argv into runtime options (before image_index) and image+command (at and after image_index) opts = old_argv[:image_index] image_and_cmd = old_argv[image_index:] # Strip existing -e / --env options from opts filtered_opts = [] i = 0 while i < len(opts): arg = opts[i] if arg in ("-e", "--env"): i += 2 # skip flag and its value continue if arg.startswith("--env="): i += 1 continue filtered_opts.append(arg) i += 1 # Format new env flags new_env_flags = [] sanitized_keys = [] for k, v in list(new_env.items())[:60]: key = "".join(c for c in str(k) if c.isalnum() or c == "_")[:64] if not key or v is None: continue val = str(v) if "\n" in val or "\r" in val: continue new_env_flags += ["-e", f"{key}={val[:1024]}"] sanitized_keys.append(key) new_argv = filtered_opts + new_env_flags + image_and_cmd new_image_index = len(filtered_opts) + len(new_env_flags) # 1. Stop and remove the old container print(f"[{time.strftime('%X')}] [📦 RECREATING CONTAINER] Recreating {info.get('name')} ({container_id[:12]}) with updated env vars...", flush=True) if info.get("sidecarId"): try: subprocess.run([runtime_path, "rm", "-f", info["sidecarId"]], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20) except Exception: pass try: subprocess.run([runtime_path, "rm", "-f", container_id], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) except Exception as e: return {"success": False, "error": f"Failed to stop old container: {e}"} # Also make sure the name isn't still reserved container_name = info.get("name") if container_name: try: subprocess.run([runtime_path, "rm", "-f", container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20) except Exception: pass # 2. Run the newly assembled argv try: res = subprocess.run(new_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=timeout) except Exception as e: return {"success": False, "error": f"Failed to start recreated container: {e}"} if res.returncode != 0: err = (res.stderr or res.stdout or "Container launch failed").strip()[:2000] print(f"[{time.strftime('%X')}] [!] Recreate failed for {container_name}: {err}", flush=True) return {"success": False, "error": err} new_id = (res.stdout or "").strip() if not new_id: return {"success": False, "error": "Runtime returned empty container ID upon recreation."} time.sleep(1.2) # 3. Check container liveness and re-launch in-container agent if requested # info["withAgent"] is always persisted (unlike spec, which may be the redacted recoverySpec # after an agent restart and so has no withAgent/agentCtx of its own) -- prefer it. with_agent = info.get("withAgent", spec.get("withAgent", True)) agent_ctx = spec.get("agentCtx") or {} parent_node_id = info.get("parentNodeId") or "" container_ports = info.get("containerPorts") or [] agent_info = None if with_agent: # Give container a moment before execing in time.sleep(1.0) agent_info = _start_in_container_agent(runtime_path, new_id, container_name, parent_node_id, container_ports, agent_ctx) # 4. Update active_containers with active_containers_lock: record = active_containers.pop(container_id, None) or info record["containerId"] = new_id record["restarts"] = int(record.get("restarts") or 0) + 1 record["lastRestartAt"] = time.time() record["agentNodeId"] = (agent_info or {}).get("nodeId") if agent_info else record.get("agentNodeId") record["sidecarId"] = (agent_info or {}).get("sidecarId") if agent_info else record.get("sidecarId") # Update spec with the new argv and image index spec["argv"] = new_argv spec["imageIndex"] = new_image_index record["spec"] = spec active_containers[new_id] = record _save_container_state() if record.get("persistent"): _apply_restart_policy(runtime_path, new_id, "always") print(f"[{time.strftime('%X')}] [📦 CONTAINER RECREATED] {container_name} ({container_id[:12]} -> {new_id[:12]}) " f"with {len(sanitized_keys)} env keys: {sanitized_keys}", flush=True) return { "success": True, "previousContainerId": container_id, "containerId": new_id, "name": container_name, "envKeys": sanitized_keys, "agent": agent_info } def _parse_size_to_bytes(s): """Parses podman/docker's "45MiB"-style size tokens (no regex needed for this simple a grammar: digits/decimal point, then a unit suffix).""" s = (s or "").strip() i = 0 while i < len(s) and (s[i].isdigit() or s[i] == "."): i += 1 if i == 0: return None try: val = float(s[:i]) except ValueError: return None unit = s[i:].strip().lower() mult = { "b": 1, "kb": 1000, "kib": 1024, "mb": 1000 ** 2, "mib": 1024 ** 2, "gb": 1000 ** 3, "gib": 1024 ** 3, "tb": 1000 ** 4, "tib": 1024 ** 4, }.get(unit) return val * mult if mult is not None else None def _parse_container_usage(cpu_str, mem_str): """Raw numeric cpu%/mem% from stats' "12.3%" and "45MiB / 512MiB" display strings, for the metrics-history buffers below -- same reasoning as get_system_metrics' cpuPercent/ramPercent: added alongside the existing display strings, not replacing them.""" cpu_pct = None try: cpu_pct = round(float((cpu_str or "").strip().rstrip("%")), 1) except ValueError: pass mem_pct = None if mem_str and "/" in mem_str: used_s, limit_s = mem_str.split("/", 1) used_b = _parse_size_to_bytes(used_s) limit_b = _parse_size_to_bytes(limit_s) if used_b is not None and limit_b: mem_pct = round((used_b / limit_b) * 100, 1) return cpu_pct, mem_pct # Cumulative RX+TX bytes and the timestamp they were observed, per container -- stats' NetIO # field is "bytes since container start", not a rate, so a rate needs the previous reading to # diff against. Single-threaded access only (the metrics collector thread below is the only # caller of collect_container_stats), so no lock. _container_net_prev = {} def _parse_net_io(net_str): """"1.2kB / 3.4MB" (rx / tx) -> (rx_bytes, tx_bytes), reusing the same size-token parser as memory usage.""" if not net_str or "/" not in net_str: return None, None rx_s, tx_s = net_str.split("/", 1) return _parse_size_to_bytes(rx_s), _parse_size_to_bytes(tx_s) def _container_net_rate_kbps(container_id, net_str): """KB/s across both directions since the last sample. None until a second sample exists, or if the runtime's counter went backwards (container restarted, counter reset at the source).""" rx_b, tx_b = _parse_net_io(net_str) if rx_b is None or tx_b is None: return None now = time.time() prev = _container_net_prev.get(container_id) _container_net_prev[container_id] = (rx_b, tx_b, now) if not prev: return None prev_rx, prev_tx, prev_ts = prev dt = now - prev_ts if dt <= 0 or rx_b < prev_rx or tx_b < prev_tx: return None return round(((rx_b - prev_rx) + (tx_b - prev_tx)) / dt / 1024, 1) # Rolling metrics history: last 1hr at 1min resolution, last 6hr at 5min resolution -- per entity # (the host itself, keyed by HOST_METRICS_KEY, and separately per active container, keyed by its # containerId). In-memory only, not persisted to disk like _save_container_state -- an agent # restart resetting this "last few hours at a glance" view is an acceptable, deliberate # simplification, not an audit trail. Bucketing runs off wall-clock time so it stays correct # regardless of the metrics collector thread's own interval rather than assuming one tick == one # sample. The client derives a third, 15-minute view from the same 1m tier (just the last 15 # samples) rather than this needing its own deque. HOST_METRICS_KEY = "__host__" _metrics_history_lock = threading.Lock() _metrics_history = {} def _metrics_entity(entity_key): entry = _metrics_history.get(entity_key) if entry is None: now0 = time.time() entry = { "1m": collections.deque(maxlen=60), # 60 x 1min = last 1hr (also covers the 15m view) "5m": collections.deque(maxlen=72), # 72 x 5min = last 6hr # Baselined to entity-creation time, not 0 -- starting these at 0 made the very first # sample's `now - 0 >= threshold` check true for both tiers at once regardless of the # actual interval, appending a premature single-sample "average" into 5m too. "_last_1m_ts": now0, "_last_5m_ts": now0, } _metrics_history[entity_key] = entry return entry def _average_samples(samples, now): keys = set() for s in samples: keys.update(k for k in s.keys() if k != "t") out = {"t": int(now)} for k in keys: vals = [s[k] for s in samples if s.get(k) is not None] out[k] = round(sum(vals) / len(vals), 1) if vals else None return out def record_metric_sample(entity_key, sample): """sample: e.g. {"cpu": 12.3, "ram": 46, "disk": 75} or {"cpu": 5.0, "mem": 40.1, "net": 12.0} -- raw numeric values only (None is fine for a metric this entity doesn't have, e.g. containers have no disk figure today). Call on every collector tick; only actually buckets once real wall-clock time has passed, so it is safe to call unconditionally.""" now = time.time() with _metrics_history_lock: entry = _metrics_entity(entity_key) if now - entry["_last_1m_ts"] < 60: return entry["1m"].append({"t": int(now), **sample}) entry["_last_1m_ts"] = now if now - entry["_last_5m_ts"] >= 300: entry["5m"].append(_average_samples(list(entry["1m"])[-5:], now)) entry["_last_5m_ts"] = now def get_metrics_history(container_id=None): entity_key = container_id or HOST_METRICS_KEY with _metrics_history_lock: entry = _metrics_history.get(entity_key) history = ({"1m": list(entry["1m"]), "5m": list(entry["5m"])} if entry else {"1m": [], "5m": []}) return {"success": True, "containerId": container_id, "history": history} def probe_lan_peers(targets): """Whether this host can open a TCP connection to other nodes on its local network. The fleet cannot answer this from the server side. The server's own port check scans a node's PUBLIC address from the relay, which says nothing about whether two nodes can reach each other directly -- and that is the only question that matters for anything wanting real node-to-node networking (a k3s cluster, chiefly) rather than a relayed tunnel. It also cannot be inferred from the reported addresses, which is the trap this exists to narrow: RFC1918 ranges collide freely, so two nodes can both sit on 10.0.0.0/24 and be on completely unrelated networks -- confirmed live in this fleet, where an Oracle Cloud VM's 10.0.0.8 and a home Mac's 10.0.0.44 share a range and cannot see each other. What this proves and what it does NOT: a successful connect proves this host can open TCP to that ADDRESS, not that the machine answering is the node we meant. On a colliding range the address may belong to some unrelated device on the local network that happens to hold the same private IP -- a home LAN handing 10.0.0.8 to a random device with port 22 open would answer, and look identical to success. So every probed port is reported, success or not, and the caller compares the resulting open/closed pattern against what that node itself reports; a mismatch means the responder is not who we were looking for. That is evidence, not proof. Proving identity needs the target to answer with a secret only it was given, which is a second round trip this does not yet do. A plain TCP connect, not ICMP: ping needs raw sockets (root) on Linux and is widely filtered, while a connect against a port the target is already known to have open is both unprivileged and a truer test -- it proves the path carries TCP, which is what the use case needs. Every target port is tried rather than stopping at the first success, because the pattern across all of them is what the fingerprint comparison above needs. Returns a per-target verdict including the measured RTT, which the caller needs anyway: reachable-but-slow matters as much as unreachable for deciding whether a node can hold a control-plane role. """ results = [] for t in (targets or []): ip = str((t or {}).get("ip") or "").strip() node = (t or {}).get("nodeId") ports = [p for p in ((t or {}).get("ports") or []) if isinstance(p, int) and 0 < p < 65536] if not ip or not ports: results.append({"nodeId": node, "ip": ip, "reachable": False, "error": "No address or no known open port to test"}) continue verdict = {"nodeId": node, "ip": ip, "reachable": False, "triedPorts": ports[:4], "openPorts": [], "closedPorts": []} for port in ports[:4]: s = None started = time.time() try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(2.0) s.connect((ip, port)) verdict["openPorts"].append(port) if not verdict["reachable"]: verdict["reachable"] = True verdict["viaPort"] = port verdict["rttMs"] = int((time.time() - started) * 1000) except Exception as e: verdict["closedPorts"].append(port) verdict.setdefault("errors", {})[str(port)] = str(e) finally: if s is not None: try: s.close() except Exception: pass results.append(verdict) return {"success": True, "probedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "results": results} def collect_container_stats(runtime_info): """Reports live state for the containers this agent provisioned, so the server can refresh each container child node instead of leaving it frozen at provision-time values. There is deliberately no agent inside the workload container — the parent agent is the only thing that can see it, so liveness and metrics have to come from here. Returns None (rather than an empty list) when we can't enumerate, so the server can tell "nothing running" apart from "couldn't ask" and avoid reaping child nodes on a transient CLI failure.""" with active_containers_lock: tracked = dict(active_containers) if not tracked or not runtime_info.get("available"): return [] if runtime_info.get("available") else None runtime_path = runtime_info.get("path") or runtime_info.get("runtime") if not runtime_path: return None # Which of ours are actually still running? try: ps = subprocess.run([runtime_path, "ps", "--format", "{{.ID}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=15) if ps.returncode != 0: return None running_short = set(l.strip() for l in (ps.stdout or "").splitlines() if l.strip()) except Exception: return None # And which still exist at all (running or stopped). A container that is merely stopped is # reported as not-running so the UI can show it offline; one that no longer exists has been # removed out-of-band (podman rm, host reboot, operator cleanup) and must be dropped from # tracking so the server can reap its now-orphaned node record. try: psa = subprocess.run([runtime_path, "ps", "-a", "--format", "{{.ID}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=15) existing_short = set(l.strip() for l in (psa.stdout or "").splitlines() if l.strip()) \ if psa.returncode == 0 else None except Exception: existing_short = None # cpu/mem/net per container. --no-stream keeps this a single sample rather than a live feed. # NetIO comes along for free in the same call -- no new subprocess. usage = {} try: st = subprocess.run([runtime_path, "stats", "--no-stream", "--format", "{{.ID}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=25) if st.returncode == 0: for line in (st.stdout or "").splitlines(): parts = line.split("\t") if len(parts) >= 3 and parts[0].strip(): usage[parts[0].strip()] = { "cpu": parts[1].strip(), "mem": parts[2].strip(), "net": parts[3].strip() if len(parts) >= 4 else None, } except Exception: pass # stats is a nice-to-have; liveness from ps above is the load-bearing part out = [] vanished = [] for cid, info in tracked.items(): short = cid[:12] if existing_short is not None and short not in existing_short and cid not in existing_short: # Gone from the runtime entirely. For an ephemeral container that is the end of it: # stop tracking and omit it so the server reaps the node record. # # A persistent one is left tracked and reported as not-running: untracking here would # take it out from under the manager, which is the only thing that can rebuild it from # a snapshot. The manager drops it itself if recovery is impossible. if not info.get("persistent"): vanished.append(cid) continue is_running = short in running_short or cid in running_short u = usage.get(short) or usage.get(cid) or {} cpu_pct, mem_pct = _parse_container_usage(u.get("cpu"), u.get("mem")) net_kbps = _container_net_rate_kbps(cid, u.get("net")) if is_running else None backups = info.get("backups") or [] out.append({ "containerId": cid, "name": info.get("name"), "running": is_running, "cpu": u.get("cpu"), "mem": u.get("mem"), "cpuPercent": cpu_pct, "memPercent": mem_pct, "netKBps": net_kbps, "expiresAt": info.get("expiresAt"), # Lifecycle state the manager owns. The server mirrors these onto the child node so the # Containers tab can show what is actually guaranteed for each container rather than # only what was requested at deploy time. "persistent": bool(info.get("persistent")), "restarts": int(info.get("restarts") or 0), "lastRestartAt": info.get("lastRestartAt"), "lastBackupAt": info.get("lastBackupAt"), "backupCount": len(backups), "lastBackupTag": backups[-1].get("tag") if backups else None, "restartPolicyApplied": bool(info.get("restartPolicyApplied")), "serviceAlias": info.get("serviceAlias"), "image": info.get("image"), "createdAt": info.get("createdAt"), "hasAgent": bool(info.get("agentNodeId") or info.get("withAgent", True)), # False after an agent restart: the creation argv is never written to disk, so the # manager can restart this container but cannot rebuild it from a snapshot. "recoverable": bool(info.get("spec") or info.get("recoverySpec")), }) if vanished: with active_containers_lock: for cid in vanished: gone = active_containers.pop(cid, None) if gone: print(f"[{time.strftime('%X')}] [📦 CONTAINER VANISHED] {gone.get('name')} " f"({cid[:12]}) removed outside Applotus — untracking", flush=True) _container_net_prev.pop(cid, None) _save_container_state() return out # ── Dedicated metrics collector thread ────────────────────────────────────────────────────── # get_system_metrics() is cheap (proc/sysctl reads), but collect_container_stats() runs up to # three runtime-CLI subprocess calls with timeouts totalling ~55s worst case. Both used to run # inline in the main heartbeat loop -- a slow/hung runtime CLI could stall every heartbeat behind # it. This thread runs independently on its own timer and publishes into a lock-protected # snapshot; the main loop just reads whatever is cached, so a stalled collection tick delays only # the next metrics refresh, never a heartbeat. METRICS_COLLECTOR_INTERVAL = 10 _metrics_snapshot_lock = threading.Lock() _metrics_snapshot = {"metrics": None, "containerStats": None} _metrics_collector_started = False # Runtime availability/capabilities as last discovered by the main loop's own (much slower, # ~30-tick) check_containers() probe -- collect_container_stats() only needs to know whether a # runtime is available and where it lives, not to re-probe that on every 10s collector tick. _latest_containers_info = {} def _metrics_collector_tick(): metrics = get_system_metrics() # Merged into the live metrics object so the heartbeat carries it without a second field, and the # server's 5m bucketer picks up gpu/gpuMem automatically. Empty on a host whose GPU utilization # cannot be read, which is most of them -- see sample_gpu. gpu_sample = {} try: gpu_sample = sample_gpu() or {} except Exception: gpu_sample = {} if gpu_sample: metrics["gpu"] = gpu_sample.get("gpu") if "gpuMem" in gpu_sample: metrics["gpuMem"] = gpu_sample["gpuMem"] host_sample = { "cpu": metrics.get("cpuPercent"), "ram": metrics.get("ramPercent"), "disk": metrics.get("diskPercent"), } host_sample.update(gpu_sample) record_metric_sample(HOST_METRICS_KEY, host_sample) with active_containers_lock: has_containers = bool(active_containers) c_stats = collect_container_stats(_latest_containers_info) if has_containers else None if c_stats: for cs in c_stats: record_metric_sample(cs["containerId"], { "cpu": cs.get("cpuPercent"), "mem": cs.get("memPercent"), "net": cs.get("netKBps") }) with _metrics_snapshot_lock: _metrics_snapshot["metrics"] = metrics _metrics_snapshot["containerStats"] = c_stats def _metrics_collector_loop(): while True: try: _metrics_collector_tick() except Exception as e: print(f"[{time.strftime('%X')}] [!] Metrics collector tick failed: {e}", flush=True) time.sleep(METRICS_COLLECTOR_INTERVAL) def _start_metrics_collector(): global _metrics_collector_started if _metrics_collector_started: return _metrics_collector_started = True # Seeded synchronously so the very first heartbeat has real metrics rather than nothing -- # the loop's own first tick is still METRICS_COLLECTOR_INTERVAL seconds away. try: _metrics_collector_tick() except Exception as e: print(f"[{time.strftime('%X')}] [!] Metrics collector seed tick failed: {e}", flush=True) threading.Thread(target=_metrics_collector_loop, daemon=True).start() print(f"[{time.strftime('%X')}] [📊 METRICS COLLECTOR] Started (every {METRICS_COLLECTOR_INTERVAL}s)", flush=True) def get_latest_metrics_snapshot(): with _metrics_snapshot_lock: return dict(_metrics_snapshot) def collect_container_network_health(): """Snapshot of the manager loop's last network-egress probe for each agent-bearing container, for the heartbeat to carry up to the server. Read-only against the registry -- the actual probing and remediation happen in _container_manager_tick on its own schedule, not here.""" with active_containers_lock: tracked = {cid: dict(info) for cid, info in active_containers.items()} out = [] for cid, info in tracked.items(): if not info.get("withAgent"): continue if info.get("networkOk") is None: continue # not checked yet this run out.append({ "containerId": cid, "ok": bool(info.get("networkOk")), "stage": (info.get("networkIssue") or {}).get("stage"), "error": (info.get("networkIssue") or {}).get("error"), "remediated": bool(info.get("networkRemediationAttempted")), "remediationDetail": info.get("networkRemediationDetail"), }) return out def collect_container_boot_health(): """Snapshot of each container's boot-readiness state, for the heartbeat to carry up. Only reports containers that actually have an unresolved bootIssue -- a container still inside its normal grace window is not news, and the server should not treat "still booting" as a fault.""" with active_containers_lock: tracked = {cid: dict(info) for cid, info in active_containers.items()} out = [] for cid, info in tracked.items(): issue = info.get("bootIssue") if not issue: continue out.append({"containerId": cid, "issue": issue}) return out def run_container_port_session(session_id, ws_url, host_port, insecure=False, host_header=None, api_key=None): """Bridges a single container-exposed TCP port to the browser over WebSocket, reusing the same generic byte-pump as run_desktop_session. host_port is always a 127.0.0.1-bound port picked by provision_container — never a container's raw port and never LAN/public-facing.""" with active_desktop_lock: # reuse the same dedup lock; sessions are keyed uniquely regardless of kind if session_id in active_desktop_sessions: return active_desktop_sessions.add(session_id) print(f"[{time.strftime('%X')}] [📦 CONTAINER PORT SESSION] {session_id} -> 127.0.0.1:{host_port}", flush=True) ws = None sock = None try: headers = _ws_auth_headers(api_key) ws = SimpleWebSocketClient(ws_url, insecure=insecure, host_header=host_header, headers=headers) ws.connect(timeout=15) try: ws.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except Exception: pass sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(5.0) sock.connect((_workload_connect_host(), host_port)) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) sock.setblocking(False) bridge_socket_over_ws(sock, ws, label="CONTAINER-PORT", eof_hint="Container may have exited or stopped listening on this port.") except Exception as e: print(f"[{time.strftime('%X')}] [-] Container port session failed: {e}", flush=True) try: if ws: ws.send_frame(json.dumps({"error": str(e)}), opcode=1) except Exception: pass finally: if sock: try: sock.close() except Exception: pass if ws: try: ws.close() except Exception: pass with active_desktop_lock: active_desktop_sessions.discard(session_id) print(f"[{time.strftime('%X')}] [📦 CONTAINER PORT CLOSED] Session {session_id} ended", flush=True) def upgrade_agent_self(download_url, target_version="latest", server_url="", insecure=False, host_header=None, tenant_id=None, api_key=None, task_id=None, node_id=None): """Downloads updated agent script, verifies syntax with py_compile, replaces current file, and executes in-place restart via execv.""" print(f"[{time.strftime('%X')}] [⬆️ UPGRADE START] Initiating upgrade to {target_version}...", flush=True) # 1. Resolve current running script path current_script = sys.argv[0] if current_script == "-c" or not os.path.exists(current_script): # Fallback to standard location or realpath cand_paths = [os.path.realpath(current_script), "/usr/local/bin/sentinel-agent.py", "./sentinel-agent.py"] found = None for p in cand_paths: if os.path.isfile(p): found = p break if not found: err = f"Cannot determine on-disk script path for {current_script} to perform in-place replacement." print(f"[{time.strftime('%X')}] [-] Upgrade failed: {err}", flush=True) return {"success": False, "error": err} current_script = found abs_current_script = os.path.abspath(current_script) tmp_new_script = abs_current_script + ".upgrade.tmp" # Prove we can write the replacement before spending the download. # # The order used to be download-then-write, so a node running from a directory it cannot write # fetched the whole 338KB agent and only then discovered it could not be installed -- once per # attempt, and the server re-queued an attempt every heartbeat. Testing the actual path the upgrade # will use (create, then remove) rather than os.access, which lies under root and with ACLs. try: with open(tmp_new_script, "wb"): pass os.remove(tmp_new_script) except Exception as e: err = (f"Cannot write {tmp_new_script}: {e}. The agent is running from a directory it cannot " f"write -- reinstall it somewhere writable, or run the upgrade with privileges for " f"{os.path.dirname(abs_current_script)}.") print(f"[{time.strftime('%X')}] [-] Upgrade failed: {err}", flush=True) return {"success": False, "error": err} # Pin the download to the server this agent was configured to talk to. # # download_url arrives verbatim from a dispatched task (task.downloadUrl over HTTP, or # msg.downloadUrl over the control socket). Whatever it points at gets fetched, written over # this very script, and exec'd -- as root, on every agent in the fleet. Without this check the # payload only had to be >500 bytes and contain the string "Applotus Sentinel", both trivially # satisfied by an attacker, so anyone able to inject a single task field had remote code # execution on every host running an agent. py_compile only proves the bytes parse as Python; # it says nothing about who wrote them. # # Same-origin is the meaningful boundary here: the agent already trusts its configured server # (that server can dispatch shell commands to it anyway), so restricting upgrades to that same # origin adds no new trust while removing the redirect-to-anywhere primitive entirely. try: from urllib.parse import urlparse want = urlparse(server_url or "") got = urlparse(download_url or "") if not want.netloc: err = "Refusing upgrade: no configured server URL to validate the download against." print(f"[{time.strftime('%X')}] [-] {err}", flush=True) return {"success": False, "error": err} if (got.scheme, got.netloc) != (want.scheme, want.netloc): err = (f"Refusing upgrade: download URL origin {got.scheme}://{got.netloc} does not match " f"the configured server {want.scheme}://{want.netloc}.") print(f"[{time.strftime('%X')}] [-] {err}", flush=True) return {"success": False, "error": err} # Defence in depth: the only legitimate upgrade payload is the agent download endpoint. if not got.path.rstrip("/").endswith(("/download/sentinel-agent.py", "/download/sentinel_agent.py")): err = f"Refusing upgrade: unexpected download path {got.path}." print(f"[{time.strftime('%X')}] [-] {err}", flush=True) return {"success": False, "error": err} except Exception as e: err = f"Refusing upgrade: could not validate download URL ({e})." print(f"[{time.strftime('%X')}] [-] {err}", flush=True) return {"success": False, "error": err} # 2. Download and validate new agent script with retry backoff max_retries = 3 success = False last_err = "" for attempt in range(1, max_retries + 1): try: headers = { "User-Agent": f"Applotus-Sentinel-Agent/{VERSION} (Upgrader)", "Referer": "https://applotus.fly.dev/" } if host_header: headers["Host"] = host_header if api_key: headers["x-api-key"] = api_key elif tenant_id: headers["x-tenant-id"] = tenant_id req = urllib.request.Request(download_url, headers=headers) ctx = ssl_unverified_context if insecure else None print(f"[{time.strftime('%X')}] [⬆️ UPGRADE] Downloading from {download_url} (attempt {attempt}/{max_retries})...", flush=True) with urllib.request.urlopen(req, timeout=30, context=ctx) as resp: content = resp.read() if len(content) < 500 or b"Applotus Sentinel" not in content: raise ValueError(f"Downloaded payload appears corrupted or invalid (size: {len(content)} bytes)") with open(tmp_new_script, "wb") as f: f.write(content) # Validate Python syntax via bytecode compilation print(f"[{time.strftime('%X')}] [⬆️ UPGRADE] Validating Python syntax...", flush=True) py_compile.compile(tmp_new_script, doraise=True) success = True break except Exception as e: last_err = str(e) if os.path.exists(tmp_new_script): try: os.remove(tmp_new_script) except Exception: pass print(f"[{time.strftime('%X')}] [-] Upgrade attempt {attempt} failed: {last_err}", flush=True) if attempt < max_retries: backoff_sec = attempt * 2 # 2s, 4s time.sleep(backoff_sec) if not success: err = f"Upgrade failed after {max_retries} attempts: {last_err}" print(f"[{time.strftime('%X')}] [-] {err}", flush=True) if server_url and task_id and node_id: try: fail_payload = { "nodeId": node_id, "taskId": task_id, "type": "upgrade_result", "command": f"upgrade_agent to {target_version}", "exit_code": 1, "stdout": "", "stderr": err, "duration": 0, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } send_request( f"{server_url}/api/sentinel/agent/result", data=fail_payload, insecure=insecure, host_header=host_header, tenant_id=tenant_id, api_key=api_key ) except Exception: pass return {"success": False, "error": err} # 4. Atomically replace the script file & restore executable permission try: orig_mode = os.stat(abs_current_script).st_mode if os.path.exists(abs_current_script) else 0o755 os.chmod(tmp_new_script, orig_mode | 0o111) # ensure executable os.replace(tmp_new_script, abs_current_script) print(f"[{time.strftime('%X')}] [✓ UPGRADE APPLIED] In-place file replaced: {abs_current_script}", flush=True) except Exception as e: if os.path.exists(tmp_new_script): try: os.remove(tmp_new_script) except Exception: pass err = f"Failed replacing {abs_current_script}: {e}" print(f"[{time.strftime('%X')}] [-] File replacement error: {err}", flush=True) return {"success": False, "error": err} # 5. Report success audit back to server if server_url and task_id provided if server_url and task_id and node_id: try: result_payload = { "nodeId": node_id, "taskId": task_id, "type": "upgrade_result", "command": f"upgrade_agent to {target_version}", "exit_code": 0, "stdout": f"Successfully updated sentinel-agent.py to {target_version}. Re-executing process...", "stderr": "", "duration": 1.2, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } send_request( f"{server_url}/api/sentinel/agent/result", data=result_payload, insecure=insecure, host_header=host_header, tenant_id=tenant_id, api_key=api_key ) print(f"[{time.strftime('%X')}] [✓ UPGRADE REPORTED] Audit log saved to server", flush=True) except Exception: pass # 6. Re-exec self in a short background thread to cleanly exit current loop def _reexec(): time.sleep(1.0) print(f"[{time.strftime('%X')}] [🚀 RESTARTING] Restarting agent to apply the upgrade...", flush=True) # Restart the freshly written script, via the platform-correct path (see _respawn_self: # execv mangles a Windows interpreter path containing spaces and kills the child). if not _respawn_self(abs_current_script): print(f"[{time.strftime('%X')}] [-] Could not restart automatically after upgrade. " f"The new agent is on disk at {abs_current_script}; restart it manually.", flush=True) threading.Thread(target=_reexec, daemon=True).start() return {"success": True, "message": f"Upgraded to {target_version} and restarting"} def handle_agent_list_files(target_path="~"): """Lists directory contents safely with metadata for remote file browsing.""" try: user_home = os.path.expanduser("~") jailed, jail_err = _jail_path(target_path) if jail_err: return {"success": False, "error": jail_err, "path": target_path, "userHome": user_home} if jailed is not None: # Under a root, that root is also what an unknown path falls back to. Falling back to # the home directory as the unconfined path does would step straight outside the jail. user_home = jailed resolved = jailed if not os.path.exists(resolved): return {"success": False, "error": f"Path does not exist: {target_path}", "path": target_path, "userHome": user_home} else: if not target_path or target_path == "" or target_path == "~": target_path = user_home resolved = os.path.abspath(os.path.expanduser(target_path)) if not os.path.exists(resolved): # Fallback to user home directory if target doesn't exist if os.path.exists(user_home): resolved = user_home else: return {"success": False, "error": f"Path does not exist: {target_path}", "path": target_path, "userHome": user_home} if not os.path.isdir(resolved): return {"success": False, "error": f"Not a directory: {target_path}", "path": resolved, "userHome": user_home} entries = [] try: with os.scandir(resolved) as it: for entry in it: try: stat = entry.stat(follow_symlinks=False) is_dir = entry.is_dir(follow_symlinks=False) is_symlink = entry.is_symlink() entries.append({ "name": entry.name, "path": os.path.join(resolved, entry.name), "isDir": is_dir, "isSymlink": is_symlink, "size": 0 if is_dir else stat.st_size, "mtime": int(stat.st_mtime * 1000), "mode": oct(stat.st_mode)[-4:] }) except (PermissionError, FileNotFoundError, OSError): entries.append({ "name": entry.name, "path": os.path.join(resolved, entry.name), "isDir": False, "isSymlink": False, "size": 0, "mtime": 0, "mode": "0000" }) except PermissionError: # If access to root or restricted folder failed, try user home directory fallback if resolved != user_home and os.path.exists(user_home): return handle_agent_list_files(user_home) return {"success": False, "error": f"Permission denied accessing directory: {resolved}", "path": resolved, "userHome": user_home} # Sort directories first, then alphabetical by name entries.sort(key=lambda x: (not x["isDir"], x["name"].lower())) parent = os.path.dirname(resolved) if resolved != "/" and resolved != os.path.splitdrive(resolved)[0] + "\\" else None return { "success": True, "path": resolved, "parent": parent, "userHome": user_home, "items": entries, "count": len(entries) } except Exception as e: return {"success": False, "error": str(e), "path": target_path, "userHome": os.path.expanduser("~")} def handle_agent_read_file(target_path, compress_format="gzip", max_bytes=100 * 1024 * 1024): """Reads a file or directory and compresses it on the fly (gzip for single files, zip for folders), returned as base64.""" try: jailed, jail_err = _jail_path(target_path) if jail_err: return {"success": False, "error": jail_err} resolved = jailed if jailed is not None else os.path.abspath(os.path.expanduser(target_path or ".")) if not os.path.exists(resolved): return {"success": False, "error": f"Path does not exist: {target_path}"} if os.path.isdir(resolved): # Compress entire folder on the fly into in-memory ZIP archive import io buf = io.BytesIO() total_size = 0 file_count = 0 with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as zf: for root, dirs, files in os.walk(resolved): for file in files: full_path = os.path.join(root, file) rel_path = os.path.relpath(full_path, resolved) try: st = os.stat(full_path) total_size += st.st_size if total_size > max_bytes: return {"success": False, "error": f"Folder exceeds maximum download quota of {max_bytes // (1024*1024)}MB"} zf.write(full_path, arcname=rel_path) file_count += 1 except (PermissionError, OSError): continue zip_bytes = buf.getvalue() b64_data = base64.b64encode(zip_bytes).decode("ascii") folder_name = os.path.basename(resolved) or "folder" return { "success": True, "path": resolved, "isDir": True, "filename": f"{folder_name}.zip", "format": "zip", "originalSize": total_size, "compressedSize": len(zip_bytes), "fileCount": file_count, "dataB64": b64_data } else: # Single file compression using gzip st = os.stat(resolved) if st.st_size > max_bytes: return {"success": False, "error": f"File exceeds maximum allowed download quota of {max_bytes // (1024*1024)}MB"} with open(resolved, "rb") as f: raw_bytes = f.read() comp_bytes = gzip.compress(raw_bytes, compresslevel=6) b64_data = base64.b64encode(comp_bytes).decode("ascii") file_name = os.path.basename(resolved) return { "success": True, "path": resolved, "isDir": False, "filename": f"{file_name}.gz", "originalName": file_name, "format": "gzip", "originalSize": len(raw_bytes), "compressedSize": len(comp_bytes), "dataB64": b64_data } except Exception as e: return {"success": False, "error": str(e)} # ── Writing files INTO the node (browser -> here) ───────────────────────────────────────────── # # The file browser has only ever read: list_files and download_file. Uploading needed a write # primitive, and so did pasting an image into a desktop session, which is just an upload to a temp # path followed by a clipboard set. # # Chunked rather than one frame carrying the whole base64 payload. Three reasons, all of them things # that actually bite on the hosts this runs on: the server is a 512 MB machine and a 64 MB upload # arriving as one JSON frame costs several times that between the raw frame, the parsed string and # the decoded bytes; read_frame above reassembles nothing, so a single huge frame also has to fit in # one allocation here; and a transfer with no chunk boundaries can report no progress, which for a # slow link is the difference between a progress bar and a UI that looks hung. UPLOAD_MAX_BYTES = 64 * 1024 * 1024 # Per-chunk ceiling. The server sends 1 MB of raw bytes per chunk (~1.37 MB of base64); this is the # limit a *malformed or hostile* chunk is held to, not the expected size, hence the generous margin. UPLOAD_MAX_CHUNK_BYTES = 8 * 1024 * 1024 # Concurrent in-flight uploads. Each holds an open descriptor and a partial file on disk, so this is # a bound on both. Counts transfers that were begun and never finished as well as live ones, so it # is set well above what a person generates (a few browser tabs, a drag of several files, a paste) # rather than at the true concurrency need: a client that dies mid-transfer ties up its slot until # the stale sweep below releases it, and a cap tight enough to be hit by that is a cap that locks # out the next legitimate upload for five minutes. UPLOAD_MAX_CONCURRENT = 8 # An upload whose last chunk arrived this long ago is presumed dead -- the browser was closed, the # tab crashed, the link dropped mid-transfer -- and its partial file is removed. Without this, every # abandoned transfer would leave a .applotus-part file behind forever. UPLOAD_STALE_SECONDS = 300 _upload_sessions = {} _upload_lock = threading.Lock() def _upload_reap_stale(now=None): """Closes and deletes partials for uploads nothing has touched in UPLOAD_STALE_SECONDS. Called on begin rather than from a timer thread: the only moment a stale partial actually costs anything is when a new upload needs a slot, and a reaper thread would be a permanent thread for an event that mostly never happens. """ now = now or time.time() doomed = [] with _upload_lock: for uid, sess in list(_upload_sessions.items()): if now - sess["touched"] > UPLOAD_STALE_SECONDS: doomed.append((uid, sess)) _upload_sessions.pop(uid, None) for uid, sess in doomed: _upload_abandon(sess, "stale") return len(doomed) def _upload_abandon(sess, why): """Closes a partial's handle and removes it. Never raises -- callers are already on an error path.""" try: if sess.get("fh"): sess["fh"].close() except Exception: pass try: if sess.get("part") and os.path.exists(sess["part"]): os.remove(sess["part"]) except Exception: pass print("[%s] [UPLOAD] abandoned %s (%s)" % (time.strftime("%X"), os.path.basename(sess.get("dest") or "?"), why), flush=True) def _safe_upload_name(name): """The leaf filename from a caller-supplied name, with any path structure stripped. basename against BOTH separators, not os.path.basename alone: on a POSIX host os.path.basename(r"..\\..\\etc\\passwd") is the whole string, so a Windows-style path would survive as one long filename -- harmless on its own, but the same input on a Windows host walks out of the target directory. Checking both means the name means the same thing on every host. """ base = os.path.basename(ntpath.basename(str(name or "").replace("\x00", ""))) base = base.strip() # "." and ".." survive basename and are not names; empty is what an all-separator input leaves. if base in ("", ".", ".."): return None if len(base) > 255: return None return base # Accounts that own a *greeter* rather than a session. A file dropped on the desktop must land in the # home of the person sitting at it, and on this fleet's Pop!_OS hosts the enumerated displays include # a :0 owned by cosmic-greeter alongside the real session on :10 -- picking the first owner would put # the file in a system account's home, where nobody will ever see it. _GREETER_ACCOUNTS = frozenset(( "cosmic-greeter", "gdm", "gdm3", "lightdm", "sddm", "kdm", "xdm", "lxdm", "greeter", "nodm", "slim", "_mbsetupuser", "root", )) def _desktop_session_user(): """(username, home, uid, gid, error) for whoever is signed in at this machine's desktop. Exists because the agent is not that person. It commonly runs as root (or SYSTEM), so expanding "~" gives /root -- a drop onto the remote desktop then wrote to /root/Desktop, which the person looking at the screen cannot see, and the file it did write was owned root:root. Resolved per OS from the identity each one actually exposes: - Linux: the owner of an X display, skipping greeter accounts (see _GREETER_ACCOUNTS); - macOS: the console user scutil reports, which is also what the desktop path already trusts; - Windows: the console session user, which is the same identity the clipboard task runs as. Returns an error string rather than raising or guessing. "No one is signed in" is a real and common state -- a headless host, a locked login window -- and the caller has to be able to say so instead of silently writing the file somewhere else. """ system = platform.system() try: if system == "Windows": raw = _console_session_user() if not raw: return (None, None, None, None, "No one is signed in at this machine's console") # "HOST\user" or "DOMAIN\user" -- the profile directory is named for the leaf. leaf = raw.split("\\")[-1].split("/")[-1] base = os.environ.get("SystemDrive", "C:") + os.sep + "Users" home = os.path.join(base, leaf) if not os.path.isdir(home): prof = os.environ.get("USERPROFILE") # Only usable when the agent IS that user; as SYSTEM it points at a service profile. if prof and os.path.basename(prof).lower() == leaf.lower(): home = prof else: return (None, None, None, None, "Could not find the profile directory for %s" % raw) # No uid/gid on Windows: ownership is not transferred, and does not need to be. A file # created inside the user's profile tree inherits that tree's ACL, which grants them # Modify -- the thing chown is for on POSIX. return (leaf, home, None, None, None) # POSIX from here down -- Windows returned above, and pwd does not exist there, so this # cannot be a module-level import. Once, before either branch that reads it: a local import # makes the name local to the whole function, so a second copy further down would leave the # first read in a dead zone on any path that reached it first. import pwd if system == "Darwin": console_user, _login_done = _darwin_console_login_state() if not console_user or console_user in _GREETER_ACCOUNTS: return (None, None, None, None, "No one is signed in at this Mac's console") rec = pwd.getpwnam(console_user) return (console_user, rec.pw_dir, rec.pw_uid, rec.pw_gid, None) owners = [] for entry in (_linux_x_displays_cached() or []): owner = (entry or {}).get("owner") if owner and owner not in _GREETER_ACCOUNTS and owner not in owners: owners.append(owner) if not owners: return (None, None, None, None, "No desktop session was found on this machine") rec = pwd.getpwnam(owners[0]) return (owners[0], rec.pw_dir, rec.pw_uid, rec.pw_gid, None) except KeyError as e: return (None, None, None, None, "No account on this machine matches %s" % e) except Exception as e: return (None, None, None, None, "Could not resolve the desktop session user: %s: %s" % (type(e).__name__, e)) def _desktop_drop_dir(home): """Where a file dropped on the desktop should land: their Desktop folder, else their home. Not created if absent. A machine whose session has no Desktop directory (a tiling WM, a server login) is not one where creating one is helpful -- the home directory is somewhere they will actually look. """ for name in ("Desktop",): p = os.path.join(home, name) if os.path.isdir(p): return p return home def handle_agent_upload_begin(target_dir, name, size=0, as_desktop_user=False): """Opens a partial file for a new upload and returns its id. The jail is applied to the parent directory, not the destination path, and that difference is the point: the destination does not exist yet, so realpath on it resolves the final component literally and a symlink planted at that name would send the write wherever it points while still passing a check on the resolved parent. Jailing the directory we are about to write into and then refusing to follow a symlink at the leaf closes both halves. as_desktop_user resolves the directory here instead of taking the caller's, and records who to hand the finished file to. It is what a drop onto the remote desktop uses: the caller cannot name the right path, because "~" on this side is the agent's home (root's, usually) rather than the home of the person looking at the screen. """ try: if not isinstance(size, int) or size < 0: return {"success": False, "error": "Invalid size"} if size > UPLOAD_MAX_BYTES: return {"success": False, "error": "File exceeds the maximum upload size of %dMB" % (UPLOAD_MAX_BYTES // (1024 * 1024))} leaf = _safe_upload_name(name) if not leaf: return {"success": False, "error": "Invalid file name"} chown_to = None if as_desktop_user: user, home, uid, gid, who_err = _desktop_session_user() if who_err: return {"success": False, "error": who_err} target_dir = _desktop_drop_dir(home) # Recorded for the end of the transfer rather than applied now: the partial is ours # while it is being written, and handing it over before it is complete would let the # recipient open a half-written file. if uid is not None: chown_to = (uid, gid, user) jailed, jail_err = _jail_path(target_dir or "~") if jail_err: return {"success": False, "error": jail_err} resolved_dir = (jailed if jailed is not None else os.path.realpath(os.path.abspath(os.path.expanduser(target_dir or "~")))) if not os.path.isdir(resolved_dir): return {"success": False, "error": "Target directory does not exist: %s" % target_dir} if not os.access(resolved_dir, os.W_OK): return {"success": False, "error": "No write permission on %s" % resolved_dir} dest = os.path.join(resolved_dir, leaf) # A symlink at the destination is refused rather than followed. Overwriting through one # writes outside the directory that was just authorized, and a jail that can be stepped # over by planting a link is not a jail. if os.path.islink(dest): return {"success": False, "error": "Refusing to write through a symlink at the destination"} if os.path.isdir(dest): return {"success": False, "error": "A directory already exists at that name"} _upload_reap_stale() with _upload_lock: if len(_upload_sessions) >= UPLOAD_MAX_CONCURRENT: return {"success": False, "error": "Too many uploads in flight on this node; try again shortly"} part = dest + ".applotus-part" fh = open(part, "wb") uid = "up_%s" % base64.urlsafe_b64encode(os.urandom(9)).decode("ascii").rstrip("=") with _upload_lock: _upload_sessions[uid] = { "fh": fh, "part": part, "dest": dest, "written": 0, "declared": size, "next_seq": 0, "touched": time.time(), "chown_to": chown_to, } print("[%s] [UPLOAD] %s <- %s (%d bytes declared%s)" % (time.strftime("%X"), dest, uid, size, ", for %s" % chown_to[2] if chown_to else ""), flush=True) return {"success": True, "uploadId": uid, "path": dest, "owner": chown_to[2] if chown_to else None} except Exception as e: return {"success": False, "error": "%s: %s" % (type(e).__name__, e)} def handle_agent_upload_chunk(upload_id, seq, data_b64): """Appends one chunk. Sequence numbers are checked, not trusted. A websocket delivers in order, so an out-of-order seq means a bug or a second writer on the same id rather than a reordered network -- either way, writing it would silently corrupt the file at an offset nobody can see, so it is refused and the upload is torn down. """ try: with _upload_lock: sess = _upload_sessions.get(upload_id) if not sess: return {"success": False, "error": "Unknown or expired upload id"} raw = base64.b64decode(data_b64 or "", validate=True) if len(raw) > UPLOAD_MAX_CHUNK_BYTES: with _upload_lock: _upload_sessions.pop(upload_id, None) _upload_abandon(sess, "oversized chunk") return {"success": False, "error": "Chunk exceeds the maximum chunk size"} if int(seq) != sess["next_seq"]: with _upload_lock: _upload_sessions.pop(upload_id, None) _upload_abandon(sess, "out-of-order chunk") return {"success": False, "error": "Chunk %s arrived out of order (expected %d)" % (seq, sess["next_seq"])} # Enforced against bytes actually received, not against the declared size, which is a claim # the caller makes and can be wrong in either direction. if sess["written"] + len(raw) > UPLOAD_MAX_BYTES: with _upload_lock: _upload_sessions.pop(upload_id, None) _upload_abandon(sess, "exceeded size cap mid-transfer") return {"success": False, "error": "Upload exceeded the maximum size of %dMB" % (UPLOAD_MAX_BYTES // (1024 * 1024))} sess["fh"].write(raw) sess["written"] += len(raw) sess["next_seq"] += 1 sess["touched"] = time.time() return {"success": True, "received": sess["written"], "nextSeq": sess["next_seq"]} except ValueError as e: with _upload_lock: sess = _upload_sessions.pop(upload_id, None) if sess: _upload_abandon(sess, "undecodable chunk") return {"success": False, "error": "Chunk was not valid base64: %s" % e} except Exception as e: with _upload_lock: sess = _upload_sessions.pop(upload_id, None) if sess: _upload_abandon(sess, "write error") return {"success": False, "error": "%s: %s" % (type(e).__name__, e)} def handle_agent_upload_end(upload_id, expected_size=None): """Closes the partial and moves it into place, or refuses and removes it. expected_size of -1 means the caller is cancelling rather than finishing -- the browser went away, the request errored partway. Distinguished from a mismatch because cancelling is a normal outcome that should not be logged or reported as a failure, and because the alternative (letting the size check reject it) would depend on -1 never coinciding with a real byte count, which is true but is a trick rather than a stated contract. os.replace on the success path, so the destination either holds the previous file or the complete new one and never a half-written prefix -- something watching that directory (an editor, a file manager, the user) must not be able to observe a torn file. """ with _upload_lock: sess = _upload_sessions.pop(upload_id, None) if not sess: return {"success": False, "error": "Unknown or expired upload id"} if expected_size is not None and int(expected_size) == -1: _upload_abandon(sess, "cancelled by caller") return {"success": True, "cancelled": True, "bytesWritten": sess["written"]} try: sess["fh"].flush() os.fsync(sess["fh"].fileno()) sess["fh"].close() except Exception as e: _upload_abandon(sess, "close failed") return {"success": False, "error": "Could not finalize the file: %s" % e} # A size mismatch means the transfer was truncated. Delivering it would hand over a file that # looks complete and is not, which is worse than failing. if expected_size is not None and int(expected_size) != sess["written"]: _upload_abandon(sess, "size mismatch") return {"success": False, "error": "Transfer incomplete: expected %s bytes, received %d" % (expected_size, sess["written"])} try: if os.path.islink(sess["dest"]): _upload_abandon(sess, "symlink appeared at destination") return {"success": False, "error": "Refusing to write through a symlink at the destination"} os.replace(sess["part"], sess["dest"]) except Exception as e: _upload_abandon(sess, "rename failed") return {"success": False, "error": "Could not move the file into place: %s" % e} # Hand the file to the person whose desktop it was dropped on. Until this point it belongs to # the agent, which is usually root -- a 0644 root-owned file sitting in someone's Desktop # folder is one they can open and cannot edit, rename or delete, which is not what dropping a # file onto your own desktop should produce. # # After the rename, not before: chowning the partial would hand over a half-written file. And # only attempted when the ids were resolved, which _desktop_session_user does not do on Windows # because ACL inheritance from the profile tree already grants the user Modify there. owner = None chown_to = sess.get("chown_to") if chown_to: want_uid, want_gid, owner = chown_to try: os.chown(sess["dest"], want_uid, want_gid) # Explicit rather than inherited from the process umask, which for a service is # frequently 0077 -- that would produce a file in their own home readable only by the # account that no longer owns it. os.chmod(sess["dest"], 0o644) except PermissionError: # Not fatal, and not silent. The file is there and readable; it is the ownership that # could not be transferred, which only root can do. Saying so beats both pretending it # worked and deleting a file the user can see. print("[%s] [UPLOAD] could not give %s to %s (agent is not root)" % (time.strftime("%X"), sess["dest"], owner), flush=True) return {"success": True, "path": sess["dest"], "bytesWritten": sess["written"], "filename": os.path.basename(sess["dest"]), "owner": None, "warning": "Saved, but it is owned by the account the agent runs as rather " "than by %s -- the agent would need root to transfer it." % owner} except Exception as e: print("[%s] [UPLOAD] chown of %s failed: %r" % (time.strftime("%X"), sess["dest"], e), flush=True) return {"success": True, "path": sess["dest"], "bytesWritten": sess["written"], "filename": os.path.basename(sess["dest"]), "owner": None, "warning": "Saved, but ownership could not be set: %s" % e} print("[%s] [UPLOAD] wrote %s (%d bytes%s)" % (time.strftime("%X"), sess["dest"], sess["written"], ", owned by %s" % owner if owner else ""), flush=True) return {"success": True, "path": sess["dest"], "bytesWritten": sess["written"], "filename": os.path.basename(sess["dest"]), "owner": owner} # ── Setting the desktop session's clipboard ─────────────────────────────────────────────────── # # Needed because RFB cannot carry an image. Its ClientCutText is text, and the extended-clipboard # pseudo-encoding that nominally has image and file formats is implemented by neither noVNC nor the # VNC servers this talks to. So an image paste goes around the protocol: the bytes come here over # the control socket and are placed on the clipboard of the session the viewer is looking at, after # which the guest's own Ctrl+V does the paste with no involvement from us. # # "The session the viewer is looking at" is the hard part, and it differs per OS: # - X11: the clipboard belongs to a display, so this needs the same DISPLAY and XAUTHORITY pair # the resolution code already resolves. The agent's own environment is not enough. # - Wayland: wl-copy talks to the compositor over WAYLAND_DISPLAY. # - Windows: the clipboard is per *session*, so a service-context agent setting it would set its # own, invisible to the interactive desktop. _run_in_console_session exists for exactly this. # - macOS: pbcopy cannot take image data at all, so this goes through osascript. CLIPBOARD_MAX_BYTES = 16 * 1024 * 1024 CLIPBOARD_IMAGE_MIMES = { "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/bmp": "bmp", "image/webp": "webp", "image/tiff": "tiff", } def _clipboard_image_tempfile(raw, ext): """Writes the image somewhere the process that will read it can actually reach. On Windows that is not the temp directory. This agent is usually a SYSTEM service, so tempfile's default lands in C:\\Windows\\Temp with an ACL the interactive user cannot read -- and the thing that reads this file is a scheduled task running AS that user, which would fail on access rather than on anything to do with the clipboard. ProgramData is where _run_in_console_session already puts the script and log it hands across the same boundary. """ base = None if platform.system() == "Windows": base = os.environ.get("ProgramData", r"C:\ProgramData") if not os.path.isdir(base): base = None fd, p = tempfile.mkstemp(prefix="applotus-clip-", suffix="." + ext, dir=base) with os.fdopen(fd, "wb") as f: f.write(raw) return p def _x_session_envs(): """Candidate environments for talking to the X session(s) on this host, best guess first. The same pairing the resolution code needs: a DISPLAY plus the XAUTHORITY cookie belonging to whoever owns that display, which is usually not the user this agent runs as. Factored out here because the clipboard has the identical problem -- a selection belongs to a display, so setting it from the agent's own bare environment sets it on nothing the viewer can see. Returns a list, not one env, because a host can serve several displays (a physical :0 and a VNC :1 are the common pair) and nothing here can tell which one the viewer is attached to. Callers try them in order and stop at the first that works. """ envs = [] try: for entry in (_linux_x_displays_cached() or []): disp = (entry or {}).get("display") if not disp: continue env = dict(os.environ) env["DISPLAY"] = disp try: xauth = _find_xauthority_for_display(disp) except Exception: xauth = None if xauth: env["XAUTHORITY"] = xauth envs.append(env) except Exception: pass return envs def _set_clipboard_linux(path, mime): """X11 first, then Wayland, reporting what was actually tried. Ordered by what is likelier to be serving the VNC session rather than by preference: the desktop path here drives Xorg/Xvfb/Xtightvnc displays, and a Wayland session that is being screen-shared over VNC is comparatively rare. """ tried = [] xclip = shutil.which("xclip") if xclip: # Fall back to the ambient environment when no display could be enumerated: an agent started # from inside the session already has the right DISPLAY, and refusing would make the common # single-user case fail for the benefit of the multi-display one. candidates = _x_session_envs() or [dict(os.environ)] for env in candidates: try: with open(path, "rb") as f: r = subprocess.run([xclip, "-selection", "clipboard", "-t", mime, "-i"], stdin=f, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, env=env) if r.returncode == 0: return True, "xclip on %s" % env.get("DISPLAY", "?"), None tried.append("xclip on %s: %s" % (env.get("DISPLAY", "?"), (r.stderr or b"").decode("utf-8", "replace").strip())) except Exception as e: tried.append("xclip on %s: %s" % (env.get("DISPLAY", "?"), e)) else: tried.append("xclip is not installed") wl = shutil.which("wl-copy") if wl: try: with open(path, "rb") as f: r = subprocess.run([wl, "--type", mime], stdin=f, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15) if r.returncode == 0: return True, "wl-copy", None tried.append("wl-copy: %s" % (r.stderr or b"").decode("utf-8", "replace").strip()) except Exception as e: tried.append("wl-copy: %s" % e) else: tried.append("wl-copy is not installed") return False, None, ("Could not set the session clipboard. Tried: %s. Install xclip (X11) or " "wl-clipboard (Wayland) on this machine to enable image paste." % "; ".join(tried)) def _set_clipboard_darwin(path, mime): """osascript, because pbcopy has no image path. The AppleScript class code is the format the clipboard is told it holds, and it must match the bytes: «class PNGf» for PNG, «class JPEG» for JPEG, TIFF for everything else because that is the one raw form the pasteboard converts from reliably. """ klass = {"png": "PNGf", "jpg": "JPEG", "tiff": "TIFF"}.get( CLIPBOARD_IMAGE_MIMES.get(mime, "png"), None) if klass is None: # GIF/BMP/WEBP have no pasteboard class code of their own. Converted to TIFF via sips first, # which ships with macOS, rather than refused. sips = shutil.which("sips") if not sips: return False, None, "This image format needs sips to convert, which was not found" conv = path + ".tiff" r = subprocess.run([sips, "-s", "format", "tiff", path, "--out", conv], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30) if r.returncode != 0: return False, None, ("Could not convert the image for the macOS clipboard: %s" % (r.stderr or b"").decode("utf-8", "replace").strip()) path, klass = conv, "TIFF" script = ('set the clipboard to (read (POSIX file "%s") as «' 'class %s»)' % (path.replace('"', '\\"'), klass)) r = subprocess.run(["osascript", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20) if r.returncode == 0: return True, "osascript (%s)" % klass, None return False, None, ("osascript could not set the clipboard: %s" % (r.stderr or b"").decode("utf-8", "replace").strip()) # Runs in the console session, not in the agent's, because the Windows clipboard is per-session: a # SYSTEM service setting it would set its own, which no interactive desktop can see. # # Reports through __LOG__ and the STARTED -> OK/ERR convention the other console scripts use, which # is the runner's actual contract -- it polls for that file and never reads the task's stdout. The # first version of this used Write-Output and was reported back as "No output was returned", because # the runner was waiting on a log the script never wrote. # # No explicit STA thread. Windows PowerShell 3.0+ already runs STA (it is 2.0 that defaulted to MTA), # and the runner invokes powershell.exe rather than pwsh, so the clipboard API's apartment # requirement is satisfied on the main thread. An earlier attempt pushed SetImage onto a # System.Threading.Thread with SetApartmentState -- a PowerShell ScriptBlock handed to a raw .NET # thread has no runspace to execute in, so it silently did nothing. The apartment is logged on the # error path so a host that somehow is not STA says so instead of just failing. _CLIPBOARD_PS = r""" $log = '__LOG__' function L($m) { $m | Out-File $log -Append -Encoding utf8 } "" | Out-File $log -Encoding utf8 # Before Add-Type, which compiles at runtime and on a cold assembly cache can outlast the caller's # patience: with this line a timeout means "started, unfinished" rather than "never ran". L 'STARTED' try { Add-Type -AssemblyName System.Windows.Forms Add-Type -AssemblyName System.Drawing $img = [System.Drawing.Image]::FromFile('__PATH__') try { # SetDataObject's retrying overload rather than SetImage, which is a single attempt. The # clipboard is a per-session singleton that must be opened exclusively, so any process holding # it loses us the race -- and a VNC server is a near-certain contender, since clipboard sync is # exactly what it is doing on a polling loop. SetImage on this node returned "Requested # Clipboard operation did not succeed" for that reason; (data, copy, retries, delayMs) exists # for it. copy=$true so the contents outlive this process, which exits immediately. [System.Windows.Forms.Clipboard]::SetDataObject($img, $true, 12, 120) L 'OK set' } finally { $img.Dispose() } } catch { $apt = [System.Threading.Thread]::CurrentThread.GetApartmentState() L ("ERR " + $_.Exception.Message + " (apartment: " + $apt + ")") } """ def _set_clipboard_windows(path, mime): body = _CLIPBOARD_PS.replace("__PATH__", path.replace("'", "''")) text = _run_in_console_session(body, "clip", wait_s=30) or "" if "OK set" in text: return True, "console session (System.Windows.Forms)", None # A log holding STARTED and nothing else means the work may still be in flight -- the same # distinction the resolution path draws, and for the same reason: reporting a definite failure # for something that then succeeds teaches the user to trust neither answer. if _console_log_pending(text): return False, None, ("The console session is still working on the clipboard; it may yet " "have applied. Try the paste, and retry this if nothing arrives.") detail = "" for line in text.splitlines(): if line.strip().startswith("ERR"): detail = line.strip()[:300] break return False, None, ("The console session did not accept the clipboard image. %s" % (detail or "No log was produced -- is anyone signed in to this machine?")) def handle_agent_set_clipboard(kind, data_b64=None, text=None, mime="image/png"): """Places an image (or text) on the clipboard of the interactive desktop session. No capability check of its own: the control dispatcher refuses set_clipboard outright on a read-only node, the same way it refuses the uploads, so a second check here would be a second place to keep in step with READONLY_CONTROL_TYPES for no added safety. """ try: if kind == "text": # Text has a native RFB path and the viewer uses it; this exists so the same route can # serve a caller that has no RFB connection, e.g. a share with desktop access that has # not connected yet. if text is None: return {"success": False, "error": "No text supplied"} return _set_clipboard_text(text) if kind != "image": return {"success": False, "error": "Unsupported clipboard kind '%s'" % kind} if mime not in CLIPBOARD_IMAGE_MIMES: return {"success": False, "error": "Unsupported image type '%s'" % mime} raw = base64.b64decode(data_b64 or "", validate=True) if not raw: return {"success": False, "error": "No image data supplied"} if len(raw) > CLIPBOARD_MAX_BYTES: return {"success": False, "error": "Image exceeds the %dMB clipboard limit" % (CLIPBOARD_MAX_BYTES // (1024 * 1024))} path = _clipboard_image_tempfile(raw, CLIPBOARD_IMAGE_MIMES[mime]) try: system = platform.system() if system == "Windows": ok, via, err = _set_clipboard_windows(path, mime) elif system == "Darwin": ok, via, err = _set_clipboard_darwin(path, mime) else: ok, via, err = _set_clipboard_linux(path, mime) finally: # The clipboard owns a copy by now on every path above -- xclip reads the file before # it returns, osascript reads it into the pasteboard, SetImage copies the bitmap -- so # the temp file is ours to remove and leaving it would accumulate one per paste. try: os.remove(path) except Exception: pass if platform.system() == "Darwin" and os.path.exists(path + ".tiff"): try: os.remove(path + ".tiff") except Exception: pass if ok: print("[%s] [CLIPBOARD] image set via %s (%d bytes)" % (time.strftime("%X"), via, len(raw)), flush=True) return {"success": True, "via": via, "bytes": len(raw)} return {"success": False, "error": err or "The clipboard could not be set"} except ValueError as e: return {"success": False, "error": "Image data was not valid base64: %s" % e} except Exception as e: return {"success": False, "error": "%s: %s" % (type(e).__name__, e)} def _set_clipboard_text(text): """Text clipboard, per OS. Separate from the image path because every OS has a simpler tool.""" try: data = text.encode("utf-8") system = platform.system() if system == "Windows": # Same log contract as the image path above -- the runner polls __LOG__ and never reads # stdout. Base64 across the boundary rather than the text itself so no quoting, newline # or code-page question arises between here, a .ps1 file and the console session. body = ("$log = '__LOG__'\n" "function L($m) { $m | Out-File $log -Append -Encoding utf8 }\n" '"" | Out-File $log -Encoding utf8\n' "L 'STARTED'\n" "try {\n" " $t = [System.Text.Encoding]::UTF8.GetString(" "[System.Convert]::FromBase64String('%s'))\n" " Set-Clipboard -Value $t\n" " L 'OK set'\n" "} catch { L ('ERR ' + $_.Exception.Message) }\n" % base64.b64encode(data).decode("ascii")) out = _run_in_console_session(body, "cliptext", wait_s=25) or "" if "OK set" in out: return {"success": True, "via": "console session (Set-Clipboard)"} return {"success": False, "error": "The console session did not accept the text. %s" % (out.strip()[:200] or "No log was produced.")} if system == "Darwin": r = subprocess.run(["pbcopy"], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15) if r.returncode == 0: return {"success": True, "via": "pbcopy"} return {"success": False, "error": "pbcopy failed"} xclip = shutil.which("xclip") if xclip: for env in (_x_session_envs() or [dict(os.environ)]): r = subprocess.run([xclip, "-selection", "clipboard"], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, env=env) if r.returncode == 0: return {"success": True, "via": "xclip on %s" % env.get("DISPLAY", "?")} wl = shutil.which("wl-copy") if wl: r = subprocess.run([wl], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15) if r.returncode == 0: return {"success": True, "via": "wl-copy"} return {"success": False, "error": "No clipboard tool is available (install xclip or wl-clipboard)"} except Exception as e: return {"success": False, "error": "%s: %s" % (type(e).__name__, e)} def get_local_ip(): """This host's own LAN address -- the one a direct SSH from the same network actually needs, which get_public_ip() cannot answer and (before this existed) occasionally returned by accident, mislabeled, whenever every public-IP provider failed. The UDP connect() here never sends a packet (UDP is connectionless; this only consults the routing table to pick an outbound interface), so unlike get_public_ip() this costs no network round trip and needs no timeout. """ try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(("8.8.8.8", 80)) ip = s.getsockname()[0] s.close() return ip except Exception: return None def get_public_ip(): providers = [ "https://api.ipify.org", "https://ifconfig.me/ip", "https://icanhazip.com" ] for url in providers: try: req = urllib.request.Request(url, headers={'User-Agent': 'curl/7.88.1'}) with urllib.request.urlopen(req, timeout=3, context=ssl_unverified_context) as resp: return resp.read().decode('utf-8').strip() except Exception: continue # Every public-IP provider failed -- an egress-restricted host, not a crash. The local address # is still better than nothing here (same shape as before this fallback existed), but a caller # that needs to tell "public" from "local" should use get_local_ip() directly rather than infer # it from this one occasionally returning a LAN address instead of a public one. return get_local_ip() or "127.0.0.1" def check_sudo(): """Detects administrative/sudo capability and running user privileges.""" if platform.system() == "Windows": is_admin = False try: if ctypes and hasattr(ctypes, "windll") and hasattr(ctypes.windll, "shell32"): is_admin = (ctypes.windll.shell32.IsUserAnAdmin() != 0) except Exception: is_admin = False # Parenthesised because Python binds the conditional looser than `or`: written as # `a or b or "Administrator" if is_admin else "User"` this evaluated as # `(a or b or "Administrator") if is_admin else "User"`, so a non-admin agent reported the # literal "User" and discarded the real username -- precisely when knowing which account # the agent runs under is what you need. user = (os.environ.get("USERNAME") or os.environ.get("USER") or ("Administrator" if is_admin else "User")) return {"passwordless": is_admin, "user": user, "is_root": is_admin} if hasattr(os, "geteuid") and os.geteuid() == 0: return {"passwordless": True, "user": "root", "is_root": True} try: res = subprocess.run(["sudo", "-n", "true"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2) has_sudo = (res.returncode == 0) except Exception: has_sudo = False user = os.environ.get("USER") or os.environ.get("LOGNAME") or "user" return {"passwordless": has_sudo, "user": user, "is_root": False} def check_local_ai(): """Discovers local AI model providers and installed models (Ollama, LM Studio, vLLM).""" providers = [] # 1. Ollama on 11434 try: req = urllib.request.Request("http://127.0.0.1:11434/api/tags", headers={"User-Agent": "Sentinel-Agent"}) with urllib.request.urlopen(req, timeout=1.5) as resp: data = json.loads(resp.read().decode("utf-8")) models = [m.get("name") for m in data.get("models", []) if m.get("name")] providers.append({"name": "Ollama", "port": 11434, "models": models}) except Exception: pass # 2. LM Studio on 1234 try: req = urllib.request.Request("http://127.0.0.1:1234/v1/models", headers={"User-Agent": "Sentinel-Agent"}) with urllib.request.urlopen(req, timeout=1.5) as resp: data = json.loads(resp.read().decode("utf-8")) models = [m.get("id") for m in data.get("data", []) if m.get("id")] providers.append({"name": "LM Studio", "port": 1234, "models": models}) except Exception: pass # 3. vLLM on 8000 try: req = urllib.request.Request("http://127.0.0.1:8000/v1/models", headers={"User-Agent": "Sentinel-Agent"}) with urllib.request.urlopen(req, timeout=1.5) as resp: data = json.loads(resp.read().decode("utf-8")) models = [m.get("id") for m in data.get("data", []) if m.get("id")] providers.append({"name": "vLLM", "port": 8000, "models": models}) except Exception: pass return { "detected": len(providers) > 0, "providers": providers } # ── Remote-desktop service discovery ──────────────────────────────────────────────────────────── # Candidate local ports, in the order they are preferred. VNC before RDP because the in-browser # viewer speaks RFB and cannot negotiate RDP at all (run_desktop_session refuses port 3389 # outright), so a host offering both is far more useful reported as VNC. # # The ranges exist because the previous code only recognised exactly 5900 and exactly 3389. A second # X display puts x11vnc on 5901, and a container publishing a guest's RDP lands on 3390 -- both were # invisible, so a node with a perfectly usable desktop reported having none. DESKTOP_VNC_PORTS = (5900, 5901, 5902, 5903, 5904, 5905) DESKTOP_RDP_PORTS = (3389, 3390, 3391) def _classify_desktop_ports(open_ports): """Maps a set of open local ports to remote-desktop endpoints, preferred first. Pure and side-effect free so the ordering and membership rules are testable without a host that happens to have the right services running. Returns a list of {"proto": "vnc"|"rdp", "port": int}. Order is meaningful: callers take the first entry of each protocol as that protocol's port, and the first entry overall as the one to advertise. """ open_ports = set(open_ports or ()) endpoints = [] for p in DESKTOP_VNC_PORTS: if p in open_ports: endpoints.append({"proto": "vnc", "port": p}) for p in DESKTOP_RDP_PORTS: if p in open_ports: endpoints.append({"proto": "rdp", "port": p}) return endpoints def _probe_local_ports(ports, timeout=0.25): """Which of `ports` accept a loopback TCP connection right now. Needed because ss/netstat can be blind: they require elevation to show other users' sockets on Darwin and Windows, and a minimal container image ships neither tool -- a container genuinely running x11vnc on 5900 reported no desktop at all for that reason. Sequential on purpose. A closed port on loopback refuses immediately rather than timing out, so the whole candidate set costs microseconds in the common case; the timeout is only a bound for a pathological local firewall, and this runs on the heartbeat path. """ found = set() for p in ports: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout) try: if s.connect_ex(("127.0.0.1", p)) == 0: found.add(p) except Exception: pass finally: try: s.close() except Exception: pass return found def _darwin_display_geometry(): """Reports the Mac's active display count and the main display's pixel size. Added because a viewer showing a black screen could not be told apart from one showing a real screen that happened to be dark. bigbox.local served a byte-identical 78KB in 7 frames on every single session, against 9275KB for a real screen, and neither waking the display nor restarting screensharingd changed it -- so the next question is simply how big the framebuffer actually is, and whether there is a display at all. A count of zero is a genuinely headless Mac, which has no framebuffer to share. ctypes is imported locally on purpose: this module does not import it at top level, and a module-level NameError inside a function that swallows exceptions would leave this silently returning nothing forever. That exact mistake was made once already with re. """ if platform.system() != "Darwin": return None, None, None try: import ctypes cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") cg.CGMainDisplayID.restype = ctypes.c_uint32 cg.CGDisplayPixelsWide.restype = ctypes.c_uint32 cg.CGDisplayPixelsHigh.restype = ctypes.c_uint32 ids = (ctypes.c_uint32 * 16)() count = ctypes.c_uint32(0) if cg.CGGetActiveDisplayList(16, ids, ctypes.byref(count)) != 0: return None, None, None main = cg.CGMainDisplayID() return int(count.value), int(cg.CGDisplayPixelsWide(main)), int(cg.CGDisplayPixelsHigh(main)) except Exception: return None, None, None def _darwin_console_login_state(): """Reports who owns the macOS console and whether their login has completed. This exists because connecting to a Mac that is sitting at the login window does not work, and the failure is silent and extremely confusing. Measured on a real host: a session opened at the login window receives exactly one full framebuffer -- 9275KB, twice, to the kilobyte -- and then no updates at all, while a session opened against a logged-in desktop streamed 283MB at 3068 KB/s. So the password is accepted but the transition is never painted, and the viewer keeps showing a stale login window as though the login had failed. Uses scutil rather than the older `stat -f%%Su /dev/console` trick, which is no longer reliable: checked on macOS 25, /dev/console reports root while a user is very much logged in on the console. Returns (console_user, login_done). Either may be None when the state cannot be determined, and callers must treat None as "unknown" rather than as a login window -- guessing wrong here would warn about a perfectly good session. """ if platform.system() != "Darwin": return None, None try: res = subprocess.run(["scutil"], input="show State:/Users/ConsoleUser", capture_output=True, text=True, timeout=5) out = res.stdout or "" # Parsed with plain string work rather than re, which this module does not import at top level # (only locally, in one unrelated function) -- and because the helper swallows exceptions, a # NameError there would have left the feature silently dead instead of failing loudly. user = None done = None for line in out.splitlines(): stripped = line.strip() if user is None and stripped.startswith("Name :"): value = stripped.split(":", 1)[1].strip() user = value or None elif stripped.startswith("kCGSessionLoginDoneKey :"): done = stripped.split(":", 1)[1].strip() == "TRUE" # loginwindow owning the console IS the login window, whatever the key above reported. if user in ("loginwindow", "_windowserver"): return user, False return user, done except Exception: return None, None def _linux_x_displays(): """Enumerates the X displays present on this host, with who owns each one. Exists because auto-selection got this wrong in a way that was very hard to see. The agent picked a display by listing /tmp/.X11-unix and sorting DESCENDING, so on a host running both a console session (X0) and xrdp (X10) it always chose :10 -- a different X server from the one the user is sitting in front of. The viewer then showed a real but idle desktop and input went into it, which looks identical to a broken connection. Read-only: nothing here starts, stops or reconfigures anything. It only reports what exists so the choice can be made deliberately instead of guessed. """ import os displays = [] sock_dir = "/tmp/.X11-unix" if not os.path.isdir(sock_dir): return displays # Map each X server process to the display it serves, so an owner can be attributed. argv carries # the display as a bare ":N" token for Xorg/Xvfb/Xtightvnc alike. servers = {} try: for pid_dir in os.listdir("/proc"): if not pid_dir.isdigit(): continue try: with open(f"/proc/{pid_dir}/cmdline", "rb") as f: argv = [a.decode("utf-8", "ignore") for a in f.read().split(b"\x00") if a] except Exception: continue if not argv: continue exe = os.path.basename(argv[0]) if not any(k in exe for k in ("Xorg", "Xvfb", "Xtightvnc", "Xwayland", "X")): continue disp = next((a for a in argv[1:] if a.startswith(":") and a[1:].split(".")[0].isdigit()), None) if not disp: continue try: uid = os.stat(f"/proc/{pid_dir}").st_uid except Exception: uid = None # A real seat is started with a vt argument; xrdp and Xvfb sessions are not. seat = any(a.startswith("vt") for a in argv) servers[disp] = {"pid": int(pid_dir), "exe": exe, "uid": uid, "seat": seat} except Exception: pass for fname in sorted(os.listdir(sock_dir)): if not (fname.startswith("X") and fname[1:].isdigit()): continue disp = ":" + fname[1:] info = servers.get(disp, {}) owner = None if info.get("uid") is not None: try: import pwd owner = pwd.getpwuid(info["uid"]).pw_name except Exception: owner = str(info["uid"]) displays.append({ "display": disp, "server": info.get("exe"), "owner": owner, # The distinction that matters: a console seat versus an auxiliary server such as xrdp's. "isSeat": bool(info.get("seat")), }) return displays # Enumeration is cached briefly: check_desktop_env runs on every heartbeat, and walking /proc to # attribute each X server costs a few hundred small reads. A minute is far shorter than a display # appearing or disappearing matters, and it keeps the heartbeat cheap on a busy node. _X_DISPLAYS_CACHE = {"at": 0.0, "value": []} _X_DISPLAYS_TTL = 60.0 def _linux_x_displays_cached(): now = time.time() if now - _X_DISPLAYS_CACHE["at"] < _X_DISPLAYS_TTL: return _X_DISPLAYS_CACHE["value"] try: value = _linux_x_displays() except Exception: value = [] _X_DISPLAYS_CACHE["at"] = now _X_DISPLAYS_CACHE["value"] = value return value def check_desktop_env(): """Detects active graphical desktop environments, display servers, and remote desktop services.""" env_name = None display = os.environ.get("DISPLAY") wayland = os.environ.get("WAYLAND_DISPLAY") desktop_session = os.environ.get("XDG_CURRENT_DESKTOP") or os.environ.get("DESKTOP_SESSION") or os.environ.get("GDMSESSION") # Check common display sockets on Linux if env var not inherited if not display and os.path.exists("/tmp/.X11-unix"): try: # Prioritize sockets owned by current process user uid = os.getuid() for fname in sorted(os.listdir("/tmp/.X11-unix"), reverse=True): if fname.startswith("X") and fname[1:].isdigit(): p = os.path.join("/tmp/.X11-unix", fname) try: if os.stat(p).st_uid == uid: display = f":{fname[1:]}" break except Exception: pass if not display: for fname in sorted(os.listdir("/tmp/.X11-unix"), reverse=True): if fname.startswith("X") and fname[1:].isdigit(): display = f":{fname[1:]}" break except Exception: pass # Detect desktop environment by running process signatures if platform.system() == "Windows": try: tasklist_out = subprocess.check_output(["tasklist"], stderr=subprocess.DEVNULL).decode("utf-8", errors="ignore").lower() if "explorer.exe" in tasklist_out: env_name = "Windows Desktop" else: env_name = "Windows Core/Server" except Exception: env_name = "Windows Desktop" else: try: ps_out = subprocess.check_output(["ps", "-A", "-o", "comm="], stderr=subprocess.DEVNULL).decode("utf-8", errors="ignore") procs = set([p.strip().lower() for p in ps_out.splitlines() if p.strip()]) if "gnome-shell" in procs: env_name = "GNOME" elif "xfce4-session" in procs or "xfwm4" in procs: env_name = "XFCE" elif "kwin_x11" in procs or "kwin_wayland" in procs or "plasmashell" in procs: env_name = "KDE Plasma" elif "mate-session" in procs: env_name = "MATE" elif "cinnamon" in procs: env_name = "Cinnamon" elif "lxsession" in procs: env_name = "LXDE" elif "openbox" in procs: env_name = "Openbox" elif "i3" in procs: env_name = "i3" elif "sway" in procs: env_name = "Sway" elif "wayfire" in procs: env_name = "Wayfire" elif "windowserver" in procs or platform.system() == "Darwin": env_name = "macOS Quartz" except Exception: pass if not env_name and desktop_session: env_name = desktop_session.strip().title() # Check for listening VNC / RDP ports. # # The scan result is unioned with a direct loopback probe of every candidate rather than probing # only as a fallback: ss/netstat can see some sockets and miss others on the same host (elevation # on Darwin/Windows), so "the scan found one VNC port" is not evidence it found them all. listening = set(get_listening_ports()) listening |= _probe_local_ports(DESKTOP_VNC_PORTS + DESKTOP_RDP_PORTS) desktop_endpoints = _classify_desktop_ports(listening) _console_user, _login_done = _darwin_console_login_state() _disp_count, _disp_w, _disp_h = _darwin_display_geometry() vnc_port = next((e["port"] for e in desktop_endpoints if e["proto"] == "vnc"), None) rdp_port = next((e["port"] for e in desktop_endpoints if e["proto"] == "rdp"), None) # Check for available helper binaries (e.g. x11vnc) for on-demand capture has_x11vnc = False for cand in ["/usr/bin/x11vnc", "/usr/local/bin/x11vnc", "x11vnc"]: try: res = subprocess.run(["which", cand], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) if res.returncode == 0 and res.stdout.strip(): has_x11vnc = True break except Exception: pass is_windows = (platform.system() == "Windows") # An agent with x11vnc and a display can *start* a VNC server on demand, so that counts as a real # capability even with nothing listening yet. can_start_vnc = bool(display and has_x11vnc) # Reports what was actually found, and nothing more. # # This used to include `(is_windows and env_name == "Windows Desktop")`, which made every Windows # host with explorer.exe running claim a desktop service regardless of whether one existed. A # Windows VM with no VNC installed and no RDP listening therefore advertised available:true, # type:"rdp", port:3389 -- all three invented -- and run_desktop_session then refused port 3389 # by design, so the UI offered an action that could only ever fail. # # Being honest here also routes the user somewhere useful: the dashboard gates its "Install # Desktop" action on `!available`, so a host with genuinely no desktop service now gets the offer # that would fix it instead of a button that errors. available = bool(vnc_port is not None or rdp_port is not None or can_start_vnc) # If VNC (TightVNC on Windows, x11vnc on Linux, ARD on macOS) is present, prefer VNC for seamless in-browser canvas streaming across any network # "none" rather than guessing "rdp" on Windows or "desktop" elsewhere. A guess here is what # produced the invented port 3389 below. service_type = "vnc" if (vnc_port or can_start_vnc) else ("rdp" if rdp_port else "none") return { "available": available, "type": service_type, "env": env_name or ("Headless/X11" if display else "None"), "display": display or ":0", # Only ever a detected port, or 5900 for the on-demand x11vnc case where the agent is about # to bind it itself. None when there is nothing to connect to -- previously this fell back to # 3389 on any Windows host, which is the value run_desktop_session refuses. "port": vnc_port or (5900 if can_start_vnc else rdp_port), "has_vnc_service": vnc_port is not None, "has_x11vnc_helper": has_x11vnc, "has_rdp_service": rdp_port is not None, # Every X display on the host, so the target can be chosen deliberately rather than guessed. # Auto-selection sorted /tmp/.X11-unix descending and therefore preferred an xrdp server (:10) # over the console (:0), which showed a real but idle desktop and was extremely hard to tell # apart from a broken connection. Empty off Linux. "displays": _linux_x_displays_cached() if platform.system() == "Linux" else [], # macOS only, and None elsewhere or when undetermined. A Mac at its login window cannot be # driven through this viewer (see _darwin_console_login_state), so the UI needs to be able to # say so before the user spends a minute typing a password into a frozen framebuffer. "consoleUser": _console_user, "loginDone": _login_done, # macOS display geometry, so a black viewer can be told apart from a real but dark screen. A # count of 0 means a headless Mac with no framebuffer to share; None means undetermined. "displayCount": _disp_count, "displayWidth": _disp_w, "displayHeight": _disp_h, # Everything found, so the server can choose rather than being handed a single guess: a host # with VNC on 5901 and RDP on 3390 now reports both, and either can be acted on. "endpoints": desktop_endpoints, # Why there is nothing to connect to, for the UI to explain instead of failing silently. "reason": None if available else ("no_remote_desktop_service" if not is_windows else "no_vnc_server_installed_and_rdp_not_listening") } def _respawn_self(script_path=None): """Restart this agent process, correctly on Windows as well as POSIX. os.execv on Windows goes through the C runtime, which re-joins argv into a single command line. An unquoted interpreter path containing a space -- the default install location is "C:\\Program Files\\Python313\\python.exe" -- is then split at that space, so the child tries to execute "C:\\Program" and resolves the remainder relative to the working directory: C:\\Program: can't open file 'C:\\Windows\\system32\\Files\\Python313\\python.exe' execv does not raise in that case, it succeeds and the *child* dies, so an except-branch fallback never fires and the agent simply disappears after upgrading. Spawning via subprocess with an argv list lets the platform quote each element, so Windows takes that route instead. """ script = os.path.abspath(script_path or sys.argv[0]) args = [sys.executable, script] + sys.argv[1:] sys.stdout.flush() sys.stderr.flush() if os.name == "nt": try: subprocess.Popen(args, close_fds=False) except Exception as e: print(f"[{time.strftime('%X')}] [-] Windows respawn failed: {e}", flush=True) return False os._exit(0) try: os.execv(sys.executable, args) except Exception as e: print(f"[{time.strftime('%X')}] [-] execv failed: {e}. Attempting subprocess spawn...", flush=True) try: subprocess.Popen(args) os._exit(0) except Exception as e2: print(f"[{time.strftime('%X')}] [-] Subprocess spawn failed: {e2}", flush=True) return False return True # Where a container runtime lives when PATH does not know about it. Checked in order, and only # after both PATH lookups below have failed. _RUNTIME_DIRS = ( "/opt/homebrew/bin", # Homebrew on Apple Silicon -- absent from launchd's PATH "/usr/local/bin", # Homebrew on Intel, and most manual installs "/opt/podman/bin", # podman's own macOS installer package "/usr/bin", "/bin", "/Applications/Docker.app/Contents/Resources/bin", ) # Windows equivalents, for a tool that drops a binary somewhere and never touches PATH. The machine # PATH is checked first (see _windows_machine_path_dirs) because it is authoritative; this is the # fallback for installers that do not update it. _WINDOWS_RUNTIME_DIRS = ( r"C:\Program Files\Podman", # podman 6.x machine-wide MSI r"C:\Program Files\RedHat\Podman", # older podman layout r"C:\Program Files\Docker\Docker\resources\bin", # Docker Desktop ) def _windows_machine_path_dirs(): r"""Directories on the MACHINE PATH, read from the registry. Empty off Windows or on failure. The agent's own PATH is a snapshot taken when the process started, so a service that has been up for days cannot see anything installed since. That is not a hypothetical: podman was installed to C:\Program Files\Podman, the MSI added it to the machine PATH, and the agent went on reporting not_installed because its own environment predated the install. The registry value is authoritative and current, which makes this the general fix rather than a list of paths that has to grow with every tool. """ if platform.system() != "Windows": return [] try: import winreg except Exception: return [] dirs = [] # Machine first, then the user hive: a per-user install is still worth finding, even though the # agent usually runs as SYSTEM and will not see another user's entries. for hive, key in ((winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Control\Session Manager\Environment"), (winreg.HKEY_CURRENT_USER, "Environment")): try: with winreg.OpenKey(hive, key) as k: raw, _ = winreg.QueryValueEx(k, "Path") except Exception: continue for part in str(raw or "").split(";"): part = os.path.expandvars(part.strip().strip('"')) if part and part not in dirs: dirs.append(part) return dirs def _which(cand_names): # shutil.which, not `which`: the latter is a Unix binary that doesn't exist on Windows, so # shelling out to it made every Windows lookup silently return None no matter what was # actually installed. shutil.which also honours PATHEXT, so bare "docker" finds docker.exe. for cand in cand_names: try: p = shutil.which(cand) if p: return p except Exception: pass # The process PATH is not evidence of absence on a service-managed agent. A LaunchAgent # inherits launchd's PATH, which has no /opt/homebrew/bin, so a Homebrew-installed podman went # undetected and the fleet reported "not_installed" for a runtime that was installed and # running. Reuses the login-shell PATH that 1.49.0 already snapshots once for dispatched # commands rather than deriving it again. try: login_path = _login_shell_path() except Exception: login_path = None if login_path: for cand in cand_names: try: p = shutil.which(cand, path=login_path) if p: return p except Exception: pass # Last resort: the usual install locations, existence- AND executable-checked. Same reasoning # as the macOS installer probing absolute python3 paths instead of trusting PATH. # The machine PATH from the registry comes first: it is current, whereas this process's own # environment is a snapshot from whenever the agent last started. search_dirs = list(_RUNTIME_DIRS) try: if platform.system() == "Windows": search_dirs = _windows_machine_path_dirs() + list(_WINDOWS_RUNTIME_DIRS) + search_dirs except Exception: pass # On Windows a bare name is not a filename. shutil.which above honours PATHEXT -- the comment # there says so -- but this fallback joined the directory with the bare candidate, so it looked # for "podman" and walked straight past podman.exe. That made the fallback useless on exactly # the platform it was just extended for, and the fallback is the half that matters when the # process PATH is stale. suffixes = [""] if platform.system() == "Windows": exts = [e for e in (os.environ.get("PATHEXT") or ".COM;.EXE;.BAT;.CMD").split(";") if e.strip()] suffixes += [e if e.startswith(".") else "." + e for e in exts] for directory in search_dirs: for cand in cand_names: for suffix in suffixes: # Do not double up on a candidate that already names its extension. if suffix and os.path.splitext(cand)[1]: continue try: p = os.path.join(directory, cand + suffix) if os.path.isfile(p) and os.access(p, os.X_OK): return p except Exception: pass return None def _probe_container_mounts(runtime=None): """Finds directories the agent can actually write container storage into, with free space. Per-container disk quotas are only honoured by some storage drivers (podman on overlay without pquota silently refuses --storage-opt size=), so the useful fallback is knowing which paths are writable and how much room they have. Returned newest-first by free space. """ sysname = platform.system() home = os.path.expanduser("~") if sysname == "Windows": candidates = [os.environ.get("ProgramData", r"C:\ProgramData"), home, os.environ.get("TEMP") or r"C:\Windows\Temp"] elif sysname == "Darwin": candidates = [os.path.join(home, ".local/share/containers"), home, "/tmp"] else: candidates = ["/var/lib/containers", "/var/lib/docker", os.path.join(home, ".local/share/containers"), home, "/opt", "/srv", "/tmp"] seen, out = set(), [] for path in candidates: if not path: continue try: real = os.path.realpath(path) except Exception: continue if real in seen: continue seen.add(real) if not os.path.isdir(real): continue # Writability has to be tested, not inferred: os.access lies under some ACL and # read-only-mount setups, so actually create and remove a probe file. writable = False try: probe = os.path.join(real, ".applotus-probe-%s" % os.urandom(4).hex()) with open(probe, "w") as fh: fh.write("x") os.remove(probe) writable = True except Exception: writable = False if not writable: continue try: usage = shutil.disk_usage(real) free_mb = int(usage.free / (1024 * 1024)) total_mb = int(usage.total / (1024 * 1024)) except Exception: continue out.append({"path": real, "freeMb": free_mb, "totalMb": total_mb}) out.sort(key=lambda m: m["freeMb"], reverse=True) return out[:6] def _podman_machine_state(podman_path): """On macOS/Windows, podman runs containers inside a Linux VM ('podman machine'). The CLI being installed says nothing about whether that VM exists or is running, so report both -- a stopped-but-initialized machine only needs a ~10s start, not a multi-GB init.""" try: res = subprocess.run([podman_path, "machine", "list", "--format", "json"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=20) if res.returncode != 0: return {"exists": False, "running": False, "probeFailed": True} machines = json.loads(res.stdout or "[]") if not machines: return {"exists": False, "running": False} # Prefer the default machine, else the first. chosen = next((m for m in machines if m.get("Default")), machines[0]) running = bool(chosen.get("Running")) info = { "exists": True, "running": running, "name": chosen.get("Name"), } # Container memory can't exceed the VM's own allocation, so surface it as a ceiling. mem = chosen.get("Memory") cpus = chosen.get("CPUs") try: # Podman reports Memory in MiB on some versions and bytes on others. mem_i = int(mem) info["memoryMb"] = mem_i if mem_i < 1024 * 1024 else int(mem_i / (1024 * 1024)) except (TypeError, ValueError): pass if isinstance(cpus, int): info["cpus"] = cpus return info except Exception: return {"exists": False, "running": False, "probeFailed": True} def _runtime_os_type(runtime_path): """Which image family the runtime will actually run: 'linux' or 'windows'. On Windows this is the difference between Linux containers (WSL2/Hyper-V backend) and native Windows containers, which can only run Windows base images -- nginx:alpine will not run there, so the image list has to follow this value.""" try: res = subprocess.run([runtime_path, "info", "--format", "{{.OSType}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=10) val = (res.stdout or "").strip().lower() if res.returncode == 0 and val in ("linux", "windows"): return val except Exception: pass try: res = subprocess.run([runtime_path, "info", "--format", "{{.Host.OS}}"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=10) val = (res.stdout or "").strip().lower() if res.returncode == 0 and val in ("linux", "windows"): return val except Exception: pass return None def check_containers(): """Detects a usable container runtime (rootless Podman preferred, Docker as fallback). Capability-based, not OS-gated: an earlier version refused anything but Linux, which was wrong -- macOS runs the identical hardened container flow once 'podman machine' is up, and Windows runs either Linux or native Windows images depending on its backend. What actually matters is whether a runtime responds, which image family it serves, and (on macOS/Windows) whether its backing VM is running.""" sysname = platform.system() cgroup_version = 2 if os.path.exists("/sys/fs/cgroup/cgroup.controllers") else (1 if sysname == "Linux" else None) base = { "platform": sysname.lower(), "cgroupVersion": cgroup_version, } podman_path = _which(["podman"]) docker_path = _which(["docker"]) # Podman is preferred, but a *broken* podman must not mask a working docker: an unusable # podman used to return immediately, so a host with both (podman rootless misconfigured, # docker fine) reported no container support at all. Failures are recorded and we fall # through; podman's reason is only surfaced if docker cannot serve either. podman_failure = None if podman_path: machine = None proceed = True if sysname in ("Darwin", "Windows"): machine = _podman_machine_state(podman_path) if machine.get("probeFailed"): # Could not ask podman about its VM. Do NOT conclude the VM is missing -- # "not initialized" would send the user off to run a multi-GB init when the # machine may well already exist and simply be unreachable right now. podman_failure = ("runtime_unreachable", machine) proceed = False elif not machine.get("exists"): podman_failure = ("vm_not_initialized", machine) proceed = False elif not machine.get("running"): # Initialized but stopped: a start task fixes this, no install needed. podman_failure = ("runtime_stopped", machine) proceed = False if proceed: rootless = True try: res = subprocess.run([podman_path, "info", "--format", "{{.Host.Security.Rootless}}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=15) if res.returncode == 0 and res.stdout.strip(): rootless = (res.stdout.strip().lower() == "true") elif res.returncode != 0: err = ((res.stderr or "") + (res.stdout or "")).lower() # Rootless podman needs user namespaces; when the kernel or sandbox refuses # them podman dies with "cannot clone"/"re-exec". Name that specifically -- # it is a host configuration problem, not a wedged daemon. if "cannot clone" in err or "re-exec" in err or "namespace" in err: podman_failure = ("rootless_userns_denied", machine) else: podman_failure = ("runtime_unreachable", machine) proceed = False except Exception: pass if proceed: out = dict(base, available=True, runtime="podman", rootless=rootless, path=podman_path, containerPlatform=_runtime_os_type(podman_path) or "linux", mounts=_probe_container_mounts("podman")) if machine: out["machine"] = machine return out if docker_path: usable = False try: res = subprocess.run([docker_path, "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15) usable = (res.returncode == 0) except Exception: usable = False out = dict(base, available=usable, runtime="docker", rootless=False, path=docker_path, containerPlatform=(_runtime_os_type(docker_path) or "linux") if usable else None, mounts=_probe_container_mounts("docker")) if not usable: # Installed but this user can't reach the daemon (not in the docker group, daemon # stopped, or on macOS/Windows no backend VM). An absent reason surfaces as a bare # "None" in the fleet view and reads like a detection bug. out["reason"] = "daemon_unreachable" if podman_failure: # Neither runtime works: report podman's reason, which is the actionable one # (e.g. a stopped VM has a one-click start; a dead docker socket does not). out["runtime"] = "podman" out["path"] = podman_path out["reason"] = podman_failure[0] if podman_failure[1]: out["machine"] = podman_failure[1] out["dockerAlsoUnavailable"] = True return out if podman_failure: return dict(base, available=False, runtime="podman", rootless=True, path=podman_path, containerPlatform="linux", reason=podman_failure[0], machine=podman_failure[1] or None, mounts=_probe_container_mounts("podman")) return dict(base, available=False, runtime=None, rootless=False, reason="not_installed", mounts=_probe_container_mounts()) def run_local_ai_inference(provider_name, model_name, prompt, timeout=120): """Executes local model inference over localhost HTTP API with token limits for snappy responses.""" try: if not provider_name or provider_name.lower() == "ollama": url = "http://127.0.0.1:11434/api/generate" body = json.dumps({ "model": model_name, "prompt": prompt, "stream": False, "options": { "num_predict": 128, "temperature": 0.2 } }).encode("utf-8") req = urllib.request.Request(url, data=body, headers={"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=timeout) as resp: data = json.loads(resp.read().decode("utf-8")) return {"success": True, "output": data.get("response", "")} else: # Standard OpenAI compatible /v1/chat/completions (LM Studio, vLLM) port = 1234 if provider_name.lower() == "lm studio" else 8000 url = f"http://127.0.0.1:{port}/v1/chat/completions" body = json.dumps({ "model": model_name, "messages": [{"role": "user", "content": prompt}], "temperature": 0.3, "max_tokens": 128 }).encode("utf-8") req = urllib.request.Request(url, data=body, headers={"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=timeout) as resp: data = json.loads(resp.read().decode("utf-8")) content = data.get("choices", [{}])[0].get("message", {}).get("content", "") return {"success": True, "output": content} except (TimeoutError, urllib.error.URLError, socket.timeout) as e: err_str = str(e) if "timed out" in err_str.lower() or isinstance(e, (TimeoutError, socket.timeout)): return {"success": False, "error": f"Local AI inference timed out after {timeout}s"} return {"success": False, "error": f"Local AI request error: {err_str}"} except Exception as e: return {"success": False, "error": str(e)} # GPU inventory is cheap but not free (lspci/ioreg/nvidia-smi are subprocesses), and hardware does not # change under a running agent, so it is detected once and reused. _gpu_inventory = None _gpu_inventory_lock = threading.Lock() def _run_quiet(argv, timeout=4): """Captured stdout for a command, or None. Never raises, never inherits our stdio.""" try: r = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=timeout) return r.stdout if r.returncode == 0 else None except Exception: return None def detect_gpus(): """A list of {vendor, model, memoryMB, utilSource} for the GPUs on this host. `utilSource` is what can actually report live utilization for that GPU -- "nvidia-smi", "sysfs", or None. None means the card is real but its utilization is not readable by this process, which is the case for Apple Silicon without root and for older AMD drivers. Callers must treat None as "do not report a percentage" rather than as zero. """ global _gpu_inventory with _gpu_inventory_lock: if _gpu_inventory is not None: return _gpu_inventory gpus = [] system = platform.system() # NVIDIA first, on every platform: it is the only source that gives utilization directly. out = _run_quiet(["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"]) if out: for line in out.strip().splitlines(): parts = [p.strip() for p in line.split(",")] if not parts or not parts[0]: continue mem = None if len(parts) > 1: try: mem = int(float(parts[1])) except (TypeError, ValueError): mem = None gpus.append({"vendor": "NVIDIA", "model": parts[0], "memoryMB": mem, "utilSource": "nvidia-smi"}) if system == "Linux" and not gpus: # DRM cards, with utilization only where the driver exposes gpu_busy_percent. try: for card in sorted(os.listdir("/sys/class/drm")): if not (card.startswith("card") and card[4:].isdigit()): continue busy = "/sys/class/drm/%s/device/gpu_busy_percent" % card model = None for p in ("/sys/class/drm/%s/device/label" % card, "/sys/class/drm/%s/device/uevent" % card): try: with open(p) as fh: txt = fh.read() if "DRIVER=" in txt: model = txt.split("DRIVER=", 1)[1].split("\n")[0].strip() elif txt.strip(): model = txt.strip().split("\n")[0] if model: break except Exception: continue gpus.append({"vendor": "DRM", "model": model or card, "memoryMB": None, "utilSource": "sysfs" if os.access(busy, os.R_OK) else None, "busyPath": busy if os.access(busy, os.R_OK) else None}) except Exception: pass if system == "Darwin" and not gpus: # Apple Silicon: the core count is readable from ioreg, utilization is not. powermetrics is # the only source and it requires root, so utilSource stays None unless we happen to be it. cores = None out = _run_quiet(["/usr/sbin/ioreg", "-l"], timeout=8) if out and "gpu-core-count" in out: try: seg = out.split("gpu-core-count", 1)[1] digits = "" for ch in seg: if ch.isdigit(): digits += ch elif digits: break cores = int(digits) if digits else None except Exception: cores = None is_root = hasattr(os, "geteuid") and os.geteuid() == 0 model = "Apple GPU" if cores: model = "Apple GPU (%d cores)" % cores gpus.append({"vendor": "Apple", "model": model, "memoryMB": None, "utilSource": "powermetrics" if is_root else None}) _gpu_inventory = gpus return gpus def sample_gpu(): """{"gpu": pct} and optionally {"gpuMem": pct}, or {} when nothing can be measured. Returning {} rather than zeros is the whole contract: a node whose GPU utilization is unreadable must produce no gpu metric at all, so the chart omits the card instead of drawing a flat line at zero that looks like an idle GPU. """ gpus = detect_gpus() if not gpus: return {} if any(g.get("utilSource") == "nvidia-smi" for g in gpus): out = _run_quiet(["nvidia-smi", "--query-gpu=utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"]) if out: utils, mem_pcts = [], [] for line in out.strip().splitlines(): parts = [p.strip() for p in line.split(",")] try: utils.append(float(parts[0])) except (IndexError, ValueError): pass try: used, total = float(parts[1]), float(parts[2]) if total > 0: mem_pcts.append(used / total * 100.0) except (IndexError, ValueError, ZeroDivisionError): pass # Averaged across cards. A max would hide a second idle GPU; a mean describes the host, # which is what a single host-level gauge should do. res = {} if utils: res["gpu"] = round(sum(utils) / len(utils), 1) if mem_pcts: res["gpuMem"] = round(sum(mem_pcts) / len(mem_pcts), 1) return res busy_paths = [g.get("busyPath") for g in gpus if g.get("utilSource") == "sysfs" and g.get("busyPath")] vals = [] for p in busy_paths: try: with open(p) as fh: vals.append(float(fh.read().strip())) except Exception: continue if vals: return {"gpu": round(sum(vals) / len(vals), 1)} return {} # ── Agent persistence ─────────────────────────────────────────────────────────── # Whether this agent comes back by itself after a reboot, and the machinery to make it so. # # Reported on every handshake so the UI can show which nodes would not survive a restart. That # visibility is the point: a foreground agent looks identical to a managed one until the machine # reboots and never comes back, which is exactly how two nodes in this fleet were silently # non-persistent for weeks. WIN_TASK_NAME = "ApplotusSentinelAgent" LINUX_UNIT_PATH = "/etc/systemd/system/sentinel-agent.service" MACOS_PLIST_REL = "Library/LaunchAgents/com.applotus.sentinel-agent.plist" def _current_username(): """The user this process runs as, for a service definition. getpass is NOT a module-level import in this agent, so calling getpass.getuser() directly would be a NameError at runtime that py_compile cannot see -- the same trap `re` and `ctypes` set earlier. Imported locally, with environment fallbacks for the case where the module is unavailable. """ try: import getpass return getpass.getuser() except Exception: return os.environ.get("USER") or os.environ.get("LOGNAME") or "root" def _own_argv(): """This process's interpreter and arguments, for regenerating its own service definition. Read from the live process rather than reconstructed, so the enrollment key and server URL are whatever is actually working right now. """ return sys.executable, list(sys.argv) _persistence_cache = {"at": 0.0, "value": None} # The check shells out to systemctl / launchctl / schtasks. Heartbeats are every ~10s and this state # changes only when someone reconfigures the host, so it is cached well beyond the heartbeat interval # and refreshed in the background rather than on the critical path. PERSISTENCE_TTL = 300 _agent_instances_cache = {"at": 0.0, "value": None} def _api_key_in(cmdline): """The --api-key value out of a command line, or ''.""" try: import re as _re m = _re.search(r"--api-key[=\s]+(\S+)", cmdline or "") return m.group(1) if m else "" except Exception: return "" def _key_fingerprint(key): """A short sha256 of an API key, or ''. A fingerprint rather than the key itself: this value travels in a heartbeat and ends up in API responses, server logs and the stored node record, and another process's credential has no business being copied into any of those. The server issued the keys, so it can match by hashing its own -- it never needs the plaintext from us. """ key = (key or "").strip().strip('"').strip("'") if not key: return "" try: import hashlib return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] except Exception: return "" def detect_agent_instances(max_age=900.0): """Other running copies of this agent on this machine. Exists because a node can quietly end up with two: ensure_persistence registers a service but deliberately leaves the running process alone, so a hand-started agent and a freshly registered launchd/systemd/schtasks one both keep polling. Observed on real nodes, where the two had different PATHs and dispatched commands therefore succeeded or failed depending on which instance won the poll. Returns {"count": n, "others": [...], "selfPid": p, "selfServiceOwned": bool|None, "ownershipKnown": bool}. count INCLUDES this process, so a healthy node reports 1. Best effort: on any failure it reports 1 rather than guessing, since a false duplicate warning is worse than none. The 15-minute default is deliberate. On Windows this scan spawns a powershell.exe, which costs about half a second and tens of megabytes every time -- unnoticeable on a healthy machine, paging pressure on a 4 GB laptop already at 88% memory, which is what prompted widening it from two minutes. A second agent appears when someone installs or starts one, so there is nothing to be gained from asking often, and the two callers that need current data (ensure_persistence and the dedupe action) pass max_age=0.0 and are unaffected. """ now = time.time() cached = _agent_instances_cache["value"] if cached is not None and (now - _agent_instances_cache["at"]) < max_age: return cached me = os.getpid() rows = [] try: if platform.system() == "Windows": # Name must be a python interpreter, not merely a command line MENTIONING the agent. # Without that, this counted its OWN dispatched probe -- a powershell.exe whose # -Command text contained "sentinel-agent" -- and reported 3 agents on a node with 2. # # SessionId 0 / owner SYSTEM is how install.bat's scheduled task runs (/ru SYSTEM), # whereas a hand-started agent runs as the logged-in user in a non-zero session. That # is the ownership test here; the POSIX ppid == 1 means nothing on Windows, where a # task is a child of svchost.exe. # No per-process Invoke-CimMethod GetOwner. Ownership used to be # `owner == SYSTEM or session == 0`, and the session test alone decides it for an # agent: a task registered with /ru SYSTEM always runs in session 0, and a # hand-started agent never does. Against the live process table the two only disagree # on unrelated system processes (winlogon as SYSTEM in session 1, svchost as NETWORK # SERVICE in session 0), never on anything that could be an agent -- and GetOwner is a # separate RPC per process, which is the part worth not paying for on a loaded box. out = _run_quiet(["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", "Get-CimInstance Win32_Process | Where-Object { " "$_.CommandLine -like '*sentinel-agent.py*' -and " "$_.Name -match '^python' } | ForEach-Object { " "$k = if ($_.CommandLine -match '--api-key\\s+(\\S+)') " "{ $matches[1] } else { '' }; " "\"$($_.ProcessId)|$($_.ParentProcessId)|$($_.SessionId)|$k\" }"], timeout=25) for ln in (out or "").splitlines(): parts = ln.strip().split("|") if len(parts) < 3 or not parts[0].isdigit() or not parts[1].isdigit(): continue session = parts[2].strip() rows.append({ "pid": int(parts[0]), "ppid": int(parts[1]), "uptime": "", "svc": session == "0", # Reported so the server can prefer a user-session agent as reporter. Session 0 # on Windows is a different user profile, not merely a different PATH: winget # and anything else installed per-user is genuinely absent there. "session": int(session) if session.isdigit() else None, "keyFp": _key_fingerprint(parts[3] if len(parts) > 3 else ""), }) else: # etime rather than lstart: a single whitespace-free token, so the command tail can be # split off cleanly. lstart contains spaces and makes this parse ambiguous. out = _run_quiet(["ps", "-Ao", "pid=,ppid=,etime=,command="], timeout=10) for ln in (out or "").splitlines(): parts = ln.strip().split(None, 3) if len(parts) < 4 or not parts[0].isdigit(): continue pid, ppid, etime, cmd = int(parts[0]), parts[1], parts[2], parts[3] if "sentinel-agent.py" not in cmd: continue # A shell command that merely MENTIONS the agent is not an agent. Dispatched # commands grepping for it would otherwise count themselves. if "grep" in cmd or " ps " in cmd or cmd.lstrip().startswith("ps "): continue if "python" not in cmd.lower() and not cmd.rstrip().endswith("sentinel-agent.py"): continue ppid_i = int(ppid) if ppid.isdigit() else 0 # Adopted by init is how launchd and systemd ownership shows up. # No session id on POSIX. The Windows session-0 distinction is about a # separate user profile; "adopted by init" is not the same thing, and a macOS # LaunchAgent is adopted by launchd while still running as the user. rows.append({"pid": pid, "ppid": ppid_i, "uptime": etime, "svc": ppid_i == 1, "session": None, "keyFp": _key_fingerprint(_api_key_in(cmd))}) except Exception: rows = [] if not rows: # Could not read the process table. Report the one instance we know about for certain. res = {"count": 1, "others": [], "selfPid": me, "ownershipKnown": False, "selfServiceOwned": None} else: mine = [r for r in rows if r.get("pid") == me] others = [r for r in rows if r.get("pid") != me] # Ownership is only "known" if we actually saw this process in the table. If the scan # missed us -- a permissions problem, or a mismatched interpreter name -- then claiming to # know which process is service-owned would be guessing about the very thing the one-click # fix acts on. ownership_known = bool(mine) res = { "count": len(rows), "others": [{"pid": r["pid"], "ppid": r["ppid"], "uptime": r["uptime"], "serviceOwned": r["svc"], "session": r.get("session"), "keyFp": r.get("keyFp") or ""} for r in others[:5]], "selfKeyFp": (mine[0].get("keyFp") or "") if mine else "", "ownershipKnown": ownership_known, # The reporting process's own PID. On most affected nodes the duplicate IS this # process, so without it the one thing needing a stop is the one the UI cannot name. "selfPid": me, "selfSession": (mine[0].get("session") if mine else None), "selfServiceOwned": mine[0]["svc"] if mine else None, } _agent_instances_cache["value"] = res _agent_instances_cache["at"] = now return res # Settings `schtasks /create` cannot express. It accepts a trigger and a principal and silently # takes the Task Scheduler default for everything else, and those defaults are tuned for a user's # own batch job on a desktop: skip it on battery, abandon it after 72 hours, never retry. For an # agent whose entire job is to be reachable they are all wrong, and on a laptop the battery pair # alone is enough to keep it offline indefinitely. # # Applied as a second step rather than by registering task XML: /create is already known to work on # these nodes, and `schtasks /xml` wants a specific file encoding that differs between Windows # builds -- a failure mode that would silently leave the task unregistered rather than unhardened. _WIN_TASK_HARDEN_PS = ( "$ErrorActionPreference='Stop';" "$s = New-ScheduledTaskSettingsSet" " -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries" " -StartWhenAvailable -MultipleInstances IgnoreNew" " -RestartCount 99 -RestartInterval (New-TimeSpan -Minutes 1);" # Assigned directly as well. New-ScheduledTaskSettingsSet quietly drops a zero -ExecutionTimeLimit # on some builds and the 72-hour default survives, which is invisible until the agent dies on day # three. 'PT0S' is the literal the scheduler reads as "no limit". "$s.ExecutionTimeLimit = 'PT0S';" # Both triggers. Boot alone means a start that fails -- no network yet, battery, a transient -- # is never retried until the machine reboots again. AtLogOn gives it a second chance that a # person actually causes to happen, and MultipleInstances IgnoreNew makes the overlap a no-op. "$g = @((New-ScheduledTaskTrigger -AtStartup), (New-ScheduledTaskTrigger -AtLogOn));" # Restated so Set-ScheduledTask cannot narrow it: the task must run as SYSTEM at highest # privileges whether or not anyone is logged in. "$p = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest;" # Settings only while an instance is running. Changing a task's settings leaves its running # instance alone; re-registering triggers and principal is a heavier rewrite that can stop it, # and on a node where the task genuinely started the agent that would mean cutting the one # channel the machine is reachable through. The battery and time-limit settings are what keep # the agent alive, so they always apply -- only the parts that are safe to defer are deferred, # and the next clean start picks them up. "if ((Get-ScheduledTask -TaskName '%s').State -eq 'Running') {" " Set-ScheduledTask -TaskName '%s' -Settings $s | Out-Null; 'HARDENED-SETTINGS-ONLY' }" "else {" " Set-ScheduledTask -TaskName '%s' -Settings $s -Trigger $g -Principal $p | Out-Null;" " 'HARDENED' }" % (WIN_TASK_NAME, WIN_TASK_NAME, WIN_TASK_NAME) ) def _harden_windows_task(): """Applies the settings schtasks cannot set. {"ok", "detail"}; never raises. A failure here is reported rather than fatal: the task is registered either way, so the agent still comes back at boot on mains power. It just keeps the defects the caller was told about. """ ps = shutil.which("pwsh.exe") or shutil.which("powershell.exe") if not ps: return {"ok": False, "detail": "no PowerShell found, so battery/time-limit defaults remain"} out = _run_quiet([ps, "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", _WIN_TASK_HARDEN_PS], timeout=45) if out and "HARDENED-SETTINGS-ONLY" in out: # The settings that keep it alive are applied; the trigger/principal rewrite was skipped so # the running instance was not disturbed. return {"ok": True, "detail": "battery stop/start and time limit fixed; triggers left alone " "because the task instance is running"} if out and "HARDENED" in out: return {"ok": True, "detail": "battery stop/start, 72h time limit and restart-on-failure fixed"} # Set-ScheduledTask on a SYSTEM task needs elevation, which is the usual reason. return {"ok": False, "detail": "could not apply settings (needs elevation); " "battery/time-limit defaults remain"} def _win_task_state(): """The scheduled task's settings as a dict of strings, or None if it is not registered. Reads the properties that decide whether the agent survives, rather than only whether a task with the right name exists. """ ps = shutil.which("pwsh.exe") or shutil.which("powershell.exe") if not ps: return None probe = ( "$t = Get-ScheduledTask -TaskName '%s' -ErrorAction SilentlyContinue;" "if (-not $t) { 'exists=0' } else {" " $s = $t.Settings;" " 'exists=1';" " 'enabled=' + $s.Enabled;" " 'battNoStart=' + $s.DisallowStartIfOnBatteries;" " 'battStop=' + $s.StopIfGoingOnBatteries;" " 'timeLimit=' + $s.ExecutionTimeLimit;" " 'restartCount=' + $s.RestartCount;" " 'startWhenAvailable=' + $s.StartWhenAvailable;" " 'triggers=' + (($t.Triggers | ForEach-Object { $_.CimClass.CimClassName }) -join ',');" " 'runAs=' + $t.Principal.UserId }" % WIN_TASK_NAME ) out = _run_quiet([ps, "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", probe], timeout=30) if not out: return None kv = {} for line in out.splitlines(): if "=" in line: k, _, v = line.partition("=") kv[k.strip()] = v.strip() return kv if kv.get("exists") == "1" else (kv if "exists" in kv else None) def _win_task_issues(kv): """Reasons the task will not keep the agent running. Empty list means it will.""" issues = [] if (kv.get("enabled") or "").lower() != "true": issues.append("task disabled") trig = (kv.get("triggers") or "").lower() if "boottrigger" not in trig and "logontrigger" not in trig: issues.append("no boot or logon trigger") if (kv.get("battStop") or "").lower() == "true": # The one that matters most on a laptop: not "fails to start" but "is killed mid-run". issues.append("stops when unplugged") if (kv.get("battNoStart") or "").lower() == "true": issues.append("will not start on battery") tl = (kv.get("timeLimit") or "").strip().upper() if tl and tl not in ("PT0S", "PT0H", "0"): issues.append("killed after %s" % tl) try: if int(kv.get("restartCount") or "0") <= 0: issues.append("no restart after failure") except Exception: pass return issues def detect_persistence_cached(force=False): now = time.time() if not force and _persistence_cache["value"] is not None \ and (now - _persistence_cache["at"]) < PERSISTENCE_TTL: return _persistence_cache["value"] val = detect_persistence() _persistence_cache["at"] = now _persistence_cache["value"] = val return val def detect_persistence(): """{"managed","kind","enabledAtBoot","detail"} for this agent, best-effort and never raising. managed something other than a shell is responsible for starting this process enabledAtBoot the stronger property: it comes back WITHOUT anyone logging in """ system = platform.system() out = {"managed": False, "kind": "none", "enabledAtBoot": False, "detail": ""} # A supervisor the per-platform probes below cannot see -- a container entrypoint, a process # manager, an init this code does not speak -- declares itself here, because it is the only # thing that knows. Checked first: inside a container the Linux branch correctly finds no # systemd unit and then wrongly concludes nothing will restart the agent, which is how the # fleet banner came to claim a node that restarts on every machine start would not come back. # # DECLARED, not verified. Nothing below this line was measured; setting the variable is an # assertion by whoever wrote the supervisor, and the detail says so explicitly so it cannot be # read as a probe that succeeded. Setting it means precisely: something outside this process # starts it again without a human logging in. declared = (_cfg("MANAGED_BY", "") or "").strip()[:40] if declared: return {"managed": True, "kind": declared, "enabledAtBoot": True, "detail": "declared by the supervisor via SENTINEL_MANAGED_BY=%s -- asserted by " "configuration, not probed" % declared} try: if system == "Linux": active = (_run_quiet(["systemctl", "is-active", "sentinel-agent"]) or "").strip() enabled = (_run_quiet(["systemctl", "is-enabled", "sentinel-agent"]) or "").strip() # INVOCATION_ID is set by systemd for the processes it starts, so it distinguishes "a unit # exists" from "this process is the one the unit started" -- the confusion that bit # popos-ant, where a unit existed and the running agent was a detached user process. under_systemd = bool(os.environ.get("INVOCATION_ID")) out["kind"] = "systemd" if (active == "active" or under_systemd) else "none" out["managed"] = under_systemd out["enabledAtBoot"] = (enabled == "enabled") out["detail"] = "unit is-active=%s is-enabled=%s, this process %s systemd" % ( active or "?", enabled or "?", "under" if under_systemd else "NOT under") elif system == "Darwin": plist = os.path.join(os.path.expanduser("~"), MACOS_PLIST_REL) loaded = "com.applotus.sentinel-agent" in (_run_quiet(["launchctl", "list"]) or "") out["kind"] = "launchd" if (loaded or os.path.exists(plist)) else "none" out["managed"] = loaded # A LaunchAgent starts at LOGIN, not boot. Reported honestly rather than counted as # boot-persistence: a headless Mac that reboots with nobody logging in stays down. out["enabledAtBoot"] = False out["detail"] = ("LaunchAgent %s, %s -- starts at login, not at boot" % ("present" if os.path.exists(plist) else "absent", "loaded" if loaded else "not loaded")) elif system == "Windows": # Was `enabledAtBoot = exists`: a name lookup, treated as proof the agent comes back. # A task that is disabled, has no boot trigger, or is set to stop itself the moment the # machine switches to battery reported exactly like a healthy one -- which is how a node # with 8% real availability showed up as persistent, and why the admin repair sweep # skipped it (it skips anything already claiming managed && enabledAtBoot). kv = _win_task_state() if kv is None: # PowerShell missing or the probe failed. Fall back to the name check, and say that # the settings are unverified rather than implying they were checked. q = _run_quiet(["schtasks", "/query", "/tn", WIN_TASK_NAME, "/fo", "LIST"], timeout=12) exists = bool(q and WIN_TASK_NAME.lower() in q.lower()) out["kind"] = "scheduled-task" if exists else "none" out["managed"] = exists out["enabledAtBoot"] = exists out["detail"] = ("task %s %s (settings unverified)" % (WIN_TASK_NAME, "registered" if exists else "not registered")) elif kv.get("exists") != "1": out["kind"] = "none" out["managed"] = False out["enabledAtBoot"] = False out["detail"] = "task %s not registered" % WIN_TASK_NAME else: issues = _win_task_issues(kv) out["kind"] = "scheduled-task" # Registered either way -- something other than a shell owns the process. out["managed"] = True # The stronger claim, and only true when nothing in the configuration will stop it. out["enabledAtBoot"] = not issues if issues: # The specific reasons, because "not persistent" on a node whose task plainly # exists is the same dead end as the false positive this replaces. out["detail"] = ("task %s registered but will not stay up: %s" % (WIN_TASK_NAME, "; ".join(issues)))[:300] else: out["detail"] = ("task %s registered (boot+logon, %s, battery-safe, no time limit)" % (WIN_TASK_NAME, kv.get("runAs") or "SYSTEM"))[:300] else: out["detail"] = "unsupported platform %s" % system except Exception as e: out["detail"] = "detection failed: %s" % e return out def _stop_pid(pid): """SIGTERM on POSIX, taskkill on Windows. True if the signal was delivered.""" try: if platform.system() == "Windows": r = subprocess.run(["taskkill", "/PID", str(pid), "/F"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15) return r.returncode == 0 os.kill(pid, signal.SIGTERM) return True except Exception: return False def collapse_to_single_agent_on_startup(): """Called once, at the very top of run_agent(), before any heartbeat. A redeploy onto a host that already has an agent running -- most commonly: this node was put to sleep from the fleet page, and running the install/deploy command again is how a tenant brings it back without waiting on an admin -- must not leave two processes both answering for the same node, one slow and dormant, one fresh. Startup is the one moment a human's intent is unambiguous ("run the agent on this host now"), so this collapses to one agent immediately rather than waiting for the server to notice and push a dedupe_agents message. Reuses stop_duplicate_agents() rather than a bespoke kill loop specifically so the same protection applies here: a service-managed instance (systemd/launchd/schtasks) is never stopped, only a stray hand-started or nohup'd one -- which is what a dormant process almost always is, since there is no "install as a dormant service" path. Never raises; a failure here must not prevent the agent from starting. """ try: dup_check = detect_agent_instances(max_age=0.0) except Exception: dup_check = {"count": 1} if (dup_check.get("count") or 1) <= 1: return try: result = stop_duplicate_agents() except Exception as e: print(f"[*] Could not check for duplicate agents: {e}", flush=True) return if result.get("changed"): print(f"[*] {result.get('detail') or 'Stopped a duplicate agent process on this host.'}", flush=True) # A stale cached key from the process just stopped must not quietly carry over -- this run # authenticates with whatever it was actually started with (the deploy command always # carries its own --api-key/--tenant), not an old process's credential. try: os.remove(_provisioned_key_path()) print("[*] Cleared the previous run's cached node API key.", flush=True) except Exception: pass elif result.get("error"): print(f"[*] {result['error']}", flush=True) def stop_duplicate_agents(): """Collapses this machine to a single agent, keeping the one that is actually connected. The keeper is THIS process -- the one answering this request -- because it is provably the agent talking to the server. An earlier version kept the service-owned process instead and stopped the rest, which on a node whose registered SYSTEM task was running but not sustaining the connection removed the only working agent and took the node offline. A service-owned process is never stopped. It may be the only thing that returns after a reboot, and a service that is running without sustaining the connection is misconfigured rather than redundant -- killing it would remove both the evidence and the persistence. """ inst = detect_agent_instances(max_age=0.0) count = inst.get("count") or 1 me = os.getpid() if count <= 1: return {"success": True, "changed": False, "count": count, "kept": me, "stopped": [], "failed": [], "detail": "Only one agent is running on this node."} if not inst.get("ownershipKnown"): return {"success": False, "changed": False, "count": count, "stopped": [], "failed": [], "error": "Could not identify this agent in the process table, so there is no safe " "way to tell which process a service manager owns. Reboot the node " "instead -- that leaves only the registered agent."} others = inst.get("others") or [] managed = [o for o in others if o.get("serviceOwned") and o.get("pid")] strays = [o for o in others if not o.get("serviceOwned") and o.get("pid")] if not strays: # Everything else is service-owned. Stopping a registered service to resolve a duplicate is # how a node gets left with nothing that works, which is exactly what happened once. return { "success": False, "changed": False, "count": count, "kept": me, "stopped": [], "failed": [], "error": ("The other agent%s here (PID %s) %s service-owned, and this agent (PID %d) " "is the one actually connected to the server. Stopping a registered service " "could leave this node unreachable, so nothing was stopped. The service is " "running but is not sustaining the connection -- check the credentials it " "was registered with, then stop this process once the service works." % ("" if len(managed) == 1 else "s", ", ".join(str(o["pid"]) for o in managed), "is" if len(managed) == 1 else "are", me)), } stopped, failed = [], [] for o in strays: p = o["pid"] # Never this process, never a service-owned one, never an impossible pid. if p == me or p <= 1: continue if _stop_pid(p): stopped.append(p) else: failed.append(p) # Give SIGTERM a moment, then report what is actually gone rather than what was asked for. time.sleep(2.5) post = detect_agent_instances(max_age=0.0) still = set([post.get("selfPid")] + [o.get("pid") for o in (post.get("others") or [])]) still.discard(None) really_stopped = [p for p in stopped if p not in still] still_running = [p for p in stopped if p in still] + failed detail = "Kept PID %d (the agent connected to this server); stopped %s." % ( me, ", ".join(map(str, really_stopped)) if really_stopped else "nothing") if managed: detail += (" Left the service-owned agent (PID %s) alone so this node still comes back " "after a reboot." % ", ".join(str(o["pid"]) for o in managed)) if not inst.get("selfServiceOwned") and not managed: detail += " No agent here is service-owned, so the node is still not persistent." return {"success": not still_running, "changed": bool(really_stopped), "count": count, "kept": me, "stopped": really_stopped, "failed": still_running, "selfExiting": False, "detail": detail, "error": ("Could not stop PID(s) %s." % ", ".join(map(str, still_running))) if still_running else None} def ensure_persistence(): """Makes this agent survive a reboot. Idempotent; returns what it did and the resulting state. Never restarts an agent that is already managed: a fleet-wide button has to be safe to press twice, and interrupting a working session to re-apply settings that are already correct would be a worse outcome than the problem. """ before = detect_persistence() system = platform.system() steps = [] try: if before.get("managed") and before.get("enabledAtBoot"): return {"success": True, "changed": False, "before": before, "after": before, "steps": ["already persistent; nothing to do"]} exe, argv = _own_argv() script = os.path.abspath(argv[0]) args = " ".join('"%s"' % a if " " in a else a for a in argv[1:]) if system == "Linux": unit = ( "[Unit]\n" "Description=Applotus Sentinel Node Agent\n" "After=network-online.target\nWants=network-online.target\n\n" "[Service]\nType=simple\nUser=%s\n" "ExecStart=%s %s %s\n" "Restart=always\nRestartSec=10\n\n" "[Install]\nWantedBy=multi-user.target\n" % (_current_username(), exe, script, args) ) # Always written from this process's own argv. An existing unit is not trusted: a stale one # pointing at a different user or path is what silently moved an agent and broke a desktop. wrote = _sudo_write(LINUX_UNIT_PATH, unit) steps.append("wrote %s" % LINUX_UNIT_PATH if wrote else "could NOT write %s (needs root)" % LINUX_UNIT_PATH) if not wrote: return {"success": False, "changed": False, "before": before, "after": detect_persistence(), "steps": steps, "error": "cannot write the systemd unit; run the agent as root or install via the documented systemd snippet"} _run_quiet(["sudo", "-n", "systemctl", "daemon-reload"]) _run_quiet(["sudo", "-n", "systemctl", "enable", "sentinel-agent"]) steps.append("enabled sentinel-agent at boot") # Deliberately NOT restarting: this process keeps serving, and systemd takes over at the # next boot. Handing over now would drop every live session on this node for no gain. steps.append("left the running process in place; systemd takes over at next boot") elif system == "Darwin": plist_path = os.path.join(os.path.expanduser("~"), MACOS_PLIST_REL) prog = "".join(" %s\n" % a for a in [exe, script] + argv[1:]) plist = ( '\n' '\n' '\n' ' Labelcom.applotus.sentinel-agent\n' ' ProgramArguments\n%s \n' ' RunAtLoad\n KeepAlive\n' ' StandardOutPath%s/Library/Logs/applotus-sentinel.log\n' ' StandardErrorPath%s/Library/Logs/applotus-sentinel.log\n' '\n' % (prog, os.path.expanduser("~"), os.path.expanduser("~")) ) try: os.makedirs(os.path.dirname(plist_path), exist_ok=True) with open(plist_path, "w") as fh: fh.write(plist) steps.append("wrote %s" % plist_path) _run_quiet(["launchctl", "unload", plist_path]) _run_quiet(["launchctl", "load", "-w", plist_path]) steps.append("loaded the LaunchAgent (starts at login; a LaunchDaemon cannot do the " "GUI work the desktop path needs, so boot-scope is not used here)") except Exception as e: return {"success": False, "changed": False, "before": before, "after": detect_persistence(), "steps": steps, "error": str(e)} elif system == "Windows": tr = '"%s" "%s" %s' % (exe, script, args) # Mirrors install.bat: at-startup trigger, SYSTEM, highest privileges. /f replaces any # stale task rather than failing, which also collapses duplicate orphaned agents. r = _run_quiet(["schtasks", "/create", "/tn", WIN_TASK_NAME, "/sc", "onstart", "/ru", "SYSTEM", "/rl", "highest", "/tr", tr, "/f"], timeout=25) if r is None: return {"success": False, "changed": False, "before": before, "after": detect_persistence(), "steps": steps, "error": "schtasks /create failed; the agent must run elevated to register a SYSTEM task"} steps.append("registered scheduled task %s (onstart, SYSTEM)" % WIN_TASK_NAME) # /create takes the scheduler's defaults for everything it was not told, and those # defaults stop the agent on battery, kill it after 72 hours and never restart it. # Registering without this step produces a task that reports healthy and does not stay up. hard = _harden_windows_task() steps.append(("hardened the task: %s" if hard["ok"] else "WARNING: %s") % hard["detail"]) steps.append("left the running process in place; the task takes over at next boot") else: return {"success": False, "changed": False, "before": before, "after": before, "steps": steps, "error": "unsupported platform %s" % system} # force=True: the cached value is now wrong by construction, and the caller writes this # straight into the node record. after = detect_persistence_cached(force=True) # Registering a service while another agent is already running leaves TWO -- this function # deliberately does not stop the running process. That is how a real node ended up with a # launchd agent and a three-week-old terminal one both polling, with different PATHs. Say # so rather than reporting unqualified success. try: inst = detect_agent_instances(max_age=0.0) if inst.get("count", 1) > 1: pids = ", ".join(str(o.get("pid")) for o in inst.get("others") or []) steps.append("WARNING: %d agent processes are now running on this node (other PIDs: %s). " "The service is registered, but the pre-existing process is still " "polling; stop it or reboot so only the managed one remains." % (inst.get("count"), pids or "unknown")) after = dict(after or {}) after["detail"] = ((after.get("detail") or "") + " -- but %d agents are running on this node" % inst.get("count")).strip() except Exception: inst = None return {"success": True, "changed": True, "before": before, "after": after, "steps": steps, "agentInstances": inst} except Exception as e: return {"success": False, "changed": False, "before": before, "after": detect_persistence(), "steps": steps, "error": str(e)} def _sudo_write(path, content): """Writes a root-owned file, directly if possible and via sudo -n otherwise. True on success.""" try: with open(path, "w") as fh: fh.write(content) return True except Exception: pass try: p = subprocess.run(["sudo", "-n", "tee", path], input=content, text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15) return p.returncode == 0 except Exception: return False def get_listening_ports(): ports = set() try: out = subprocess.check_output(["ss", "-tulpn"], stderr=subprocess.DEVNULL).decode('utf-8', errors='ignore') for line in out.splitlines(): parts = line.split() if len(parts) >= 5 and ('LISTEN' in line or 'udp' in line): addr = parts[4] if ':' in addr: p = addr.split(':')[-1] if p.isdigit(): ports.add(int(p)) except Exception: pass if not ports: try: out = subprocess.check_output(["netstat", "-tuln"], stderr=subprocess.DEVNULL).decode('utf-8', errors='ignore') for line in out.splitlines(): parts = line.split() if len(parts) >= 4 and 'LISTEN' in line: addr = parts[3] if ':' in addr: p = addr.split(':')[-1] if p.isdigit(): ports.add(int(p)) except Exception: pass # macOS fallback: lsof -iTCP -sTCP:LISTEN -n -P if not ports and platform.system() == "Darwin": try: out = subprocess.check_output(["lsof", "-iTCP", "-sTCP:LISTEN", "-n", "-P"], stderr=subprocess.DEVNULL).decode('utf-8', errors='ignore') for line in out.splitlines(): parts = line.split() if len(parts) >= 9: addr = parts[8] if ':' in addr: p = addr.split(':')[-1] if p.isdigit(): ports.add(int(p)) except Exception: pass # Windows netstat fallback: netstat -ano -p tcp if not ports and platform.system() == "Windows": try: out = subprocess.check_output(["netstat", "-ano", "-p", "tcp"], stderr=subprocess.DEVNULL).decode('utf-8', errors='ignore') for line in out.splitlines(): if "LISTENING" in line.upper(): parts = line.split() if len(parts) >= 2: addr = parts[1] if ':' in addr: p = addr.split(':')[-1] if p.isdigit(): ports.add(int(p)) except Exception: pass return sorted(list(ports)) _last_cpu_idle = None _last_cpu_total = None def get_cpu_metric(): global _last_cpu_idle, _last_cpu_total cores = os.cpu_count() or 1 pct_str = None try: if platform.system() == "Windows": # 1. Try kernel32.GetSystemTimes delta via ctypes if ctypes and hasattr(ctypes, "windll") and hasattr(ctypes.windll, "kernel32"): try: class FILETIME(ctypes.Structure): _fields_ = [("dwLowDateTime", ctypes.c_uint), ("dwHighDateTime", ctypes.c_uint)] def _filetime_to_int(ft): return (ft.dwHighDateTime << 32) | ft.dwLowDateTime idle_ft = FILETIME() kernel_ft = FILETIME() user_ft = FILETIME() if ctypes.windll.kernel32.GetSystemTimes(ctypes.byref(idle_ft), ctypes.byref(kernel_ft), ctypes.byref(user_ft)): idle_t = _filetime_to_int(idle_ft) kernel_t = _filetime_to_int(kernel_ft) user_t = _filetime_to_int(user_ft) total_t = kernel_t + user_t if _last_cpu_idle is not None and _last_cpu_total is not None: idle_delta = idle_t - _last_cpu_idle total_delta = total_t - _last_cpu_total _last_cpu_idle = idle_t _last_cpu_total = total_t if total_delta > 0: pct = max(0.0, min(100.0, 100.0 * (1.0 - (idle_delta / total_delta)))) pct_str = f"{pct:.1f}%" else: _last_cpu_idle = idle_t _last_cpu_total = total_t except Exception: pass # 2. Fallback to wmic or PowerShell if delta not yet calculated if not pct_str: try: w_out = subprocess.check_output(["wmic", "cpu", "get", "loadpercentage"], stderr=subprocess.DEVNULL).decode().strip() lines = [l.strip() for l in w_out.splitlines() if l.strip() and l.strip().isdigit()] if lines: pct_str = f"{lines[0]}%" except Exception: pass elif os.path.exists("/proc/stat"): with open("/proc/stat", "r") as f: fields = [float(x) for x in f.readline().split()[1:8]] idle = fields[3] total = sum(fields) if _last_cpu_idle is not None and _last_cpu_total is not None: idle_delta = idle - _last_cpu_idle total_delta = total - _last_cpu_total _last_cpu_idle = idle _last_cpu_total = total if total_delta > 0: pct = max(0.0, min(100.0, 100.0 * (1.0 - (idle_delta / total_delta)))) pct_str = f"{pct:.1f}%" else: _last_cpu_idle = idle _last_cpu_total = total load = os.getloadavg()[0] pct = min(100.0, (load / cores) * 100.0) pct_str = f"{pct:.1f}%" elif hasattr(os, "getloadavg"): # macOS / BSD fallback via 1-minute load average normalized across cores load = os.getloadavg()[0] pct = min(100.0, (load / cores) * 100.0) pct_str = f"{pct:.1f}%" except Exception: pct_str = "N/A" return pct_str or "N/A" def get_system_metrics(): cpu_str = get_cpu_metric() metrics = { "cpu": cpu_str, "ram": "N/A", "disk": "N/A", "cores": os.cpu_count() or 1, "arch": platform.machine() or "x86_64" } # Raw numeric percentages alongside the formatted display strings above -- added for the # metrics-history feature, which needs numbers to bucket/average, not "42% (8 GB)" text. # Kept as extra fields rather than changing what metrics["cpu"] etc. already mean, since # existing code (client rendering, other agent logic) reads those as display strings today. try: metrics["cpuPercent"] = round(float(cpu_str.rstrip('%')), 1) if cpu_str and cpu_str != "N/A" else None except (ValueError, AttributeError): metrics["cpuPercent"] = None # RAM - Windows (GlobalMemoryStatusEx) if platform.system() == "Windows": try: if ctypes and hasattr(ctypes, "windll") and hasattr(ctypes.windll, "kernel32"): stat = MEMORYSTATUSEX() stat.dwLength = ctypes.sizeof(MEMORYSTATUSEX) if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)): total_bytes = stat.ullTotalPhys avail_bytes = stat.ullAvailPhys used_bytes = max(0, total_bytes - avail_bytes) used_pct = int(round((used_bytes / total_bytes) * 100)) if total_bytes > 0 else 0 total_gb = int(round(total_bytes / (1024**3))) metrics["ram"] = f"{used_pct}% ({total_gb} GB)" metrics["ramPercent"] = used_pct metrics["ramTotalMb"] = int(total_bytes / (1024 * 1024)) except Exception: pass # RAM - Linux (/proc/meminfo) elif os.path.exists("/proc/meminfo"): try: with open("/proc/meminfo", "r") as f: lines = f.readlines() total_kb = 0 avail_kb = 0 for line in lines: if line.startswith("MemTotal:"): total_kb = int(line.split()[1]) elif line.startswith("MemAvailable:"): avail_kb = int(line.split()[1]) if total_kb > 0: used_pct = int(round(100 - (avail_kb / total_kb * 100))) metrics["ram"] = f"{used_pct}% ({int(total_kb / 1024 / 1024 * 10) / 10} GB)" metrics["ramPercent"] = used_pct metrics["ramTotalMb"] = int(total_kb / 1024) except Exception: pass # RAM - macOS / Darwin (sysctl hw.memsize + vm_stat) elif platform.system() == "Darwin": try: total_bytes = int(subprocess.check_output(["sysctl", "-n", "hw.memsize"], stderr=subprocess.DEVNULL).decode().strip()) vm_out = subprocess.check_output(["vm_stat"], stderr=subprocess.DEVNULL).decode() pages = {} for line in vm_out.splitlines(): if ":" in line: k, v = line.split(":", 1) v = v.strip().rstrip(".") if v.isdigit(): pages[k.strip()] = int(v) page_size = 4096 free_pages = pages.get("Pages free", 0) speculative = pages.get("Pages speculative", 0) inactive = pages.get("Pages inactive", 0) avail_bytes = (free_pages + speculative + inactive) * page_size used_bytes = max(0, total_bytes - avail_bytes) used_pct = int(round((used_bytes / total_bytes) * 100)) if total_bytes > 0 else 0 total_gb = int(round(total_bytes / (1024**3))) metrics["ram"] = f"{used_pct}% ({total_gb} GB)" metrics["ramPercent"] = used_pct metrics["ramTotalMb"] = int(total_bytes / (1024 * 1024)) except Exception: pass # Disk - cross-platform shutil.disk_usage (works on Windows, Linux, macOS) try: disk_path = "C:\\" if platform.system() == "Windows" else "/" du = shutil.disk_usage(disk_path) total_bytes = du.total avail_bytes = du.free used_pct = int(round(100 - (avail_bytes / total_bytes * 100))) if total_bytes > 0 else 0 metrics["disk"] = f"{used_pct}% ({int(total_bytes / (1024**3))} GB)" metrics["diskPercent"] = used_pct except Exception: if hasattr(os, "statvfs"): try: stat = os.statvfs("/") total_bytes = stat.f_frsize * stat.f_blocks avail_bytes = stat.f_frsize * stat.f_bavail used_pct = int(round(100 - (avail_bytes / total_bytes * 100))) metrics["disk"] = f"{used_pct}% ({int(total_bytes / (1024**3))} GB)" metrics["diskPercent"] = used_pct except Exception: pass return metrics # PATH that a real interactive login shell on this box would have, resolved once and reused. # # Dispatched commands must see what this node's own terminal sees. On macOS that is frequently NOT # what any profile file provides: verified on a real Mac where Homebrew's bin directory came from # ~/.bashrc (reached via a .zshrc symlink), which only INTERACTIVE shells read, with no # `brew shellenv` line in any profile. `zsh -lc` could not find brew there; `zsh -lic` could. # # Snapshotted rather than run per command, because rc files print banners, run compinit and can be # slow, and none of that belongs in a command's stdout. _LOGIN_PATH_MARK_A = "__APL_PATH_BEGIN__" _LOGIN_PATH_MARK_B = "__APL_PATH_END__" _login_path_cache = {"resolved": False, "value": None} def _user_login_shell(): """This user's real login shell. $SHELL first, but it is frequently ABSENT under launchd and systemd -- which is exactly the situation this whole code path exists for -- so fall back to the password database, which is authoritative and needs no environment. Defaults to zsh on macOS (the system default since Catalina) rather than bash, which on a modern Mac is an ancient 3.2 kept only for scripts and reads a different profile file. """ sh = os.environ.get("SHELL") if not sh or not os.path.exists(sh): try: import pwd sh = pwd.getpwuid(os.getuid()).pw_shell except Exception: sh = None if not sh or not os.path.exists(sh): if platform.system() == "Darwin" and os.path.exists("/bin/zsh"): sh = "/bin/zsh" else: sh = "/bin/bash" if not os.path.exists(sh): sh = "/bin/sh" return sh def _login_shell_path(timeout=15): if _login_path_cache["resolved"]: return _login_path_cache["value"] _login_path_cache["resolved"] = True if platform.system() == "Windows": return None shell = _user_login_shell() # Interactive AND login, the combination a PTY session gets: profiles supply Homebrew's # shellenv on a standard install, rc files supply hand-rolled PATH edits like this node's. # Markers because an rc file may print a banner either side of the value. script = 'printf "%s%%s%s" "$PATH"' % (_LOGIN_PATH_MARK_A, _LOGIN_PATH_MARK_B) try: res = subprocess.run( ["-" + os.path.basename(shell), "-ic", script], executable=shell, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL, text=True, timeout=timeout, ) out = res.stdout or "" i = out.find(_LOGIN_PATH_MARK_A) j = out.find(_LOGIN_PATH_MARK_B) if i != -1 and j > i: found = out[i + len(_LOGIN_PATH_MARK_A):j].strip() # Only accept something that looks like a PATH and is richer than what we already # have. A shell that failed in an interesting way could print anything. if found and ":" in found and "/" in found: _login_path_cache["value"] = found print("[%s] [PATH] resolved interactive login PATH from %s (%d entries)" % (time.strftime("%X"), shell, len(found.split(":"))), flush=True) except Exception as e: # Best effort by design: a missing, hanging or hostile rc file must not stop commands # running, it just leaves them with the inherited PATH they had before. print("[%s] [PATH] could not resolve a login PATH (%s); using inherited PATH" % (time.strftime("%X"), e.__class__.__name__), flush=True) return _login_path_cache["value"] def _ps_rejected_cmd_operators(cmd, stderr): """True when PowerShell refused a command purely because it uses cmd.exe's && or ||. Windows PowerShell 5.1 -- still what `powershell.exe` resolves to on a current Windows -- did not gain && and || until PowerShell 7 and rejects them while parsing. Keyed on the FullyQualifiedErrorId, which is stable across locales, with the English message text only as a secondary signal. Also requires the command to actually contain one of the operators, so an unrelated syntax error is not quietly re-run under a different shell. """ if not stderr: return False if "&&" not in cmd and "||" not in cmd: return False return ("InvalidEndOfLine" in stderr) or ("not a valid statement separator" in stderr) def execute_command(cmd, timeout=30): # New session/process group so a timeout can kill the whole tree, not just the immediate # shell. proc.kill() alone only signals that one process -- shell=True's actual command # (and anything it spawns, like osascript keeping a graphical prompt open) is its child, not # it, and was verified to survive the "timeout" as an orphan holding a dialog open on-screen. is_windows = platform.system() == "Windows" popen_kwargs = {} if not is_windows: popen_kwargs["preexec_fn"] = os.setsid else: popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP # shell=True has no POSIX-shell equivalent on Windows: it always runs the command through # cmd.exe, which cannot resolve PowerShell cmdlets (Get-Process, Get-Service, ...) that the AI # assist routinely suggests once it sees a Windows node -- they fail with "not recognized as an # internal or external command" even though the cmdlet itself is valid. Route through # powershell.exe explicitly, the same way the interactive PTY session already does, and only # fall back to shell=True/cmd.exe if PowerShell truly is not on this box. # pwsh.exe FIRST. powershell.exe is Windows PowerShell 5.1 on a default install, which rejects # cmd.exe's && and || at parse time (they arrived in PowerShell 7). pwsh accepts both those and # the cmdlets this branch exists to support, so it strictly dominates 5.1 when it is there. ps_exe = (shutil.which("pwsh.exe") or shutil.which("powershell.exe")) if is_windows else None use_powershell = ps_exe is not None # Set for the POSIX login-shell branch below; stays None on Windows, where shell=True/cmd.exe # (or the PowerShell branch) is already the right thing. login_shell = None if use_powershell: popen_cmd = [ps_exe, "-NoProfile", "-NonInteractive", "-Command", cmd] elif not is_windows: # A LOGIN shell, by the same argv[0]-dash convention and for exactly the same reason as the # interactive PTY path above: Homebrew's installer writes `eval "$(brew shellenv)"` into # ~/.zprofile, which `/bin/sh -c` never reads. # # Without this, PATH here is whatever the agent process inherited. An agent started by # launchd or systemd gets the service manager's minimal PATH, which has no # /opt/homebrew/bin -- so a dispatched `brew install podman` failed with "command not # found" on a Mac where brew was installed and on PATH everywhere else, while the very # same command typed into that node's own Applotus terminal worked. Anything else a user # puts on PATH in their profile (nvm, pyenv, cargo, conda) was equally invisible. # # Worse than merely failing, it failed NON-DETERMINISTICALLY on a node running two agents # -- one launchd-started, one left over in a terminal -- because whichever instance won the # poll decided whether PATH had Homebrew on it. # # A leading dash rather than -l: sh and dash understand the former, not the latter. # Letting profile scripts run is safe here because this function has exactly one caller, # the user-dispatched shell task, and nothing internal parses its stdout. login_shell = _user_login_shell() popen_cmd = ["-" + os.path.basename(login_shell), "-c", cmd] popen_kwargs["executable"] = login_shell # The login shell alone covers profile-based setups. It does NOT cover a PATH set in an rc # file, which only interactive shells read -- so also hand over the snapshotted # interactive-login PATH. Cheap after the first call, and leaves the inherited environment # untouched if the probe could not resolve anything. _lp = _login_shell_path() if _lp: _env = dict(os.environ) _env["PATH"] = _lp popen_kwargs["env"] = _env else: popen_cmd = cmd try: proc = subprocess.Popen( popen_cmd, shell=not use_powershell and login_shell is None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, **popen_kwargs ) stdout, stderr = proc.communicate(timeout=timeout) # PowerShell 5.1 refused this purely for using cmd.exe's && / ||. Re-run under cmd.exe. # # Safe for one specific reason: PowerShell parses the whole -Command string before running # any of it, so a PARSE error guarantees that no statement executed and this retry cannot # double-execute anything. A blind retry on any failure would not be defensible. if (use_powershell and proc.returncode != 0 and _ps_rejected_cmd_operators(cmd, stderr)): print("[%s] [SHELL] PowerShell rejected cmd.exe operators; retrying via cmd.exe" % time.strftime("%X"), flush=True) try: proc2 = subprocess.Popen( ["cmd.exe", "/c", cmd], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, **popen_kwargs ) except Exception as e2: # cmd.exe unavailable is conceivable on a stripped image; report the ORIGINAL # PowerShell failure rather than this secondary one, which would be more confusing. print("[%s] [SHELL] cmd.exe retry could not start (%s)" % (time.strftime("%X"), e2.__class__.__name__), flush=True) return {"exit_code": proc.returncode, "stdout": stdout, "stderr": stderr} try: out2, err2 = proc2.communicate(timeout=timeout) except subprocess.TimeoutExpired: try: proc2.kill() except Exception: pass out2, err2 = proc2.communicate() return {"exit_code": None, "stdout": out2 or "", "stderr": ((err2 or "") + "\nTimed out after %ss (cmd.exe)" % timeout).strip()} return {"exit_code": proc2.returncode, "stdout": out2, "stderr": err2} return { "exit_code": proc.returncode, "stdout": stdout, "stderr": stderr } except subprocess.TimeoutExpired: try: if is_windows: # taskkill /T walks the whole tree; CTRL_BREAK_EVENT alone can be ignored by # console subprocesses the way SIGTERM can be on POSIX. subprocess.run(["taskkill", "/F", "/T", "/PID", str(proc.pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10) else: os.killpg(os.getpgid(proc.pid), signal.SIGKILL) except Exception: proc.kill() # last resort: at least the immediate process stops return { "exit_code": 124, "stdout": "", "stderr": f"Command timed out after {timeout} seconds" } except Exception as e: return { "exit_code": 1, "stdout": "", "stderr": str(e) } # Persistent HTTP(S) connections to the control plane, keyed by (host, port, https, insecure). # Every heartbeat, task result and mesh poll used to pay a full new TCP handshake plus a full TLS # handshake -- verified with a packet capture during this pass: two full round trips before a # single byte of the actual request went out, repeated every 2-10s forever. Reusing one # keep-alive connection per destination cuts that to the occasional reconnect (idle timeout, a # network blip, or the first call in a fresh process), which is the only case that should ever # pay handshake cost again. _conn_pool = {} _conn_pool_locks = {} _conn_pool_master_lock = threading.Lock() def _conn_lock_for(key): with _conn_pool_master_lock: lock = _conn_pool_locks.get(key) if lock is None: lock = threading.Lock() _conn_pool_locks[key] = lock return lock def _new_pooled_connection(host, port, use_https, insecure, timeout): if use_https: ctx = ssl_unverified_context if insecure else ssl.create_default_context() return http.client.HTTPSConnection(host, port, timeout=timeout, context=ctx) return http.client.HTTPConnection(host, port, timeout=timeout) def send_request(url, data=None, timeout=30, insecure=True, host_header=None, tenant_id=None, api_key=None): headers = { "Content-Type": "application/json", "User-Agent": f"Applotus-Sentinel-Agent/{VERSION}", "Referer": "https://applotus.fly.dev/", "Connection": "keep-alive", # The server already runs gzip/deflate compression middleware on every response, but only # activates it for a request that says it accepts encoded content -- http.client, unlike a # browser or requests.Session, never sends this on its own, so every response over this # pooled connection has been going out uncompressed the whole time regardless of size. "Accept-Encoding": "gzip", } if host_header: headers["Host"] = host_header if tenant_id: headers["x-tenant-id"] = tenant_id if api_key: headers["x-api-key"] = api_key body = json.dumps(data).encode('utf-8') if data else None # The server's express.json() (via body-parser) already transparently decompresses a # Content-Encoding: gzip/deflate/br request body -- zero server-side change needed. Skipped # below a small threshold where gzip's own header/footer overhead would exceed the savings on # an already-tiny payload. if body is not None and len(body) >= 500: body = gzip.compress(body, compresslevel=6) headers["Content-Encoding"] = "gzip" parsed = urllib.parse.urlsplit(url) use_https = parsed.scheme == "https" host = parsed.hostname port = parsed.port or (443 if use_https else 80) path = parsed.path or "/" if parsed.query: path += "?" + parsed.query key = (host, port, use_https, bool(insecure)) lock = _conn_lock_for(key) method = "POST" if body is not None else "GET" with lock: conn = _conn_pool.get(key) last_exc = None # Exactly one retry on a fresh connection: a keep-alive connection can go stale between # calls for reasons that have nothing to do with the request itself (the server's own # idle timeout, a brief network blip), and that failure mode is indistinguishable from # "first request in a new process" until it's actually tried. for attempt in range(2): if conn is None: conn = _new_pooled_connection(host, port, use_https, insecure, timeout) try: conn.request(method, path, body=body, headers=headers) resp = conn.getresponse() resp_body = _maybe_decompress(resp, resp.read()) if resp.status >= 400: err_msg = "" try: err_data = json.loads(resp_body.decode('utf-8')) err_msg = err_data.get("error", "") except Exception: pass raise RuntimeError(f"HTTP Error {resp.status}" + (f": {err_msg}" if err_msg else "")) _conn_pool[key] = conn return json.loads(resp_body.decode('utf-8')) except RuntimeError: _conn_pool[key] = conn # a real HTTP error response, not a dead connection raise except Exception as e: last_exc = e try: conn.close() except Exception: pass conn = None _conn_pool.pop(key, None) raise last_exc def _maybe_decompress(resp, raw_body): """http.client, unlike a browser, never auto-decompresses a gzip response even though it was the one that asked for it via Accept-Encoding -- that part is left entirely to the caller.""" encoding = (resp.getheader("Content-Encoding") or "").lower() if encoding == "gzip": try: return gzip.decompress(raw_body) except Exception: return raw_body # malformed/unexpected body; let the JSON parse surface the real error return raw_body # Whether this agent process is the one running *inside* a workload container. Set once at startup. # It decides how the container-port bridge reaches the workload: see _workload_connect_host. _AGENT_IN_CONTAINER = False def _own_container_ip(): """This container's address on the tenant bridge, or None. The container's hostname is mapped to its bridge address in /etc/hosts by the runtime, which is the cheapest reliable source from inside the namespace.""" try: ip = socket.gethostbyname(socket.gethostname()) if ip and not ip.startswith("127."): return ip except Exception: pass try: # Fallback: the address the kernel would use to reach off-box, read without sending anything. probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) probe.connect(("10.255.255.255", 1)) ip = probe.getsockname()[0] probe.close() if ip and not ip.startswith("127."): return ip except Exception: pass return None def _workload_connect_host(): """Where the in-container bridge should connect to reach the workload. Its own bridge address rather than 127.0.0.1, because the source address of this connection is what the workload sees -- and services authorise on it. MySQL keeps separate root@localhost and root@% accounts and matches the most specific, so a bridged connection arriving from 127.0.0.1 hit a different account than the one a sibling container on the bridge matches, and was denied with credentials that work directly. Same class of problem for pg_hba.conf and redis protected-mode. Connecting via the bridge address makes a meshed connection indistinguishable from a direct one. Host agents keep 127.0.0.1: there the target really is a loopback-published host port. """ if _AGENT_IN_CONTAINER: ip = _own_container_ip() if ip: return ip return "127.0.0.1" # ── Service mesh client (milestone 2: transparent cross-host names) ────────── # Milestone 1 proved a container can reach a service on another machine by opening the server's # container-port bridge explicitly. That is reachability, not transparency: `mysql -h mysql` still # failed, because nothing in the container answered to that name. # # This closes it. For every tenant service that this container cannot already resolve on its own # host bridge, bind a listener and point the name at it via /etc/hosts, then pump each accepted # connection over the bridge. An unmodified client connects to the real name on the real port and # never knows. # # Each alias gets its own loopback ADDRESS rather than its own port -- 127.0.0.2:3306 for mysql, # 127.0.0.3:3306 for a second mysql -- so the port the user was told to use is the port that works, # and two services wanting 3306 do not collide. All of 127.0.0.0/8 binds with no extra capability. MESH_REFRESH_INTERVAL = 30 MESH_HOSTS_BEGIN = "# BEGIN APPLOTUS MESH (managed - do not edit)" MESH_HOSTS_END = "# END APPLOTUS MESH" _mesh_bound = {} # (serviceKey, port) -> listening socket _mesh_ok_services = set() # services with at least one bound listener; only these get hosts entries _mesh_addr_for_service = {} # serviceKey -> 127.0.0.x, stable once assigned _mesh_names_for_service = {} # serviceKey -> [names it should answer to] _mesh_bind_failures = {} # label -> {"port", "error"} -- surfaced on this container's own card, since # "can't reach X" is otherwise invisible until someone tries -h X and hits # a plain DNS error with no explanation anywhere in the UI. _mesh_node_listeners = {} # ("node:", remotePort) -> what we bound, reported in the heartbeat so # the card can show the real URL: the local port is not always the # service's own port. _mesh_lock = threading.Lock() _mesh_started = False def _mesh_get_services(server_url, api_key, tenant_id=None, insecure=False, host_header=None): """Reads the tenant service registry. Aliases, ports and node ids only -- no credentials. Goes through send_request's connection pool: this polls every MESH_REFRESH_INTERVAL (30s) for as long as the container lives, and was paying a full fresh TCP+TLS handshake every single time before that pool existed. data=None makes it a GET, same as any other read here.""" url = server_url.rstrip("/") + "/api/sentinel/mesh/services" return send_request(url, data=None, timeout=20, insecure=insecure, host_header=host_header, tenant_id=tenant_id, api_key=api_key) def _mesh_resolves_locally(alias): """Whether the host bridge already answers for this name. Same-machine services must keep their direct path: routing them through the server would add a round trip for no benefit, and writing /etc/hosts for them would actively shadow working DNS.""" try: socket.getaddrinfo(alias, None) return True except Exception: return False def _mesh_assign_addr(service_key): """One loopback address per service, shared by every name that service answers to.""" with _mesh_lock: if service_key in _mesh_addr_for_service: return _mesh_addr_for_service[service_key] used = set(_mesh_addr_for_service.values()) for i in range(2, 255): cand = "127.0.0.%d" % i if cand not in used: _mesh_addr_for_service[service_key] = cand return cand return None def _mesh_write_hosts(): """Points each meshed alias at its loopback address, rewriting only our own marked block. Best-effort: a container whose /etc/hosts is not writable still has working listeners, so a caller can connect by address. Losing the name is a degradation, not a failure.""" with _mesh_lock: # Only services we are actually serving. An assigned address whose bind failed must never # reach /etc/hosts. entries = [] for key, addr in _mesh_addr_for_service.items(): if key not in _mesh_ok_services: continue for nm in _mesh_names_for_service.get(key, []): entries.append((nm, addr)) entries.sort() if not entries: return block = [MESH_HOSTS_BEGIN] + ["%s %s" % (addr, alias) for alias, addr in entries] + [MESH_HOSTS_END] try: try: with open("/etc/hosts") as fh: lines = fh.read().split("\n") except Exception: lines = [] out, skipping = [], False for ln in lines: if ln.strip() == MESH_HOSTS_BEGIN: skipping = True continue if ln.strip() == MESH_HOSTS_END: skipping = False continue if not skipping: out.append(ln) while out and not out[-1].strip(): out.pop() with open("/etc/hosts", "w") as fh: fh.write("\n".join(out + block) + "\n") return True except Exception as e: print(f"[{time.strftime('%X')}] [!] Could not update /etc/hosts for the mesh ({e}); " f"meshed services are reachable by address but not by name", flush=True) return False def _mesh_pump(client_sock, ws): """Bidirectional copy between an accepted local connection and the bridge.""" def ws_to_sock(): try: while True: opcode, payload = ws.read_frame() if opcode is None or opcode == 8: break if payload: client_sock.sendall(payload) except Exception: pass finally: try: client_sock.shutdown(socket.SHUT_RDWR) except Exception: pass t = threading.Thread(target=ws_to_sock, daemon=True) t.start() try: while True: data = client_sock.recv(65536) if not data: break # Binary frames: this is an arbitrary TCP stream, and text framing would corrupt any # byte that is not valid UTF-8 -- which a MySQL handshake is full of. ws.send_frame(data, opcode=2) except Exception: pass finally: try: ws.close() except Exception: pass try: client_sock.close() except Exception: pass def _mesh_ws_url(ctx, path, target_node_id, remote_port): """Relay URL for one bridge. `remote_port` is always the port at the FAR end -- for a node service the local listener may be on a different port, and sending the local one would ask the server to open a tunnel to a port that does not exist there.""" base = ctx["server_url"].rstrip("/").replace("https://", "wss://").replace("http://", "ws://") return base + path + "?nodeId=%s&port=%d" % (urllib.parse.quote(target_node_id), remote_port) def _mesh_open_bridge(client_sock, ws_url, ctx, label): """Opens the relay for one accepted connection and pumps it. Shared by both mesh halves.""" try: ws = SimpleWebSocketClient( ws_url, insecure=ctx.get("insecure", False), host_header=ctx.get("host_header"), headers=_ws_auth_headers(ctx.get("api_key"))) # The constructor only records the target; connect() performs the handshake. Without it every # send silently no-ops (send_frame returns early on a None socket), so the client saw an # immediate EOF -- "Lost connection reading initial communication packet" -- with nothing # logged anywhere. ws.connect(timeout=20) try: ws.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except Exception: pass except Exception as e: print(f"[{time.strftime('%X')}] [!] Mesh bridge to {label} failed: {e}", flush=True) try: client_sock.close() except Exception: pass return _mesh_pump(client_sock, ws) def _mesh_bind_local(preferred): """Binds a loopback listener for a node service, preferring the service's own port number. Preferring the same port is the point of the whole feature: a client left at its default -- ollama's http://localhost:11434 -- then reaches the remote service with no configuration at all. When that port is already taken locally this machine is running its own copy, and displacing it would be the worst outcome available: a local program would silently start talking to a different machine. So the bind is allowed to fail and an alternate is chosen instead. Offsets come first because they stay recognisable (21434 reads as "11434, elsewhere"), then an OS-assigned port so a listener always exists rather than the service being unreachable. SO_REUSEADDR is set for the usual reason -- so a restarted agent is not blocked by TIME_WAIT from its own previous connections. It does not allow stealing a live listener on either Linux or macOS; SO_REUSEPORT would, and is deliberately never set. """ candidates = [preferred] candidates += [preferred + off for off in (10000, 20000, 30000) if preferred + off <= 65535] candidates.append(0) # let the OS choose; guarantees a listener exists last = None for cand in candidates: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: s.bind(("127.0.0.1", cand)) s.listen(64) return s, s.getsockname()[1] except Exception as e: last = e try: s.close() except Exception: pass raise last if last else OSError("no local port available") def _mesh_serve_node(target_node_id, label, remote_port, local_port, ctx, srv): """Accept loop for one meshed node service. The socket arrives already bound, which is what lets the caller report the port it actually got before the first connection rather than after. """ where = "127.0.0.1:%d" % local_port note = "" if local_port == remote_port else " (%d was in use locally)" % remote_port print(f"[{time.strftime('%X')}] [\U0001F517 MESH LISTEN] {where} -> " f"{target_node_id}:{remote_port} [{label}]{note}", flush=True) while True: try: client, _ = srv.accept() except Exception: break def handle(c=client): _mesh_open_bridge(c, _mesh_ws_url(ctx, "/ws/node-port", target_node_id, remote_port), ctx, "%s (%s:%d)" % (label, target_node_id, remote_port)) threading.Thread(target=handle, daemon=True).start() def _mesh_refresh_nodes(ctx, reg): """Binds a local listener for each shared node service this node is allowed to reach. Add-only within a process, same as the container half: tearing a listener down under a live connection would drop it, and a service that has gone away already fails at connect time. The registry only lists anything at all when this node has been opted in, and /ws/node-port re-checks that opt-in, so an agent that ignored this would still be refused. """ if not reg.get("meshConsumer"): return my_node = ctx.get("node_id") for svc in reg.get("nodeServices") or []: target = svc.get("nodeId") if not target or target == my_node: continue try: remote_port = int(svc.get("preferredPort")) except (TypeError, ValueError): continue if not 1 <= remote_port <= 65535: continue key = ("node:" + target, remote_port) with _mesh_lock: if key in _mesh_bound: continue label = svc.get("label") or target try: srv, local_port = _mesh_bind_local(remote_port) except Exception as e: print(f"[{time.strftime('%X')}] [!] Mesh could not open any local port for " f"'{label}' ({target}:{remote_port}): {e}", flush=True) with _mesh_lock: _mesh_bind_failures[label] = {"port": remote_port, "error": str(e)} continue with _mesh_lock: _mesh_bound[key] = srv _mesh_node_listeners[key] = {"nodeId": target, "label": label, "remotePort": remote_port, "localPort": local_port} _mesh_bind_failures.pop(label, None) threading.Thread(target=_mesh_serve_node, args=(target, label, remote_port, local_port, ctx, srv), daemon=True).start() def _mesh_serve(service_key, label, addr, port, target_node_id, ctx, ready=None): """Accept loop for one meshed service. Signals `ready` with True only once bound.""" try: srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) srv.bind((addr, port)) srv.listen(64) except Exception as e: # The common cause is the workload in *this* container already listening on this port with a # wildcard bind, which owns the port on every local address including our alias addresses. # A container that itself serves port N cannot mesh another service on port N. print(f"[{time.strftime('%X')}] [!] Mesh could not bind {addr}:{port} for '{label}': {e}. " f"If something in this container already listens on {port} (0.0.0.0), that port cannot " f"be meshed here -- the name is left unresolved rather than pointed at the wrong " f"service.", flush=True) with _mesh_lock: _mesh_bind_failures[label] = {"port": port, "error": str(e)} if ready is not None: ready.set() return with _mesh_lock: _mesh_bound[(service_key, port)] = srv _mesh_ok_services.add(service_key) _mesh_bind_failures.pop(label, None) if ready is not None: ready.set() print(f"[{time.strftime('%X')}] [\U0001F517 MESH LISTEN] {label}:{port} -> {addr}:{port} " f"-> {target_node_id}", flush=True) while True: try: client, _ = srv.accept() except Exception: break def handle(c=client): _mesh_open_bridge(c, _mesh_ws_url(ctx, "/ws/container-port", target_node_id, port), ctx, "%s:%d" % (label, port)) threading.Thread(target=handle, daemon=True).start() def _mesh_refresh(ctx): """Bind anything newly reachable. Add-only within a process: tearing a listener down under a live connection would drop it, and a service that goes away already fails at connect time.""" try: reg = _mesh_get_services(ctx["server_url"], ctx.get("api_key"), ctx.get("tenant_id"), ctx.get("insecure", False), ctx.get("host_header")) except Exception as e: print(f"[{time.strftime('%X')}] [!] Mesh registry unreachable: {e}", flush=True) return # The node half first, because it is the only half a host agent runs. _mesh_refresh_nodes(ctx, reg) # The container half -- tenant service aliases on 127.0.0.x addresses, advertised through # /etc/hosts -- only applies inside a container, and is now explicitly gated rather than relying # on only containers ever reaching this code. Measured on macOS 27.0.1: 127.0.0.2 cannot be bound # at all (only 127.0.0.1 is on lo0 without a root-added alias) and /etc/hosts is not writable by # the agent's user, so running this half on a Mac host would produce nothing but bind failures. if not _AGENT_IN_CONTAINER: return my_node = ctx.get("node_id") my_host = reg.get("callerHostNodeId") changed = False for svc in reg.get("services", []): target = svc.get("containerNodeId") # Names the service answers to: its service alias if it has one, and its container name -- # which is what the Connect panel hands out when no alias is set. names = svc.get("names") or ([svc["alias"]] if svc.get("alias") else []) if not target or not names or target == my_node: continue # never mesh to ourselves if not svc.get("running"): continue label = names[0] # Same machine? Decide from the registry, not from DNS. Asking the resolver was a feedback # loop: our own /etc/hosts entry makes the alias resolve, so after the first successful run a # restarted agent saw "already resolves", skipped binding, and left the name pointing at an # address nothing was listening on. The registry knows which machine each service is on and # cannot be confused by our own edits. if my_host and svc.get("hostNodeId") == my_host: continue # host bridge already handles it; do not shadow working DNS if not my_host and _mesh_resolves_locally(label): # No host identity (not a container-scoped key): fall back to the resolver, accepting # that it can be fooled by a previous run's entry. continue addr = _mesh_assign_addr(target) if not addr: continue with _mesh_lock: _mesh_names_for_service[target] = list(names) for port in svc.get("ports") or []: with _mesh_lock: if (target, port) in _mesh_bound: continue ready = threading.Event() threading.Thread(target=_mesh_serve, args=(target, label, addr, int(port), target, ctx, ready), daemon=True).start() # Wait for the bind to succeed or fail before deciding whether to advertise the name. ready.wait(timeout=5) changed = True # Always reconcile, not only on change: an alias we no longer serve must lose its entry, or the # name keeps resolving to an address with nothing behind it. _mesh_write_hosts() def start_service_mesh(server_url, api_key, tenant_id, node_id, insecure=False, host_header=None): """Starts the mesh client, which now has two halves. Inside a container it resolves tenant service aliases to other machines, as before. On a host agent it runs only the consumer half -- local listeners for node services another workspace has shared with this one -- and only once that node has been opted in. """ global _mesh_started if _mesh_started: return # The registry read is authenticated. A legacy agent running on tenant id alone has nothing to # authenticate with, and without this guard it would log a registry failure every 30s forever on a # node that was never going to mesh anything anyway. if not api_key: return _mesh_started = True ctx = {"server_url": server_url, "api_key": api_key, "tenant_id": tenant_id, "node_id": node_id, "insecure": insecure, "host_header": host_header} def loop(): while True: try: _mesh_refresh(ctx) except Exception as e: print(f"[{time.strftime('%X')}] [!] Mesh refresh failed: {e}", flush=True) time.sleep(MESH_REFRESH_INTERVAL) threading.Thread(target=loop, daemon=True).start() print(f"[{time.strftime('%X')}] [\U0001F517 SERVICE MESH] Client started " f"(registry refresh every {MESH_REFRESH_INTERVAL}s)", flush=True) def run_agent(server_url, node_name, node_id, poll_interval=10, insecure=True, host_header=None, tenant_id=None, api_key=None, node_kind=None, parent_node_id=None, workload_container_id=None): global _latest_containers_info server_url = server_url.rstrip('/') print(f"[*] Applotus Sentinel Agent v{VERSION} starting...") print(f"[*] Target Server: {server_url}") print(f"[*] Node Name: {node_name} (ID: {node_id})") collapse_to_single_agent_on_startup() # Resolve the effective credential BEFORE the banner below, so it reports what will actually # be used rather than announcing tenant-header mode and only then silently switching to a key. # An explicit --api-key/env key still wins, so an operator can always override the cached one. if not api_key: cached_key = load_provisioned_api_key() if cached_key: api_key = cached_key print("[*] Using the node API key issued by the server on a previous run.") if api_key: print(f"[*] API Key Auth: {api_key[:10]}... (Account authenticated)") elif tenant_id: print(f"[*] Tenant Workspace: {tenant_id}") print(f"[*] Polling Interval: {poll_interval}s | Insecure SSL: {insecure}") print(f"[*] Virtual Host Header: {host_header or 'Default'}") # Initial Deep Discovery on Startup sudo_info = check_sudo() ai_info = check_local_ai() desktop_info = check_desktop_env() containers_info = check_containers() _latest_containers_info = containers_info cores_detected = os.cpu_count() or 1 arch_detected = platform.machine() or "x86_64" print(f"[*] Hardware: CPU {cores_detected} Core ({arch_detected})") print(f"[*] Security: Passwordless Sudo: {sudo_info.get('passwordless')} (User: {sudo_info.get('user')})") if ai_info.get("detected"): prov_summary = ", ".join([f"{p['name']} ({len(p['models'])} models)" for p in ai_info.get("providers", [])]) print(f"[*] AI Providers Detected: {prov_summary}") else: print("[*] Local AI: No active providers detected on standard ports (11434, 1234, 8000)") if desktop_info.get("available"): print(f"[*] Desktop Environment: {desktop_info.get('env')} (Display: {desktop_info.get('display')}, Type: {desktop_info.get('type').upper()}, Port: {desktop_info.get('port')})") else: print(f"[*] Desktop: No active VNC/GUI service ({desktop_info.get('env')})") # In-container agents join the service mesh so tenant service names resolve to services on # other machines. A host agent does not: its own processes are not what needs a tenant alias. global _AGENT_TENANT_ID, _AGENT_API_KEY _AGENT_TENANT_ID = tenant_id or "" _AGENT_API_KEY = api_key or "" if node_kind == "container": global _AGENT_IN_CONTAINER _AGENT_IN_CONTAINER = True # Started on every agent now, for the consumer half: a host that has been opted in binds local # ports for services another workspace shared with it. On a node that was never opted in this # costs one registry read every MESH_REFRESH_INTERVAL and binds nothing -- the server returns an # empty node list until the owner turns it on. start_service_mesh(server_url, api_key, tenant_id, node_id, insecure=insecure, host_header=host_header) if containers_info.get("available"): print(f"[*] Containers: {containers_info.get('runtime')} available (rootless: {containers_info.get('rootless')})") # Set the manager context first: the registry file is keyed by node id, so loading before # the id is known would read (and later overwrite) the wrong file. _set_manager_context({ "server_url": server_url, "insecure": insecure, "host_header": host_header, "tenant_id": tenant_id, "api_key": api_key, }, node_id) # Pick up containers a previous run of this agent provisioned, so a container marked # persistent keeps being watched across the agent restart that an auto-upgrade causes. _load_container_state() else: print(f"[*] Containers: No usable runtime detected ({containers_info.get('reason')})") print("[*] Press Ctrl+C to stop agent.\n") is_handshake = True last_ports = [] last_ai_summary = "" last_desktop_summary = "" last_containers_summary = "" # Set by a task worker thread when it changed something a re-probe would notice (installed # a package, toggled a runtime); cleared and acted on by the main loop only, since # desktop_info/sudo_info/ai_info/containers_info are shared, unlocked state that also feeds # every heartbeat -- see _execute_sync_task. _reprobe_needed = threading.Event() tick_count = 0 active_interval = poll_interval # adaptive: speeds up on tasks, backs off when idle idle_ticks = 0 # Background Persistent WebSocket Control Channel for Zero-Latency PTY & Desktop Launch def agent_control_worker(): ws_base = server_url.replace("https://", "wss://").replace("http://", "ws://") ctrl_url = f"{ws_base}/ws/agent-control?nodeId={urllib.parse.quote(node_id)}" # Capped exponential backoff rather than a flat retry: reconnecting every 5s against a # server that is genuinely down (a deploy, an outage) just adds load to whatever is # already struggling, for no benefit -- nothing answers any faster for trying more often. # Resets to the floor on every successful connect, so a brief blip still recovers fast. reconnect_delay = 2 while True: if _dormant_mode.is_set(): # Dormant: do not hold the control channel open at all -- an idle persistent # socket is exactly the per-connection cost a sleeping fleet is meant to avoid. # The main loop clears this flag the moment a heartbeat says to wake up, at which # point this reconnects on its own within a few seconds, same as any cold start. time.sleep(5) continue try: headers = _ws_auth_headers(api_key) ws = SimpleWebSocketClient(ctrl_url, insecure=insecure, host_header=host_header, headers=headers) ws.connect(timeout=8) # NOT resetting the backoff here, which is what made a permanent failure into a hot # loop. connect() only completes the WebSocket handshake; the server authorizes # afterwards and closes with 1008 if this credential is scoped to a different node. # Treating the handshake as success pinned the delay at its 2s floor forever -- one # misconfigured host produced a rejected connection every 2.2s for as long as it ran. authorized = False # The server pings every 25s (see the agent-control handler). Without a read # timeout here, a connection black-holed by a NAT or firewall with no FIN/RST # would leave this thread blocked in recv() forever -- reachable in theory, dead # in practice, and never reconnected. Comfortably above two missed pings so # ordinary latency jitter never trips it. ws.sock.settimeout(60) while True: opcode, payload = ws.read_frame() if opcode is not None and opcode != 8 and not authorized: # A frame arrived, so the server kept the socket: a rejection is delivered # as a close frame before anything else. Only now is this a real session, # and only now may the backoff return to its floor. authorized = True reconnect_delay = 2 print(f"[{time.strftime('%X')}] [⚡ CONTROL WS] Connected to server for instant command dispatch", flush=True) if opcode is None or opcode == 8: break if opcode == 9: ws.send_frame(payload, opcode=10) elif opcode in (1, 2) and payload: try: msg = json.loads(payload.decode("utf-8")) msg_type = msg.get("type") if not _capability_allowed("control", msg_type): # Answered in the shape the caller waits for where there is a # reqId; start_pty and start_desktop carry none and instead wait on # the agent dialling their session URL, so for those the log is the # only channel available. The dashboard hides both for a read-only # node so it does not normally come to that. print(f"[{time.strftime('%X')}] [⛔ REFUSED] control message " f"'{msg_type}' -- agent mode is '{AGENT_MODE}'", flush=True) _reply = CONTROL_REPLY_TYPES.get(msg_type) if _reply and msg.get("reqId"): try: ws.send_frame(json.dumps({ "type": _reply, "reqId": msg.get("reqId"), "success": False, "applied": False, "available": False, "readOnly": True, "modes": [], "current": None, "error": _capability_refusal(msg_type), "reason": _capability_refusal(msg_type), }).encode("utf-8")) except Exception: pass elif msg_type == "start_pty": s_id = msg.get("sessionId") w_url = msg.get("wsUrl") c_id = msg.get("containerId") if s_id and w_url: threading.Thread( target=run_pty_session, args=(s_id, w_url, insecure, host_header, api_key, c_id), daemon=True ).start() elif msg_type == "start_desktop": s_id = msg.get("sessionId") w_url = msg.get("wsUrl") d_port = msg.get("port") d_vnc_pw = msg.get("vncPassword", "") d_w = msg.get("desiredWidth") d_h = msg.get("desiredHeight") d_match = msg.get("matchResolution") d_display = msg.get("preferredDisplay") d_pref = msg.get("desiredFromPreference") if s_id and w_url: threading.Thread( target=run_desktop_session, args=(s_id, w_url, d_port, insecure, host_header, api_key, d_vnc_pw, d_w, d_h, d_match, d_display), kwargs={"desired_from_preference": d_pref}, daemon=True ).start() elif msg_type == "start_container_port": s_id = msg.get("sessionId") w_url = msg.get("wsUrl") c_port = msg.get("hostPort") if s_id and w_url and c_port: threading.Thread( target=run_container_port_session, args=(s_id, w_url, c_port, insecure, host_header, api_key), daemon=True ).start() elif msg_type == "enter_dormant": # Pushed when a tenant removes this machine from their fleet while # the agent is live on the control channel -- the immediate path. # The fallback (agent not connected here right now) is the same # instruction arriving inside a heartbeat response instead; both # set the same flag, so the main loop does not need to know which # one happened. _rid = msg.get("reqId") _dormant_mode.set() print(f"[{time.strftime('%X')}] [💤 DORMANT] Removed from fleet — " f"going quiet, checking in every {msg.get('intervalSec') or DORMANT_INTERVAL_SEC}s", flush=True) try: ws.send_frame(json.dumps({ "type": "enter_dormant_result", "reqId": _rid, "success": True, "nodeId": node_id }).encode("utf-8")) except Exception: pass # The control channel itself is the thing dormancy is meant to shed # -- an idle open socket costs the agent nothing to hold, but it is # also pointless for a host nobody can send commands to on purpose. # Close it; the reconnect loop below sees _dormant_mode set and does # not open another one until a heartbeat clears the flag. try: ws.close() except Exception: pass elif msg_type == "ensure_persistence": _rid = msg.get("reqId") try: _res = ensure_persistence() except Exception as _e: _res = {"success": False, "error": str(_e)} _res["type"] = "ensure_persistence_result" _res["reqId"] = _rid _res["nodeId"] = node_id try: ws.send_frame(json.dumps(_res).encode("utf-8")) except Exception: pass elif msg_type == "dedupe_agents": _rid = msg.get("reqId") try: _res = stop_duplicate_agents() except Exception as _e: _res = {"success": False, "error": str(_e)} _res["type"] = "dedupe_agents_result" _res["reqId"] = _rid _res["nodeId"] = node_id # No self-exit branch any more. The keeper is always THIS # process, because it is the one provably connected to the server, # so this action can no longer end with nothing connected. try: ws.send_frame(json.dumps(_res).encode("utf-8")) except Exception: pass elif msg_type == "upgrade_agent": d_url = msg.get("downloadUrl") or f"{server_url}/download/sentinel-agent.py" t_ver = msg.get("version", "latest") t_id = msg.get("taskId") or f"upgrade-{int(time.time())}" threading.Thread( target=upgrade_agent_self, kwargs={ "download_url": d_url, "target_version": t_ver, "server_url": server_url, "insecure": insecure, "host_header": host_header, "tenant_id": tenant_id, "api_key": api_key, "task_id": t_id, "node_id": node_id }, daemon=True ).start() elif msg_type == "restart_agent": print(f"[{time.strftime('%X')}] [⚡ RESTART] Restart requested via control channel", flush=True) def _ctrl_reexec(): time.sleep(0.8) _respawn_self() threading.Thread(target=_ctrl_reexec, daemon=True).start() elif msg_type == "list_files": req_id = msg.get("reqId") target_p = msg.get("path", "~") def _do_list(r_id, p): res = handle_agent_list_files(p) res["type"] = "list_files_response" res["reqId"] = r_id try: ws.send_frame(json.dumps(res).encode("utf-8")) except Exception: pass threading.Thread(target=_do_list, args=(req_id, target_p), daemon=True).start() elif msg_type == "get_display_modes": _rid = msg.get("reqId") # Only meaningful on Linux, where a host can run several X servers # and reporting the wrong one would be worse than reporting nothing. _disp = msg.get("display") def _do_getmodes(r_id, disp): try: res = handle_get_display_modes(disp) except Exception as _e: res = {"available": False, "readOnly": True, "modes": [], "current": None, "reason": "Agent error: %s: %s" % (type(_e).__name__, _e)} res["type"] = "get_display_modes_result" res["reqId"] = r_id try: ws.send_frame(json.dumps(res).encode("utf-8")) except Exception: pass # Threaded for the same reason as the write: the console-session read # waits on a scheduled task for up to 25s. threading.Thread(target=_do_getmodes, args=(_rid, _disp), daemon=True).start() elif msg_type == "set_display_mode": _rid = msg.get("reqId") _w, _h = msg.get("width"), msg.get("height") _cap = msg.get("maxPixels") _bw, _bh = msg.get("maxWidth"), msg.get("maxHeight") def _do_setmode(r_id, ww, hh, cap, bw, bh): # Wrapped because the first version was not: an exception in the # handler killed this daemon thread, no reply was ever sent, and # the only symptom anywhere was the caller's 45s timeout with no # cause. The process knew what went wrong and had no way to say # it. A failure is now an answer rather than a silence. try: res = handle_set_display_mode(ww, hh, cap, bw, bh) except Exception as _e: res = {"applied": False, "reason": "Agent error: %s: %s" % (type(_e).__name__, _e)} print("[%s] [!] set_display_mode failed: %r" % (time.strftime("%X"), _e), flush=True) res["type"] = "set_display_mode_result" res["reqId"] = r_id try: ws.send_frame(json.dumps(res).encode("utf-8")) except Exception: pass # Threaded: the console-session path waits on a scheduled task for up # to 20s, and blocking this loop would stall every other dispatch. threading.Thread(target=_do_setmode, args=(_rid, _w, _h, _cap, _bw, _bh), daemon=True).start() elif msg_type == "download_file": req_id = msg.get("reqId") target_p = msg.get("path", "") def _do_download(r_id, p): res = handle_agent_read_file(p) res["type"] = "download_file_response" res["reqId"] = r_id try: ws.send_frame(json.dumps(res).encode("utf-8")) except Exception: pass threading.Thread(target=_do_download, args=(req_id, target_p), daemon=True).start() # The three upload messages are handled INLINE, unlike every other # handler here, and that is deliberate. Each chunk appends at the # current file offset, so correctness depends on them being applied in # the order they arrived; one thread per chunk would let two writes # race and interleave, and the sequence check in the handler would then # tear down a perfectly good transfer. `end` is inline for the same # reason -- threaded, it could rename the partial out from under a # chunk still being written. The work itself is a buffered write of # about a megabyte, so the reader loop is held for well under the # heartbeat budget that made the other handlers threaded in the first # place. elif msg_type == "upload_file_begin": res = handle_agent_upload_begin(msg.get("path", "~"), msg.get("name", ""), int(msg.get("size") or 0), as_desktop_user=bool( msg.get("asDesktopUser"))) res["type"] = "upload_file_begin_response" res["reqId"] = msg.get("reqId") try: ws.send_frame(json.dumps(res).encode("utf-8")) except Exception: pass elif msg_type == "upload_file_chunk": res = handle_agent_upload_chunk(msg.get("uploadId"), msg.get("seq"), msg.get("dataB64")) res["type"] = "upload_file_chunk_response" res["reqId"] = msg.get("reqId") try: ws.send_frame(json.dumps(res).encode("utf-8")) except Exception: pass elif msg_type == "upload_file_end": res = handle_agent_upload_end(msg.get("uploadId"), msg.get("size")) res["type"] = "upload_file_end_response" res["reqId"] = msg.get("reqId") try: ws.send_frame(json.dumps(res).encode("utf-8")) except Exception: pass elif msg_type == "set_clipboard": # Threaded, unlike the uploads: this shells out to xclip, osascript # or a scheduled task in the console session, and the Windows path # alone can take 30s. Nothing about it is order-dependent. def _do_clip(r_id, m): res = handle_agent_set_clipboard( m.get("kind", "image"), data_b64=m.get("dataB64"), text=m.get("text"), mime=m.get("mime", "image/png")) res["type"] = "set_clipboard_response" res["reqId"] = r_id try: ws.send_frame(json.dumps(res).encode("utf-8")) except Exception: pass threading.Thread(target=_do_clip, args=(msg.get("reqId"), msg), daemon=True).start() except Exception: pass except Exception: pass time.sleep(reconnect_delay) reconnect_delay = min(reconnect_delay * 2, 30) def _execute_sync_task(task): """Runs one dispatched task to completion and reports its result -- the part of task handling that can take anywhere from milliseconds (a persistence toggle) to minutes (an image pull, a package install). Called from the thread pool below, never from the main loop directly: that loop's job is to keep the heartbeat/telemetry cadence alive and pick up newly dispatched work on schedule, not to sit blocked on whatever the slowest task in a batch happens to be doing. """ task_id = task.get("id") task_type = task.get("type", "shell") cmd = task.get("command") prompt = task.get("prompt") model_name = task.get("model") provider_name = task.get("provider", "Ollama") t_start = time.time() if not _capability_allowed("task", task_type): # Reported as a completed task with a non-zero code, not dropped. A refused task that # answers immediately is far easier to act on than one that hangs to its timeout, and # 126 is the shell's own "recognised but cannot execute". print(f"[{time.strftime('%X')}] [⛔ REFUSED] task type '{task_type}' ID: {task_id} " f"-- agent mode is '{AGENT_MODE}'", flush=True) result_payload = { "nodeId": node_id, "taskId": task_id, "exit_code": 126, "stdout": "", "stderr": _capability_refusal(task_type), "duration": round(time.time() - t_start, 2), "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "ai_prompt" and prompt: if not model_name: detected_models = [] for pr in ai_info.get("providers", []): for m in pr.get("models", []): detected_models.append((pr.get("name", "Ollama"), m)) if detected_models: provider_name, model_name = detected_models[0] else: provider_name, model_name = "Ollama", "tinyllama:latest" prompt_preview = prompt if len(prompt) <= 80 else f"{prompt[:77]}..." print(f"[{time.strftime('%X')}] [🤖 AI TASK] ID: {task_id} | Provider: {provider_name} | Model: {model_name}\n Prompt: \"{prompt_preview}\"", flush=True) ai_res = run_local_ai_inference(provider_name, model_name, prompt, timeout=120) duration = round(time.time() - t_start, 2) success = ai_res.get("success", False) status_str = "SUCCESS" if success else "FAILED" out_preview = (ai_res.get("output", "") or ai_res.get("error", "")).strip().replace("\n", " ") if len(out_preview) > 90: out_preview = f"{out_preview[:87]}..." print(f"[{time.strftime('%X')}] [✨ AI COMPLETED] ID: {task_id} | Status: {status_str} ({duration}s)\n Result: {out_preview}", flush=True) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "ai_result", "model": model_name, "prompt": prompt, "provider": provider_name, "exit_code": 0 if success else 1, "stdout": ai_res.get("output", ""), "stderr": ai_res.get("error", ""), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "provision_container": image = task.get("image") print(f"[{time.strftime('%X')}] [📦 PROVISION TASK] ID: {task_id} | Image: {image}", flush=True) prov_res = provision_container( containers_info, image, cpus=task.get("cpus", 0.5), memory_mb=task.get("memoryMb", 256), disk_mb=task.get("diskMb", 1024), ports=task.get("ports", []), ttl_seconds=task.get("ttlSeconds", 1800), node_id=node_id, report_ctx={ "server_url": server_url, "insecure": insecure, "host_header": host_header, "tenant_id": tenant_id, "api_key": api_key }, with_agent=bool(task.get("withAgent")), command=task.get("command"), network=task.get("network"), service_alias=task.get("serviceAlias"), env=task.get("env"), persistent=bool(task.get("persistent")), image_args=task.get("imageArgs"), agent_ctx={ "server_url": server_url, "insecure": insecure, "tenant_id": tenant_id, # Dedicated per-container key minted by the server; falls back to this # agent's own key only if none was supplied. "api_key": task.get("containerApiKey") or api_key, "sidecar_image": task.get("sidecarImage"), } ) duration = round(time.time() - t_start, 2) success = prov_res.get("success", False) print(f"[{time.strftime('%X')}] [{'✓' if success else '✗'} PROVISION {'COMPLETED' if success else 'FAILED'}] " f"ID: {task_id} ({duration}s) | {prov_res if not success else prov_res.get('containerId', '')[:12]}", flush=True) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "provision_container_result", "image": image, "exit_code": 0 if success else 1, "stdout": json.dumps(prov_res) if success else "", "stderr": "" if success else prov_res.get("error", "Unknown provisioning error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "remove_container": c_id = task.get("containerId") print(f"[{time.strftime('%X')}] [📦 REMOVE TASK] ID: {task_id} | Container: {(c_id or '')[:12]}", flush=True) rm_res = remove_container(c_id) if c_id else {"success": False, "error": "containerId required"} duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "remove_container_result", "containerId": c_id, "exit_code": 0 if rm_res.get("success") else 1, "stdout": json.dumps(rm_res) if rm_res.get("success") else "", "stderr": "" if rm_res.get("success") else rm_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "extend_container_ttl": c_id = task.get("containerId") add_s = task.get("addSeconds", 3600) print(f"[{time.strftime('%X')}] [📦 EXTEND TASK] ID: {task_id} | " f"Container: {(c_id or '')[:12]} | +{add_s}s", flush=True) ext_res = (extend_container_ttl(c_id, add_s) if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "extend_container_ttl_result", "containerId": c_id, "exit_code": 0 if ext_res.get("success") else 1, "stdout": json.dumps(ext_res) if ext_res.get("success") else "", "stderr": "" if ext_res.get("success") else ext_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "set_container_persistence": c_id = task.get("containerId") want = bool(task.get("persistent")) print(f"[{time.strftime('%X')}] [📦 PERSISTENCE TASK] ID: {task_id} | " f"Container: {(c_id or '')[:12]} | persistent={want}", flush=True) pers_res = (set_container_persistence(c_id, want, fallback_ttl_seconds=task.get("ttlSeconds", 1800)) if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "set_container_persistence_result", "containerId": c_id, "exit_code": 0 if pers_res.get("success") else 1, "stdout": json.dumps(pers_res) if pers_res.get("success") else "", "stderr": "" if pers_res.get("success") else pers_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "hibernate_container": c_id = task.get("containerId") print(f"[{time.strftime('%X')}] [💤 HIBERNATE TASK] ID: {task_id} | " f"Container: {(c_id or '')[:12]}", flush=True) hib_res = (hibernate_container(c_id) if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "hibernate_container_result", "containerId": c_id, "exit_code": 0 if hib_res.get("success") else 1, "stdout": json.dumps(hib_res) if hib_res.get("success") else "", "stderr": "" if hib_res.get("success") else hib_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "wake_container": c_id = task.get("containerId") print(f"[{time.strftime('%X')}] [⚡ WAKE TASK] ID: {task_id} | " f"Container: {(c_id or '')[:12]}", flush=True) wake_res = (wake_container(c_id) if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "wake_container_result", "containerId": c_id, "exit_code": 0 if wake_res.get("success") else 1, "stdout": json.dumps(wake_res) if wake_res.get("success") else "", "stderr": "" if wake_res.get("success") else wake_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "install_container_desktop": c_id = task.get("containerId") print(f"[{time.strftime('%X')}] [🖥️ CONTAINER DESKTOP TASK] ID: {task_id} | " f"Container: {(c_id or '')[:12]}", flush=True) desk_res = (install_container_desktop(c_id) if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "install_container_desktop_result", "containerId": c_id, "exit_code": 0 if desk_res.get("success") else 1, "stdout": json.dumps(desk_res) if desk_res.get("success") else "", "stderr": "" if desk_res.get("success") else desk_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "backup_container": c_id = task.get("containerId") print(f"[{time.strftime('%X')}] [📦 BACKUP TASK] ID: {task_id} | " f"Container: {(c_id or '')[:12]}", flush=True) bk_res = (backup_container(c_id, reason="manual") if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "backup_container_result", "containerId": c_id, "exit_code": 0 if bk_res.get("success") else 1, "stdout": json.dumps(bk_res) if bk_res.get("success") else "", "stderr": "" if bk_res.get("success") else bk_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "get_container_env": c_id = task.get("containerId") print(f"[{time.strftime('%X')}] [📦 GET CONTAINER ENV] ID: {task_id} | " f"Container: {(c_id or '')[:12]}", flush=True) env_res = (get_container_env(c_id) if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "get_container_env_result", "containerId": c_id, "exit_code": 0 if env_res.get("success") else 1, "stdout": json.dumps(env_res) if env_res.get("success") else "", "stderr": "" if env_res.get("success") else env_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "get_container_logs": c_id = task.get("containerId") print(f"[{time.strftime('%X')}] [📜 GET CONTAINER LOGS] ID: {task_id} | " f"Container: {(c_id or '')[:12]}", flush=True) logs_res = (get_container_logs(c_id, tail=task.get("tail", 200)) if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "get_container_logs_result", "containerId": c_id, "exit_code": 0 if logs_res.get("success") else 1, "stdout": json.dumps(logs_res) if logs_res.get("success") else "", "stderr": "" if logs_res.get("success") else logs_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "get_agent_logs": print(f"[{time.strftime('%X')}] [📜 GET AGENT LOGS] ID: {task_id}", flush=True) logs_res = get_agent_logs(tail=task.get("tail", 200)) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "get_agent_logs_result", "exit_code": 0 if logs_res.get("success") else 1, "stdout": json.dumps(logs_res) if logs_res.get("success") else "", "stderr": "" if logs_res.get("success") else logs_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "get_metrics_history": c_id = task.get("containerId") print(f"[{time.strftime('%X')}] [📈 GET METRICS HISTORY] ID: {task_id} | " f"{'host' if not c_id else 'Container: ' + c_id[:12]}", flush=True) hist_res = get_metrics_history(container_id=c_id) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "get_metrics_history_result", "containerId": c_id, "exit_code": 0 if hist_res.get("success") else 1, "stdout": json.dumps(hist_res) if hist_res.get("success") else "", "stderr": "" if hist_res.get("success") else hist_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "probe_lan_peer": targets = task.get("targets") or [] print(f"[{time.strftime('%X')}] [🔗 LAN PROBE] ID: {task_id} | {len(targets)} target(s)", flush=True) probe_res = probe_lan_peers(targets) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "probe_lan_peer_result", "exit_code": 0, "stdout": json.dumps(probe_res), "stderr": "", "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "update_container_env": c_id = task.get("containerId") new_env = task.get("env") or {} timeout_val = int(task.get("timeout", 120)) print(f"[{time.strftime('%X')}] [📦 UPDATE CONTAINER ENV] ID: {task_id} | " f"Container: {(c_id or '')[:12]} | {len(new_env)} keys", flush=True) up_res = (update_container_env(c_id, new_env, timeout=timeout_val) if c_id else {"success": False, "error": "containerId required"}) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "update_container_env_result", "containerId": up_res.get("containerId") or c_id, "exit_code": 0 if up_res.get("success") else 1, "stdout": json.dumps(up_res) if up_res.get("success") else "", "stderr": "" if up_res.get("success") else up_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif task_type == "cleanup_node": want_stop = bool(task.get("stopAgent")) print(f"[{time.strftime('%X')}] [\U0001F9F9 CLEANUP TASK] ID: {task_id} | " f"stopAgent={want_stop}", flush=True) try: clean_res = cleanup_node_resources(containers_info, node_id=node_id, tenant_id=tenant_id, stop_agent=want_stop) clean_ok = True except Exception as e: clean_res, clean_ok = {"error": str(e)}, False duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "cleanup_node_result", "exit_code": 0 if clean_ok else 1, "stdout": json.dumps(clean_res) if clean_ok else "", "stderr": "" if clean_ok else str(clean_res.get("error", "cleanup failed")), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } # Scheduled rather than immediate: the result POST below is the only record the # server gets of what was reclaimed, and exiting first would lose it. if clean_ok and want_stop: stop_agent_soon(delay=3.0) elif task_type == "start_container_runtime": # macOS/Windows only: the runtime is installed and its VM already exists, it # just isn't running. This is a ~10s start, not an install. print(f"[{time.strftime('%X')}] [📦 START RUNTIME TASK] ID: {task_id}", flush=True) rt = containers_info.get("runtime") rt_path = containers_info.get("path") if rt != "podman" or not rt_path: start_res = {"success": False, "error": "Starting a runtime backend is only supported for podman machine."} else: try: # Generous timeout: first boot after a host restart is slower. r = subprocess.run([rt_path, "machine", "start"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=300) if r.returncode == 0: start_res = {"success": True, "output": (r.stdout or "").strip()[:2000]} else: err = (r.stderr or r.stdout or "").strip() # Already-running is a success from the caller's point of view. if "already running" in err.lower(): start_res = {"success": True, "output": "already running"} else: start_res = {"success": False, "error": err[:2000] or f"exit {r.returncode}"} except subprocess.TimeoutExpired: start_res = {"success": False, "error": "podman machine start timed out after 300s"} except Exception as e: start_res = {"success": False, "error": str(e)} # A fresh read happens on the main loop thread only (see _reprobe_needed) -- this # function may run on a worker thread, and containers_info/last_containers_summary # are shared with every other task and with heartbeat composition. _reprobe_needed.set() duration = round(time.time() - t_start, 2) ok = start_res.get("success", False) print(f"[{time.strftime('%X')}] [{'✓' if ok else '✗'} START RUNTIME " f"{'COMPLETED' if ok else 'FAILED'}] ID: {task_id} ({duration}s)", flush=True) result_payload = { "nodeId": node_id, "taskId": task_id, "type": "start_container_runtime_result", "exit_code": 0 if ok else 1, "stdout": json.dumps(start_res) if ok else "", "stderr": "" if ok else start_res.get("error", "Unknown error"), "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } elif not cmd: # Reaches here when task_type matched none of the specific elif branches above AND # there is no shell command either -- either a genuinely malformed task, or the real # case this guards against: a server newer than this agent dispatching a task type # this build predates (exactly what happened testing get_agent_logs/ # get_metrics_history against an agent still on the previous version). Previously this # fell into the plain shell branch below with cmd=None, crashing execute_command with # "'NoneType' object is not iterable" instead of reporting a clean error. print(f"[{time.strftime('%X')}] [!] UNRECOGNIZED TASK TYPE '{task_type}' ID: {task_id} " f"-- this agent build may be out of date.", flush=True) duration = round(time.time() - t_start, 2) result_payload = { "nodeId": node_id, "taskId": task_id, "exit_code": 1, "stdout": "", "stderr": f"Unrecognized task type '{task_type}'. This agent may need to be upgraded to a newer version.", "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } else: cmd_timeout = int(task.get("timeout") or 180) print(f"[{time.strftime('%X')}] [💻 SHELL TASK] ID: {task_id} (timeout: {cmd_timeout}s) | Command: \"{cmd}\"", flush=True) result = execute_command(cmd, timeout=cmd_timeout) duration = round(time.time() - t_start, 2) out_preview = (result["stdout"] or result["stderr"]).strip().replace("\n", " ") if len(out_preview) > 90: out_preview = f"{out_preview[:87]}..." print(f"[{time.strftime('%X')}] [✓ SHELL COMPLETED] ID: {task_id} | Exit: {result['exit_code']} ({duration}s)\n Output: {out_preview or '(empty)'}", flush=True) result_payload = { "nodeId": node_id, "taskId": task_id, "command": cmd, "exit_code": result["exit_code"], "stdout": result["stdout"], "stderr": result["stderr"], "duration": duration, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id } # Send result back with retry reported = False for attempt in range(3): try: send_request( f"{server_url}/api/sentinel/agent/result", data=result_payload, insecure=insecure, host_header=host_header, tenant_id=tenant_id, api_key=api_key ) reported = True print(f"[{time.strftime('%X')}] [*] Task {task_id} reported back to server.", flush=True) break except Exception as post_err: time.sleep(1.5) if not reported: print(f"[{time.strftime('%X')}] [-] Failed to report task {task_id} to server after 3 attempts.", flush=True) # Any task except a pure AI prompt might have changed installed packages/services (a # container runtime, a desktop stack, sudo config). The actual re-probe runs on the main # loop thread only -- see _reprobe_needed -- since this function may run concurrently with # other tasks on worker threads, and desktop_info/sudo_info/ai_info/containers_info are # shared, unlocked state read by heartbeat composition every tick. if task_type != "ai_prompt": _reprobe_needed.set() def _run_task_safely(task): """Thin safety net around _execute_sync_task for whatever that function's own defensive code does not already catch. Runs on a task-pool thread, so an uncaught exception here would otherwise just print a traceback nobody watches and silently drop the task -- the server would wait forever for a result that is never coming.""" try: _execute_sync_task(task) except Exception as e: task_id = task.get("id") print(f"[{time.strftime('%X')}] [!] Unhandled error executing task {task_id}: {e}", flush=True) try: send_request( f"{server_url}/api/sentinel/agent/result", data={ "nodeId": node_id, "taskId": task_id, "type": task.get("type", "shell"), "exit_code": 1, "stdout": "", "stderr": f"Agent-side error: {e}", "duration": 0, "completedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "tenantId": tenant_id, }, insecure=insecure, host_header=host_header, tenant_id=tenant_id, api_key=api_key, ) except Exception: pass def _run_task_serialized(container_id, task): with _container_task_lock(container_id): _run_task_safely(task) # Bounded rather than one thread per task: a burst of dispatched work (many containers healed # or upgraded at once) should not spawn an unbounded number of threads on a small host. I/O-bound # (subprocess calls, HTTP requests) so a modest worker count goes a long way despite the GIL. _task_executor = ThreadPoolExecutor(max_workers=6, thread_name_prefix="task-worker") # Coarse liveness check on the main loop itself, distinct from everything above: this is what # notices if the loop's *own* work -- not a dispatched task, which now runs off to the side -- # somehow gets stuck (a DNS resolution that never returns, some call that ignores the timeout # passed to it). Updated at the top of every tick; a long silence past what any single tick # should ever take is a real hang, not routine slowness. _last_tick_started_at = time.time() def _agent_watchdog(): while True: time.sleep(30) stalled_for = time.time() - _last_tick_started_at threshold = max(90, active_interval * 6) if stalled_for > threshold: print(f"[{time.strftime('%X')}] [⚠️ WATCHDOG] Main loop has not started a new tick in " f"{int(stalled_for)}s (expected within ~{threshold}s) -- something in this tick is stuck.", flush=True) # Scaled to active_interval, not a flat 600s: a dormant node's own loop deliberately # does not tick for its whole interval (30 min by default, longer with an admin # override), and a flat 600s respawned it like clockwork every ~10 minutes -- # restarting the process, which clears _dormant_mode and sends a fresh handshake, # silently un-dormanting every node whose sleep interval exceeds 10 minutes. For any # interval of 50s or less this is still exactly 600s (unchanged from before); for a # longer one -- dormant chief among them -- it grows with it instead of firing early. hard_threshold = max(600, active_interval * 12) if stalled_for > hard_threshold: print(f"[{time.strftime('%X')}] [⚠️ WATCHDOG] Stuck for over {int(hard_threshold)}s with no " f"sign of recovering on its own -- restarting the agent process.", flush=True) _respawn_self() threading.Thread(target=_agent_watchdog, daemon=True).start() threading.Thread(target=agent_control_worker, daemon=True).start() _start_metrics_collector() # 5s, not the 2s this used to be. Fast mode is a *fallback* path for task pickup: the control # channel (agent_control_worker, started above) is what delivers dispatched work in real time # over the persistent WebSocket, so shortening the heartbeat poll buys very little latency while # multiplying server request volume by 5x per node for as long as tasks keep arriving. On a # single shared vCPU that amplification was a meaningful share of the load during the exact # bursts when the server had least headroom. Raise this back only if the control channel stops # being the primary dispatch path. FAST_INTERVAL = 5 IDLE_TICKS_TO_BACKOFF = 2 # ~10s in fast mode with no new tasks => back to normal # Fallback only: the server's own value (sent in every dormant heartbeat's response) wins # whenever present, so a platform-wide change to the cadence takes effect without an agent # upgrade. This just covers an agent talking to a server old enough not to send one. DORMANT_INTERVAL_SEC = int(_cfg("DORMANT_INTERVAL_SEC", "1800")) # 30 min while True: _last_tick_started_at = time.time() tasks = [] server_interval = None dormant_hint = False wake_hint = False try: tick_count += 1 # Read whatever the metrics collector thread last published rather than collecting # inline -- see _metrics_collector_tick. metrics/c_stats are only None if the very # first collector tick (seeded synchronously before this loop starts) somehow never # ran; falling back to a fresh get_system_metrics() call keeps that heartbeat honest # instead of sending an empty metrics block. snapshot = get_latest_metrics_snapshot() metrics = snapshot["metrics"] if snapshot["metrics"] is not None else get_system_metrics() c_stats = snapshot["containerStats"] # Deep discovery runs on startup, and checks periodically every 30 ticks (~5 mins) if tick_count % 30 == 0: ai_info = check_local_ai() sudo_info = check_sudo() desktop_info = check_desktop_env() containers_info = check_containers() _latest_containers_info = containers_info # A task worker finished something that might have changed one of the above (installed # a package, toggled a runtime) -- re-probe now rather than waiting out the rest of the # 30-tick cycle. Done here, and only here, so nothing else is writing these same # variables while a heartbeat a few lines down reads them. if _reprobe_needed.is_set(): _reprobe_needed.clear() try: desktop_info = check_desktop_env() sudo_info = check_sudo() ai_info = check_local_ai() containers_info = check_containers() _latest_containers_info = containers_info last_desktop_summary = "" last_containers_summary = "" except Exception: pass # Port check runs on handshake or every 3 ticks (~30s) if is_handshake or tick_count % 3 == 0: current_ports = sorted(list(get_listening_ports())) ports_changed = (current_ports != last_ports) last_ports = current_ports else: current_ports = None ports_changed = False # Compose Optimized Heartbeat heartbeat_payload = { "nodeId": node_id, "name": node_name, "version": VERSION, # Every beat, not just the handshake: the dashboard uses this to decide whether to # offer Terminal and Desktop at all, and a node flipped back to full mode should # recover on the next beat rather than needing to re-handshake. Absent means full, # which is what every agent before 1.72.0 sends. "mode": AGENT_MODE, # Whether the file browser is confined. Reported separately from the mode because # the two are orthogonal -- a full-mode node can be jailed too. "fileRoot": bool(FILE_ROOT), "metrics": metrics, "cores": os.cpu_count() or 1, "arch": platform.machine() or "x86_64" } if is_handshake: heartbeat_payload["handshake"] = True heartbeat_payload["version"] = VERSION heartbeat_payload["host"] = get_public_ip() # Separate from host on purpose: host is -- or on a bad day, falls back to -- # whatever get_public_ip() could determine, and nothing downstream can tell which # happened. This is always this machine's own LAN address (or absent, never a # guess), for an operator who has direct network access and needs it instead. heartbeat_payload["localIp"] = get_local_ip() # Hardware inventory, independent of whether utilization is readable: a node with a # GPU that cannot be sampled should still show that it has one. Indented to sit INSIDE # the handshake block alongside os/arch/cores: at a shallower indent it closed that # block early and turned every following line into the body of this except, so a # successful detect_gpus() meant the node never reported its OS, ports or sudo state # at all. try: _gpus = detect_gpus() if _gpus: heartbeat_payload["gpus"] = _gpus except Exception: pass heartbeat_payload["os"] = f"{platform.system()} {platform.release()}" heartbeat_payload["arch"] = platform.machine() or "x86_64" heartbeat_payload["cores"] = os.cpu_count() or 1 heartbeat_payload["openPorts"] = last_ports heartbeat_payload["sudo"] = sudo_info # Also sent on every heartbeat below; kept here so the very first handshake carries # it rather than waiting for the next tick. try: heartbeat_payload["persistence"] = detect_persistence_cached() except Exception: pass try: heartbeat_payload["agentInstances"] = detect_agent_instances() except Exception: pass heartbeat_payload["ai"] = ai_info heartbeat_payload["desktop"] = desktop_info heartbeat_payload["containers"] = containers_info heartbeat_payload["tenantId"] = tenant_id # Sidecar identity: lets the server file this agent under its host node instead # of listing it as another standalone host in the fleet. if node_kind: heartbeat_payload["kind"] = node_kind if parent_node_id: heartbeat_payload["parentNodeId"] = parent_node_id if workload_container_id: heartbeat_payload["containerId"] = workload_container_id else: # Delta heartbeat: only include ports, AI, desktop, or containers if changed if ports_changed: heartbeat_payload["openPorts"] = last_ports current_ai_summary = json.dumps(ai_info) if current_ai_summary != last_ai_summary: heartbeat_payload["ai"] = ai_info last_ai_summary = current_ai_summary current_desktop_summary = json.dumps(desktop_info) if current_desktop_summary != last_desktop_summary: heartbeat_payload["desktop"] = desktop_info last_desktop_summary = current_desktop_summary current_containers_summary = json.dumps(containers_info) if current_containers_summary != last_containers_summary: heartbeat_payload["containers"] = containers_info last_containers_summary = current_containers_summary # Live state for containers we provisioned, as last published by the metrics collector # thread (see _metrics_collector_tick, which also records the metrics-history sample -- # not repeated here). Sent as its own key (not folded into the delta-compared # `containers` block) because these values change constantly and must not suppress # capability deltas. if c_stats is not None: heartbeat_payload["containerStats"] = c_stats # Network egress health per container, from the manager loop's own periodic probes # (see _container_manager_tick). Reported by the parent, same reason containerStats # is: a container whose control-plane path is broken cannot self-report anything, so # the only place this can come from is the agent that can still see it from outside. if active_containers: net_health = collect_container_network_health() if net_health: heartbeat_payload["containerNetwork"] = net_health # Boot-readiness issues per container. Always sent (even empty) once there is anything # tracked at all -- omitting the key when empty would mean the server never learns the # last lingering issue on this host cleared, and leaves that stale note on the card # forever with nothing left to ever correct it. if active_containers: heartbeat_payload["containerBoot"] = collect_container_boot_health() # Mesh bind failures for *this* container (only meaningful when it is itself a mesh # client, i.e. node_kind == "container"): a sibling's port collided with something this # container already listens on, so its name never resolved here. Without this it just # looks like a plain DNS error to whoever tries -h , with no clue why. Same # always-send reasoning as containerBoot above. # Bind failures were previously reported only from inside a container. A host agent can # now have them too, and a failure the UI never sees is exactly the invisible case this # field was added for. with _mesh_lock: if _AGENT_IN_CONTAINER or _mesh_bind_failures: heartbeat_payload["meshIssues"] = [ {"label": k, "port": v["port"], "error": v["error"]} for k, v in _mesh_bind_failures.items()] # What the consumer half actually bound. Reported rather than inferred: the local port # is not always the service's own, so only this agent knows the real URL. if _mesh_node_listeners: heartbeat_payload["meshListeners"] = list(_mesh_node_listeners.values()) # On every heartbeat, not just the handshake. ensure_persistence deliberately does not # restart a healthy agent, so without this a successful fix was never reported and the UI # kept warning about nodes that had already been fixed. Cached, so this is a dict lookup # on all but one tick in thirty. try: heartbeat_payload["persistence"] = detect_persistence_cached() except Exception: pass # Cheap after the first call (cached for two minutes), and a node can acquire a second # agent at any time -- most often right after being made persistent. try: heartbeat_payload["agentInstances"] = detect_agent_instances() except Exception: pass # 1. Send Heartbeat (Server responds with piggybacked pending tasks) hb_resp = send_request( f"{server_url}/api/sentinel/agent/heartbeat", data=heartbeat_payload, insecure=insecure, host_header=host_header, tenant_id=tenant_id, api_key=api_key ) if hb_resp.get("tenantId") and not tenant_id: tenant_id = hb_resp.get("tenantId") # The server issues a node-scoped key to any agent still authenticating with a bare # workspace id, which is no longer accepted for anything but this heartbeat. Adopt it # immediately (every later request, HTTP and WS alike, then carries a real credential) # and cache it so a restart does not regress to header auth. issued = hb_resp.get("provisionedApiKey") if issued and issued != api_key: api_key = issued globals()["_AGENT_API_KEY"] = issued if save_provisioned_api_key(issued): print(f"[{time.strftime('%X')}] [*] Received and stored a node API key from the " f"server; future requests will authenticate with it.", flush=True) if is_handshake: print(f"[{time.strftime('%X')}] Registered with server (Registration Handshake complete, Agent v{VERSION})", flush=True) is_handshake = False else: cpu_val = metrics.get('cpu', 'N/A') ram_val = metrics.get('ram', 'N/A') cores_cnt = metrics.get('cores') or os.cpu_count() or 1 arch_val = metrics.get('arch') or platform.machine() or "x86_64" print(f"[{time.strftime('%X')}] Pulse synced (CPU: {cpu_val} ({cores_cnt} Core, {arch_val}), RAM: {ram_val}, interval: {active_interval}s)", flush=True) # Check if server requested full re-synchronization if hb_resp.get("sync_required"): is_handshake = True # Honor server-driven heartbeat interval hint server_interval = hb_resp.get("heartbeat_interval") # Dormancy, the other direction: the server slowing THIS agent down (or waking it back # up), rather than speeding it up for a task. Handled in the Adaptive Heartbeat Rate # section below, since that is the one place active_interval is decided for the tick. dormant_hint = bool(hb_resp.get("dormant")) wake_hint = bool(hb_resp.get("wake")) # 2. Process Piggybacked Tasks (or fallback to tasks array if present) tasks = hb_resp.get("tasks", []) for task in tasks: task_id = task.get("id") task_type = task.get("type", "shell") # "shell", "ai_prompt", "start_pty", "start_desktop", "upgrade_agent", "provision_container", "remove_container", or "start_container_runtime" if task_type == "start_pty": session_id = task.get("sessionId") ws_url = task.get("wsUrl") container_id = task.get("containerId") if session_id and ws_url: print(f"[{time.strftime('%X')}] [💻 PTY TASK] ID: {task_id} | Session: {session_id}", flush=True) threading.Thread( target=run_pty_session, args=(session_id, ws_url, insecure, host_header, api_key, container_id), daemon=True ).start() continue if task_type == "start_desktop": session_id = task.get("sessionId") ws_url = task.get("wsUrl") d_port = task.get("port") d_vnc_pw = task.get("vncPassword", "") d_w = task.get("desiredWidth") d_h = task.get("desiredHeight") d_match = task.get("matchResolution") d_pref = task.get("desiredFromPreference") if session_id and ws_url: print(f"[{time.strftime('%X')}] [🖥️ DESKTOP TASK] ID: {task_id} | Session: {session_id}", flush=True) threading.Thread( target=run_desktop_session, args=(session_id, ws_url, d_port, insecure, host_header, api_key, d_vnc_pw, d_w, d_h, d_match), kwargs={"desired_from_preference": d_pref}, daemon=True ).start() continue if task_type == "upgrade_agent": d_url = task.get("downloadUrl") or f"{server_url}/download/sentinel-agent.py" t_ver = task.get("version", "latest") print(f"[{time.strftime('%X')}] [⬆️ UPGRADE TASK] ID: {task_id} | Target: {t_ver}", flush=True) threading.Thread( target=upgrade_agent_self, kwargs={ "download_url": d_url, "target_version": t_ver, "server_url": server_url, "insecure": insecure, "host_header": host_header, "tenant_id": tenant_id, "api_key": api_key, "task_id": task_id, "node_id": node_id }, daemon=True ).start() continue if task_type == "restart_agent": print(f"[{time.strftime('%X')}] [⚡ RESTART TASK] ID: {task_id} | Restarting agent in-place via execv", flush=True) def _task_reexec(): time.sleep(0.8) _respawn_self() threading.Thread(target=_task_reexec, daemon=True).start() continue # Everything else runs on the task pool, not inline -- one slow task (an image # pull, a package install) must never delay the next heartbeat or a sibling task # for a different container. Tasks that target the same container still serialize # against each other, in dispatch order, via _container_task_lock. cid = task.get("containerId") if cid: _task_executor.submit(_run_task_serialized, cid, task) else: _task_executor.submit(_run_task_safely, task) except Exception as e: print(f"[{time.strftime('%X')}] [-] Error in agent loop: {e}", flush=True) time.sleep(min(active_interval, 5)) continue # ── Adaptive Heartbeat Rate ─────────────────────────────────────────── if wake_hint: # Server already flipped the record out of dormant before replying -- this agent just # needs to reconnect promptly rather than wait out whatever the dormant interval was. _dormant_mode.clear() idle_ticks = 0 active_interval = FAST_INTERVAL print(f"[{time.strftime('%X')}] [⚡ WAKE] Activated — reconnecting now", flush=True) time.sleep(active_interval) continue if dormant_hint: # Overrides the normal fast/idle logic entirely: the server explicitly asked for a # sparse, scheduled check-in, which is a different thing from "nothing to do right # now" -- the idle backoff above still eventually re-speeds up on its own, this must not. _dormant_mode.set() idle_ticks = 0 active_interval = server_interval or DORMANT_INTERVAL_SEC time.sleep(active_interval) continue had_tasks = bool(tasks) # type: ignore if had_tasks: idle_ticks = 0 if active_interval != FAST_INTERVAL: active_interval = FAST_INTERVAL print(f"[{time.strftime('%X')}] [⚡ FAST MODE] Task received — heartbeat interval → {FAST_INTERVAL}s", flush=True) else: # Honor server hint if it says to speed up if server_interval is not None and server_interval < active_interval: # type: ignore active_interval = server_interval idle_ticks = 0 elif active_interval == FAST_INTERVAL: idle_ticks += 1 if idle_ticks >= IDLE_TICKS_TO_BACKOFF: active_interval = poll_interval idle_ticks = 0 print(f"[{time.strftime('%X')}] [💤 IDLE MODE] No new tasks — heartbeat interval → {poll_interval}s", flush=True) time.sleep(active_interval) def main(): _install_agent_log_capture() parser = argparse.ArgumentParser(description="Applotus Sentinel Node Agent v1.1.0") # Every option falls back to an environment variable, because the agent also ships as a container # image: baking connection details into CMD would mean rebuilding the image to point it at a # different server, and env is how a Kubernetes Deployment or a fly machine expects to be # configured. Flags still win when both are given. parser.add_argument("--server", default=_cfg("SERVER", "https://applotus.fly.dev"), help="Applotus server URL (env: APPLOTUS_SERVER)") parser.add_argument("--name", default=_cfg("NODE_NAME") or socket.gethostname(), help="Friendly name for this server node (env: APPLOTUS_NODE_NAME)") parser.add_argument("--id", default=_cfg("NODE_ID"), help="Custom Node ID, auto-generated if empty (env: APPLOTUS_NODE_ID)") parser.add_argument("--api-key", default=_cfg("API_KEY"), help="Account / Tenant / node API Key (env: APPLOTUS_API_KEY)") parser.add_argument("--tenant", default=_cfg("TENANT_ID"), help="Tenant Workspace ID (env: APPLOTUS_TENANT_ID)") parser.add_argument("--interval", type=int, default=int(_cfg("INTERVAL", "10")), help="Heartbeat & polling interval in seconds (env: APPLOTUS_INTERVAL)") parser.add_argument("--insecure", "-k", action="store_true", default=_cfg("INSECURE").lower() in ("1", "true", "yes"), help="Skip SSL certificate verification (env: APPLOTUS_INSECURE)") parser.add_argument("--host-header", default=_cfg("HOST_HEADER") or None, help="Host header for virtual host routing (env: APPLOTUS_HOST_HEADER)") # Set when this agent runs inside a container alongside a workload (sidecar). It makes the # agent register as a child of its host node rather than as another standalone host. parser.add_argument("--kind", default=_cfg("NODE_KIND"), help="Node kind: blank for a host, 'container' for a sidecar agent") parser.add_argument("--parent-node-id", default=_cfg("PARENT_NODE_ID"), help="Host node ID this container runs on (sidecar agents only)") parser.add_argument("--container-id", default=_cfg("CONTAINER_ID"), help="Workload container ID this sidecar is attached to") args = parser.parse_args() node_id = args.id if not node_id: node_id = f"node-{socket.gethostname().lower().replace(' ', '-')}" run_agent( server_url=args.server, node_name=args.name, node_id=node_id, poll_interval=args.interval, insecure=args.insecure, host_header=args.host_header, tenant_id=args.tenant or os.environ.get("APPLOTUS_TENANT_ID"), api_key=args.api_key, node_kind=(args.kind or None), parent_node_id=(args.parent_node_id or None), workload_container_id=(args.container_id or None) ) if __name__ == "__main__": main()