#!/usr/bin/env python3 """Linux/Docker reproduction runner. Creates isolated, temporary lab resources.""" import datetime import json import os import pathlib import secrets import subprocess import sys import tempfile import time KIT = pathlib.Path(__file__).resolve().parent IMAGE = 'docker.n8n.io/n8nio/n8n:2.35.4' STAMP = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d%H%M%S') PREFIX = 'marsen-approval-lab-' + STAMP WORK = pathlib.Path(tempfile.mkdtemp(prefix=PREFIX + '-')) RESULT = KIT / 'test-results.json' LOG = KIT / 'runtime.log' NAMES = [PREFIX + '-n8n', PREFIX + '-fixture'] BASELINE = subprocess.check_output(['docker', 'ps', '--format', '{{.Names}}'], text=True).splitlines() logs = [] def run(args, *, check=True): proc = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logs.append(proc.stdout) if proc.returncode and check: raise RuntimeError('Lab command failed: ' + ' '.join(args[:3]) + '\n' + proc.stdout[-2500:]) return proc.stdout def wait_ready(): for _ in range(90): # Inspect exit status; an empty stdout is not evidence of readiness. ready = subprocess.run(['docker', 'exec', NAMES[1], 'node', '-e', "Promise.all([fetch('http://fixture:8787/health'),fetch('http://n8n:5678/healthz/readiness')]).then(r=>{if(r.some(x=>!x.ok))process.exit(1)}).catch(()=>process.exit(1))"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if ready.returncode == 0: return time.sleep(2) raise RuntimeError('Isolated n8n startup timed out') try: for folder in ['n8n-state', 'fixture-state']: (WORK / folder).mkdir(mode=0o700) if os.geteuid() == 0: os.chown(WORK, 1000, 1000) for folder in ['n8n-state', 'fixture-state']: os.chown(WORK / folder, 1000, 1000) elif os.getuid() != 1000: raise RuntimeError('Run on Linux as uid 1000 or root so temporary state is writable by the image user') reviewer, service, control = [secrets.token_hex(32) for _ in range(3)] env = WORK / 'lab.env' env.write_text(f'LAB_REVIEWER_TOKEN={reviewer}\nLAB_SERVICE_TOKEN={service}\nLAB_CONTROL_TOKEN={control}\n') env.chmod(0o600) credentials = WORK / 'credentials.json' credentials.write_text(json.dumps([ {'id': 'marsenLabReviewer', 'name': 'LAB ONLY — reviewer-a header', 'type': 'httpHeaderAuth', 'data': {'name': 'x-lab-reviewer-token', 'value': reviewer}}, {'id': 'marsenLabService', 'name': 'LAB ONLY — synthetic fixture service', 'type': 'httpHeaderAuth', 'data': {'name': 'x-lab-service-token', 'value': service}}, ])) credentials.chmod(0o600) if os.geteuid() == 0: os.chown(credentials, 1000, 1000) run(['docker', 'network', 'create', '--internal', PREFIX]) common = ['docker', 'run', '--rm', '--network', PREFIX, '--memory', '768m', '--cpus', '.75', '--pids-limit', '256', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '-v', f'{KIT}:/lab:ro', '-v', f'{WORK}/n8n-state:/home/node/.n8n', '-e', 'N8N_DIAGNOSTICS_ENABLED=false', '-e', 'N8N_VERSION_NOTIFICATIONS_ENABLED=false', '-e', 'N8N_TEMPLATES_ENABLED=false', '-e', 'N8N_LICENSE_AUTO_RENEW_ENABLED=false', '-e', 'N8N_LICENSE_RENEW_ON_INIT_ENABLED=false', '-e', 'N8N_RUNNERS_ENABLED=false'] run(common + ['-v', f'{credentials}:/lab-credentials.json:ro', '--entrypoint', '/bin/sh', IMAGE, '-c', 'n8n import:workflow --input=/lab/workflow.json && n8n import:credentials --input=/lab-credentials.json && n8n publish:workflow --id=marsenRecoveryLabV1']) # Detached runtime: no host port, live n8n data or production credentials mounted. runtime = common.copy() runtime.remove('--rm') run(runtime + ['-d', '--name', NAMES[0], '--network-alias', 'n8n', '-e', 'N8N_HOST=n8n', '-e', 'N8N_SECURE_COOKIE=false', IMAGE, 'start']) run(['docker', 'run', '-d', '--name', NAMES[1], '--network', PREFIX, '--network-alias', 'fixture', '--memory', '256m', '--cpus', '.25', '--pids-limit', '128', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--env-file', str(env), '-v', f'{KIT}:/lab:ro', '-v', f'{WORK}/fixture-state:/state', '--entrypoint', 'node', IMAGE, '/lab/fixture-server.mjs']) wait_ready() initial = run(['docker', 'exec', NAMES[1], 'node', '/lab/test-lab.mjs', 'initial']) if 'READY_FOR_PROCESS_RESTART' not in initial: raise RuntimeError('Initial recovery checkpoint missing') restart_started = datetime.datetime.now(datetime.timezone.utc).isoformat() for name in NAMES: run(['docker', 'kill', '--signal=KILL', name]) for name in NAMES: run(['docker', 'start', name]) wait_ready() after = run(['docker', 'exec', NAMES[1], 'node', '/lab/test-lab.mjs', 'after-restart']) if 'ALL_RECOVERY_CASES_PASSED' not in after: raise RuntimeError('Post-restart checks incomplete') cases = json.loads((WORK / 'fixture-state/test-progress.json').read_text()) before = json.loads((WORK / 'fixture-state/pre-restart-evidence.json').read_text()) executions = run(['docker', 'exec', NAMES[0], 'node', '-e', "const {DatabaseSync}=require('node:sqlite');const d=new DatabaseSync('/home/node/.n8n/database.sqlite',{readOnly:true});console.log(JSON.stringify(d.prepare('SELECT id,status,workflowId,startedAt,stoppedAt FROM execution_entity WHERE workflowId=? ORDER BY id').all('marsenRecoveryLabV1')))" ]) # sqlite may print an experimental warning after the JSON: select the JSON line. execution_rows = json.loads(next(line for line in executions.splitlines() if line.startswith('['))) data = { 'labVersion': '1.0.0', 'testedAt': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runtime': {'n8n': '2.35.4', 'node': '24.18.1', 'image': IMAGE, 'mode': 'single n8n process, HTTP webhook, private Docker network'}, 'scope': 'Synthetic proposals and destination; separate SQLite decision and receipt databases; one authenticated reviewer credential mapped to fixed reviewer-a.', 'faultModel': 'Synthetic HTTP 503 before write and HTTP 504 after committed write; not a measured provider network timeout.', 'restart': {'method': 'SIGKILL then start both isolated containers', 'startedAt': restart_started, 'checkpoint': 'One pending proposal; two approved actions pending recovery; tests had completed their webhook responses before SIGKILL.'}, 'cases': cases, 'total': len(cases), 'passed': sum(c['passed'] for c in cases), 'executions': execution_rows, 'preRestartEvidence': before, 'limits': ['No real CRM or customer data', 'No Wait-node resumption test', 'No human approval portal or multi-reviewer authentication', 'No multi-worker/queue-mode or multi-process database contention test', 'No SIGKILL during the destination database transaction', 'No load, availability, throughput or production reliability claim'], } RESULT.write_text(json.dumps(data, indent=2) + '\n') print(json.dumps({'total': data['total'], 'passed': data['passed'], 'executionCount': len(execution_rows), 'results': str(RESULT)})) finally: for name in NAMES: logs.append(run(['docker', 'logs', '--tail', '80', name], check=False)) run(['docker', 'rm', '-f', name], check=False) run(['docker', 'network', 'rm', PREFIX], check=False) for secret_path in [WORK / 'lab.env', WORK / 'credentials.json']: if secret_path.exists(): secret_path.unlink() # Destroy the ephemeral encryption key with the disposable n8n state after evidence export. import shutil shutil.rmtree(WORK / 'n8n-state', ignore_errors=True) LOG.write_text('\n'.join(logs)) remaining = subprocess.check_output(['docker', 'ps', '--format', '{{.Names}}'], text=True).splitlines() cleanup = {'baselineCount': len(BASELINE), 'productionContainersStillRunning': all(n in remaining for n in BASELINE), 'labContainersRemoved': all(n not in remaining for n in NAMES), 'retainedSyntheticState': str(WORK / 'fixture-state')} (KIT / 'cleanup.json').write_text(json.dumps(cleanup, indent=2) + '\n') print(json.dumps(cleanup))