From 988afecf02bcefa13b40385276d77c283f719ea7 Mon Sep 17 00:00:00 2001
From: RetroGameDeveloper
<40120498+RetroGameDeveloper@users.noreply.github.com>
Date: Wed, 28 Oct 2026 23:14:44 +0000
Subject: [PATCH 1/5] Add resumable deep permutation passes across unmatched
functions
---
ROADMAP.md | 6 +
src/decompnet/app/deep_permuter.py | 11 +-
src/decompnet/app/deep_sweep.py | 191 ++++++++++++++++++++++++
src/decompnet/app/engine/gui.py | 5 +-
src/decompnet/app/routes/projects.py | 14 ++
src/decompnet/app/server.py | 5 +
src/decompnet/app/static/index.html | 14 ++
src/decompnet/app/static/js/base.js | 4 +-
src/decompnet/app/static/js/function.js | 2 +
src/decompnet/app/static/js/steering.js | 57 +++++++
tests/gui_progress.js | 31 +++-
tests/test_app_server.py | 14 ++
tests/test_deep_sweep.py | 190 +++++++++++++++++++++++
13 files changed, 539 insertions(+), 5 deletions(-)
create mode 100644 src/decompnet/app/deep_sweep.py
create mode 100644 tests/test_deep_sweep.py
diff --git a/ROADMAP.md b/ROADMAP.md
index a7ae8e8b..fd414353 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -888,3 +888,9 @@ agent briefs and deep-search output. Planned: compiler-specific diagnostic flags
opt-in diagnostic build command for imported projects (their ordinary build diagnostics remain available).
Reuse discovery batches fingerprint names/cache reads and persists a separate cheap-discovery cursor; at most
half the remaining shared reuse time goes to that phase, leaving time for wider matching.
+
+Project settings now offer a resumable deep permutation pass across unmatched functions in ROM and imported
+projects. Each function's completed bounded turn is persisted with its source/build scope and budget. Unfinished
+turns resume after restart; changed inputs become eligible again. Busy functions and candidates without scored C
+wait without being marked complete. The pass uses the same local/random searches, claims and judges as a single
+function, with a visible progress bar and stop button. No paid helper is launched.
diff --git a/src/decompnet/app/deep_permuter.py b/src/decompnet/app/deep_permuter.py
index 096ca351..b7b126b5 100644
--- a/src/decompnet/app/deep_permuter.py
+++ b/src/decompnet/app/deep_permuter.py
@@ -100,6 +100,10 @@ def _score(self, agent, proj, fn, source, best, fork):
def run(net, slug, name, minutes, max_variants, log):
"""Visit the supported local repairs, then spend the remaining budget on random permutations."""
proj, fn = validate(net, slug, name, minutes, max_variants)
+ sweeps = getattr(net, 'deep_sweeps', None)
+ if sweeps:
+ from ..candidate_scope import key
+ build_scope = key(net, proj, fn)
search = Search(net, minutes, max_variants, log)
state = State()
agent, _ = net.local_agent('permuter')
@@ -157,5 +161,8 @@ def run(net, slug, name, minutes, max_variants, log):
reason = str(error)
search.report()
log(f'{reason}: {search.tested} candidates tested, {search.skipped} cached misses skipped, {search.rejected} invalid calls skipped. Verified improvements are saved immediately.')
- return {'function': name, 'reason': reason, 'tested': search.tested, 'skipped': search.skipped,
- 'rejected': search.rejected, 'matched': net._function(name, slug)['status'] == 'matched', 'best_score': search.best}
+ result = {'function': name, 'reason': reason, 'tested': search.tested, 'skipped': search.skipped,
+ 'rejected': search.rejected, 'matched': net._function(name, slug)['status'] == 'matched', 'best_score': search.best}
+ if sweeps:
+ sweeps.completed(slug, fn, minutes, max_variants, result, build_scope)
+ return result
diff --git a/src/decompnet/app/deep_sweep.py b/src/decompnet/app/deep_sweep.py
new file mode 100644
index 00000000..ee261802
--- /dev/null
+++ b/src/decompnet/app/deep_sweep.py
@@ -0,0 +1,191 @@
+"""Durable project-wide deep passes; each candidate gets a bounded turn through the normal judges."""
+import hashlib
+import json
+import threading
+import time
+
+from .. import candidate_scope
+from ..network import NetworkError
+from . import agent_tokens, deep_permuter
+
+POLL_SECONDS = 15
+SCHEMA = '''CREATE TABLE IF NOT EXISTS deep_permutation_passes(
+ function_id INTEGER PRIMARY KEY, scope TEXT NOT NULL, state TEXT NOT NULL,
+ minutes INTEGER NOT NULL, max_variants INTEGER NOT NULL, completed_at REAL,
+ result TEXT NOT NULL DEFAULT '{}', error TEXT NOT NULL DEFAULT '')'''
+
+
+def signature(net, proj, fn, minutes, variants):
+ return hashlib.sha256(repr(('deep-pass-v1', agent_tokens.target_scope(net, fn, ''),
+ fn['best_source'], minutes, variants)).encode()).hexdigest()
+
+
+class Sweeps:
+ def __init__(self, net, jobs):
+ self.net, self.jobs = net, jobs
+ self.active = {}
+ self.lock = threading.RLock()
+ with net.store.tx() as db:
+ db.execute(SCHEMA)
+
+ def config(self, slug):
+ proj = self.net.project_row(slug)
+ row = self.net.store.one('SELECT value FROM settings WHERE key=?', f'deep-sweep:{proj["id"]}')
+ return json.loads(row['value']) if row else {'enabled': False, 'minutes': 5, 'max_variants': 10000}
+
+ def save(self, slug, cfg):
+ proj = self.net.project_row(slug)
+ with self.net.store.tx() as db:
+ db.execute('INSERT OR REPLACE INTO settings(key,value) VALUES(?,?)',
+ (f'deep-sweep:{proj["id"]}', json.dumps(cfg)))
+
+ def history(self, slug, fn):
+ row = self.net.store.one('SELECT * FROM deep_permutation_passes WHERE function_id=?', fn['id'])
+ if not row:
+ return {'state': 'not tried', 'completed': False}
+ result = dict(row)
+ result['stale'] = row['scope'] != signature(self.net, self.net.project_row(slug), fn,
+ row['minutes'], row['max_variants'])
+ result['completed'] = row['state'] == 'completed' and not result['stale']
+ result['result'] = json.loads(row['result'])
+ return result
+
+ def status(self, slug):
+ proj = self.net.project_row(slug)
+ counts = self.net.store.one("SELECT COUNT(*) total, SUM(best_source IS NOT NULL AND best_score IS NOT NULL) ready "
+ "FROM functions WHERE project_id=? AND status='open'", proj['id'])
+ completed = self.net.store.one("SELECT COUNT(*) n FROM deep_permutation_passes d JOIN functions f ON f.id=d.function_id "
+ "WHERE f.project_id=? AND d.state='completed'", proj['id'])['n']
+ errors = self.net.store.one("SELECT COUNT(*) n FROM deep_permutation_passes d JOIN functions f ON f.id=d.function_id WHERE f.project_id=? AND d.state='error'", proj['id'])['n']
+ with self.lock:
+ current = dict(self.active.get(slug, {}))
+ job = self.jobs.jobs.get(current.get('job'), {})
+ return self.config(slug) | {'total': counts['total'], 'ready': counts['ready'] or 0,
+ 'completed': completed, 'errors': errors, 'state': job.get('state', 'idle'), **current,
+ 'progress': job.get('progress', {}), 'error': job.get('error')}
+
+ def start(self, slug, minutes=5, max_variants=10000, repeat=False):
+ if (isinstance(minutes, bool) or not isinstance(minutes, int) or not 1 <= minutes <= 120 or
+ isinstance(max_variants, bool) or not isinstance(max_variants, int) or not 1 <= max_variants <= 1000000):
+ raise NetworkError('Choose 1–120 minutes and 1–1,000,000 variants per function.')
+ self.net.project_row(slug)
+ if slug in self.net.links:
+ raise NetworkError('Run the deep pass on the room host’s project.')
+ with self.lock:
+ previous = self.active.get(slug, {})
+ if self.jobs.jobs.get(previous.get('job'), {}).get('state') == 'running':
+ raise NetworkError('A deep pass is already running for this project. Stop it before starting another.')
+ if repeat:
+ with self.net.store.tx() as db:
+ db.execute("DELETE FROM deep_permutation_passes WHERE function_id IN "
+ "(SELECT id FROM functions WHERE project_id=? AND status='open')",
+ self.net.project_row(slug)['id'])
+ self.save(slug, {'enabled': True, 'minutes': minutes, 'max_variants': max_variants})
+ job = self.jobs.start('deep-permutation-pass', lambda log: self.run(slug, minutes, max_variants, log),
+ cancel_message='Deep pass stopped. Verified improvements and completed passes are saved.')
+ self.active[slug] = {'job': job}
+ return self.status(slug)
+
+ def stop(self, slug):
+ with self.lock:
+ cfg = self.config(slug)
+ self.save(slug, cfg | {'enabled': False})
+ job = self.active.get(slug, {}).get('job')
+ if job:
+ self.jobs.cancel(job)
+ return self.status(slug)
+
+ def resume(self):
+ # An interrupted function is retried; only completed bounded turns are skipped.
+ for proj in self.net.store.all('SELECT slug FROM projects'):
+ cfg = self.config(proj['slug'])
+ if cfg['enabled']:
+ try:
+ self.start(proj['slug'], cfg['minutes'], cfg['max_variants'])
+ except NetworkError as error:
+ self.save(proj['slug'], cfg | {'enabled': False, 'resume_error': str(error)})
+ self.jobs.logger.warning('Deep pass could not resume: %s', error)
+
+ def shutdown(self):
+ # Preserve enabled intent for the next launch, but let current searches release claims.
+ with self.lock:
+ for current in self.active.values():
+ self.jobs.cancel(current['job'])
+
+ def completed(self, slug, before, minutes, variants, result, build_scope):
+ fresh = self.net._function(before['name'], slug)
+ proj = self.net.project_row(slug)
+ stable = build_scope == candidate_scope.key(self.net, proj, before)
+ self.record(fresh, signature(self.net, proj, fresh, minutes, variants),
+ 'completed' if stable else 'pending', minutes, variants, result)
+
+ def record(self, fn, scope, state, minutes, variants, result=None, error=''):
+ with self.net.store.tx() as db:
+ db.execute('INSERT OR REPLACE INTO deep_permutation_passes '
+ '(function_id,scope,state,minutes,max_variants,completed_at,result,error) VALUES(?,?,?,?,?,?,?,?)',
+ (fn['id'], scope, state, minutes, variants, time.time() if state == 'completed' else None,
+ json.dumps(result or {}), error))
+
+ def run(self, slug, minutes, variants, log):
+ """Finish one turn per unchanged candidate; wait for claims or usable C without marking them done."""
+ visited = tested = matched = 0
+ while True:
+ log.check_cancel()
+ proj = self.net.project_row(slug)
+ pending = waiting = 0
+ rows = self.net.store.all("SELECT * FROM functions WHERE project_id=? AND status='open' ORDER BY size,id", proj['id'])
+ for row in rows:
+ log.check_cancel()
+ fn = self.net._function(row['name'], slug)
+ if fn['status'] != 'open':
+ continue
+ if not fn['best_source'] or fn['best_score'] is None:
+ waiting += 1
+ continue
+ scope = signature(self.net, proj, fn, minutes, variants)
+ previous = self.net.store.one('SELECT * FROM deep_permutation_passes WHERE function_id=?', fn['id'])
+ if previous and previous['scope'] == scope and previous['state'] in ('completed', 'error'):
+ continue
+ pending += 1
+ if self.net.store.one('SELECT 1 FROM claims WHERE function_id=? AND expires_at>?', fn['id'], time.time()):
+ waiting += 1
+ continue
+ build_scope = candidate_scope.key(self.net, proj, fn)
+ self.record(fn, scope, 'running', minutes, variants)
+ log(f'Deep pass: {fn["name"]} · up to {minutes} minutes / {variants:,} variants')
+ total = len(rows)
+ def progress(value, name=fn['name'], done=visited, wins=matched, total=total, blocked=waiting):
+ log.progress({'function': name, 'functions_tested': done, 'matches': wins,
+ 'total_unmatched': total, 'waiting': blocked} | value)
+ def function_log(line):
+ log(line)
+ function_log.check_cancel = log.check_cancel
+ function_log.progress = progress
+ try:
+ result = deep_permuter.run(self.net, slug, fn['name'], minutes, variants, function_log)
+ # Also records test backends and older search providers that do not publish history.
+ # The actual single-function search uses the same completion gate.
+ self.completed(slug, fn, minutes, variants, result, build_scope)
+ visited += 1
+ tested += result['tested']
+ matched += int(result['matched'])
+ except NetworkError as error:
+ # Claim races are temporary. A bad unchanged candidate stays visible as an error,
+ # not completed, and a changed source/header or an explicit repeat retries it.
+ busy = self.net.store.one('SELECT 1 FROM claims WHERE function_id=? AND expires_at>?', fn['id'], time.time())
+ self.record(fn, scope, 'pending' if busy or 'claim' in str(error).lower() or str(error).startswith('This function is working') else 'error', minutes, variants, error=str(error))
+ log(f'{fn["name"]}: {error}')
+ except BaseException:
+ self.record(fn, scope, 'pending', minutes, variants)
+ raise
+ progress({'phase': 'moving to next function', 'tested': tested, 'functions_tested': visited, 'matches': matched})
+ if not pending and not waiting:
+ self.save(slug, self.config(slug) | {'enabled': False})
+ log('Deep pass finished. Completed bounded passes are saved; this is not an exhaustive search of all C.')
+ return {'functions_tested': visited, 'tested': tested, 'matched': matched}
+ log.progress({'phase': 'waiting for unclaimed functions or scored C', 'functions_tested': visited,
+ 'matches': matched, 'waiting': waiting, 'total_unmatched': len(rows)})
+ # Cancellation remains responsive while another worker owns the remaining functions.
+ for _ in range(POLL_SECONDS):
+ log.check_cancel()
+ time.sleep(1)
diff --git a/src/decompnet/app/engine/gui.py b/src/decompnet/app/engine/gui.py
index 25ab15ca..68b52abd 100644
--- a/src/decompnet/app/engine/gui.py
+++ b/src/decompnet/app/engine/gui.py
@@ -256,8 +256,10 @@ def function_diff(self, slug: str, name: str, which: str = "latest") -> dict:
def function_detail(self, slug: str, name: str) -> dict:
from ..function_names import state as name_state
+ sweep = getattr(self, 'deep_sweeps', None)
+ deep_pass = sweep.history(slug, self._function(name, slug)) if sweep else {'state': 'not tried', 'completed': False}
if (fork := self.fork_slug(slug)):
- return self.forks.function_detail(fork, name) | name_state(self._function(name, slug))
+ return self.forks.function_detail(fork, name) | name_state(self._function(name, slug)) | {'deep_pass': deep_pass}
fn = self._function(name, slug)
proj = self.store.one("SELECT * FROM projects WHERE id=?", fn["project_id"])
attempts = self.store.all(
@@ -275,6 +277,7 @@ def function_detail(self, slug: str, name: str) -> dict:
return {
**name_state(fn), "name": fn["name"], "vram": fn["vram"], "rom_offset": fn["rom_offset"], "size": fn["size"],
"insns": fn["n_insns"], "status": fn["status"], "best": fn["best_score"], **set_aside(fn),
+ "deep_pass": deep_pass,
"compiler_test": self.compiler_test_result(proj, fn),
"asm": self._asm_text(fn, proj),
"source": fn["matched_source"] or fn["best_source"] or (latest["source"] if latest else None) or fn["seed"],
diff --git a/src/decompnet/app/routes/projects.py b/src/decompnet/app/routes/projects.py
index 9493aed0..b9302ee4 100644
--- a/src/decompnet/app/routes/projects.py
+++ b/src/decompnet/app/routes/projects.py
@@ -237,6 +237,19 @@ def work(log):
return {'job': jobs.start('source-preparation', work)}
return source_preparation.configure(net, proj, data)
+ def p_deep_sweep(slug, request, data):
+ sweeps = net.deep_sweeps
+ if request.method == 'GET':
+ return sweeps.status(slug)
+ action = data.get('action', 'start')
+ if action == 'stop':
+ return sweeps.stop(slug)
+ if action != 'start':
+ raise ValueError('Choose start or stop for the deep pass.')
+ if not isinstance(data.get('repeat', False), bool):
+ raise ValueError('Repeat must be true or false.')
+ return sweeps.start(slug, data.get('minutes', 5), data.get('max_variants', 10000), data.get('repeat', False))
+
def p_deep_permuter(slug, request, data):
from .. import deep_permuter
name = str(data.get('function') or '')
@@ -332,6 +345,7 @@ def p_join_rom(slug, request, data):
"/api/try": (p_try, ["POST"]), "/api/release": (p_release, ["POST"]), "/api/reopen": (p_reopen, ["POST"]), "/api/setaside": (p_set_aside, ["POST"]),
"/api/recheck": (p_recheck, ["POST"]),
"/api/permuter": (p_permuter, ["POST"]),
+ "/api/permuter/pass": (p_deep_sweep, ["GET", "POST"]),
"/api/permuter/function": (p_deep_permuter, ["POST"]),
"/api/source-preparation": (p_source_preparation, ["GET", "POST"]),
"/api/autopilot": (p_autopilot, ["POST"]), "/api/instructions": (p_instructions, ["POST"]), "/api/title": (p_title, ["POST"]),
diff --git a/src/decompnet/app/server.py b/src/decompnet/app/server.py
index 376fbfaa..30102802 100644
--- a/src/decompnet/app/server.py
+++ b/src/decompnet/app/server.py
@@ -36,6 +36,8 @@ def build(net: RomNetwork, home: Path, port: int, secret: str):
build_id = (updates.installed().get("commit") or "")[:12] # a branch install: the commit it came from
manager = agents.Manager(net, home, mcp_url)
jobs = Jobs(home / "logs" / "decomp.log")
+ from .deep_sweep import Sweeps
+ net.deep_sweeps = Sweeps(net, jobs)
def busy() -> bool:
return bool(manager.running()) or any(j["state"] == "running" for j in jobs.jobs.values())
@@ -52,6 +54,7 @@ def window_ended():
def quit_app() -> None:
"""Quit means quit: the window, every helper with everything it started, then the server."""
window.close_app()
+ net.deep_sweeps.shutdown()
manager.shutdown()
os._exit(0)
window.WINDOW = window.Window(home, f"http://127.0.0.1:{port}/", secret, window_ended)
@@ -110,6 +113,7 @@ def options_now(kind: str, options: dict) -> dict | None:
return None
return options | (session_options(saved) if saved is not None else {})
manager.options_now = options_now
+ net.deep_sweeps.resume()
manager.resume() # helpers that were working when the app last stopped carry on (their records say which)
mcp = tools.build_mcp(net, agent_of, observe=manager.record_agent)
@@ -157,6 +161,7 @@ def fail(e: Exception, code: int = 400) -> JSONResponse:
return JSONResponse({"error": str(e)}, status_code=code)
def before_restart() -> None: # helpers stop now and resume in the new version; the window closes with the process
+ net.deep_sweeps.shutdown()
manager.shutdown()
# the app's parts add their routes (routes/__init__.py says which is where); forks come after state and projects,
diff --git a/src/decompnet/app/static/index.html b/src/decompnet/app/static/index.html
index 7cfa2b90..4f011f98 100644
--- a/src/decompnet/app/static/index.html
+++ b/src/decompnet/app/static/index.html
@@ -385,6 +385,19 @@
ROM map
+
+
Deep permutation pass
all unmatched functions · no AI tokens
+
+
Give each function a longer turn through local repairs and random permutations. Completed passes are saved, so you can stop, close the app and resume later. Changed source, headers or compiler inputs make a function eligible again.
+
+
+
+
One function at a time. Busy functions and those without scored C wait for their agent or draft. Only the ordinary byte judge can save a match. A completed pass means its bounded turn finished, not that every possible rewrite was tried.
+
+
+
+
+
Compiler
@@ -574,6 +587,7 @@
Deep permuter · this function
+
Tries the supported local repairs, then searches random rewrites of this function’s saved C. No AI tokens. This is a bounded search, not every possible C program. Exact matches and better candidates are saved immediately.
diff --git a/src/decompnet/app/static/js/base.js b/src/decompnet/app/static/js/base.js
index a523db4b..442e1145 100644
--- a/src/decompnet/app/static/js/base.js
+++ b/src/decompnet/app/static/js/base.js
@@ -130,9 +130,11 @@ function beginWork(log, label, controls = [], completed = "Finished.") {
};
}
-async function followJob(id, logEl, onUpdate) {
+async function followJob(id, logEl, onUpdate, active = () => true) {
for (;;) {
+ if (!active()) return null;
const job = await api(`/api/job?id=${id}`);
+ if (!active()) return null;
if (logEl) {
const text = job.log.join("\n");
if (logEl.textContent !== text) {
diff --git a/src/decompnet/app/static/js/function.js b/src/decompnet/app/static/js/function.js
index bc887e8a..2fa74667 100644
--- a/src/decompnet/app/static/js/function.js
+++ b/src/decompnet/app/static/js/function.js
@@ -126,6 +126,8 @@ async function refreshFn(first) {
: d.renamed ? `Renamed from ${d.naming.original_name}` : d.named ? "Named function" : "Generated name";
fnBadge(d);
$("fnDeepPanel").hidden = !deepPermuterBusy && (d.status !== "open" || d.best == null);
+ const pass = d.deep_pass || {};
+ $("fnDeepHistory").textContent = pass.completed ? `Deep pass completed · ${pass.minutes} min / ${Number(pass.max_variants).toLocaleString()} variants budget` : pass.stale ? "Previous pass saved; inputs changed, so this function is eligible again." : pass.state === "error" ? `Deep pass needs attention: ${pass.error}` : `Deep pass: ${pass.state || "not tried"}`;
$("fnCompilerTest").hidden = S.ov.kind === "fork" || d.status !== "open" || !S.ov.compilers?.length;
if (!compilerComparisonBusy && S.ov.kind !== "fork") {
const history = d.compiler_test;
diff --git a/src/decompnet/app/static/js/steering.js b/src/decompnet/app/static/js/steering.js
index aacd415d..0cf79631 100644
--- a/src/decompnet/app/static/js/steering.js
+++ b/src/decompnet/app/static/js/steering.js
@@ -432,6 +432,7 @@ function projectCompilerFacts(ov) {
}
async function renderProjectSettings() {
+ renderDeepSweep();
if ($("compilerImprovements").dataset.slug !== S.slug) $("compilerImprovements").replaceChildren();
if ($("compilerProgress").dataset.slug !== S.slug) {
$("compilerProgress").hidden = true;
@@ -807,3 +808,59 @@ $("sourcePrepRun").onclick = async () => {
refreshMap();
} catch (e) { showJobError(log, e.message); } finally { finish(); }
};
+
+
+let deepSweepFollowing = null;
+async function renderDeepSweep() {
+ const slug = S.slug;
+ if (!slug) return;
+ try {
+ const status = await api(`/api/permuter/pass?slug=${encodeURIComponent(slug)}`);
+ if (S.slug !== slug) return;
+ if ($("deepSweepPanel").dataset.slug !== slug) {
+ $("deepSweepPanel").dataset.slug = slug;
+ $("deepSweepMinutes").value = status.minutes;
+ $("deepSweepVariants").value = status.max_variants;
+ $("deepSweepRepeat").checked = false;
+ }
+ $("deepSweepStart").disabled = status.state === "running";
+ $("deepSweepStop").hidden = status.state !== "running";
+ $("deepSweepStatus").textContent = `${status.completed} saved completed passes · ${status.errors} need attention · ${status.ready} of ${status.total} unmatched functions have scored C`;
+ if (status.error) $("deepSweepStatus").textContent += ` · ${status.error}`;
+ if (status.state === "running" && deepSweepFollowing !== status.job) followDeepSweep(slug, status.job);
+ } catch (error) { if (S.slug === slug) showJobError($("deepSweepLog"), error.message); }
+}
+async function followDeepSweep(slug, job) {
+ deepSweepFollowing = job;
+ const host = $("deepSweepLog"), log = document.createElement("pre");
+ host.textContent = ""; host.append(log);
+ $("deepSweepStart").disabled = true;
+ const done = beginWork(log, "Deep permutation pass", [], "Deep pass finished");
+ try {
+ await followJob(job, log, (_, state) => {
+ const progress = state.progress || {};
+ if (S.slug !== slug) return;
+ $("deepSweepStatus").textContent = `${progress.function || "Queue"} · ${progress.phase || "working"} · ${progress.functions_tested || 0} functions tested · ${progress.matches || 0} matches`;
+ }, () => S.slug === slug);
+ } catch (error) { if (S.slug === slug) showJobError(log, error.message); }
+ finally {
+ done();
+ if (deepSweepFollowing === job) deepSweepFollowing = null;
+ if (S.slug === slug) renderDeepSweep();
+ }
+}
+$("deepSweepStart").onclick = async () => {
+ const slug = S.slug, log = $("deepSweepLog");
+ const done = beginWork(log, "Starting deep pass", [$("deepSweepStart")], "Deep pass started");
+ let job = null;
+ try {
+ const result = await api("/api/permuter/pass", {method:"POST", body:{slug, action:"start", minutes:Number($("deepSweepMinutes").value), max_variants:Number($("deepSweepVariants").value), repeat:$("deepSweepRepeat").checked}});
+ job = result.job;
+ } catch (error) { showJobError(log, error.message); }
+ finally { done(); }
+ if (job && S.slug === slug) { $("deepSweepStop").hidden = false; followDeepSweep(slug, job); }
+};
+$("deepSweepStop").onclick = async () => {
+ try { await api("/api/permuter/pass", {method:"POST", body:{slug:S.slug, action:"stop"}}); }
+ catch (error) { showJobError($("deepSweepLog"), error.message); }
+};
diff --git a/tests/gui_progress.js b/tests/gui_progress.js
index b2c20c5f..40befc92 100644
--- a/tests/gui_progress.js
+++ b/tests/gui_progress.js
@@ -53,7 +53,7 @@ function setup(file, marker, end, logId, buttonId, ranges = "") {
let polls = 0;
const ctx = {
$: get, document: { createElement: element },
- deepPermuterBusy: false, deepPermuterJob: null, refreshFn: async () => {},
+ followDeepSweep: () => {}, deepPermuterBusy: false, deepPermuterJob: null, refreshFn: async () => {},
S: { slug: "test", fn: "f", ov: { kind: "fork" } }, info: {}, box, body: box, n: get("compilerNotice"), log, inst: button,
b: Object.assign(button, { dataset: { inst: "test" } }), a: { id: "test", name: "Test" }, ROOM_PORT: 6481,
results: get("localResults"), scan: { folder: "/synthetic/collection", projects: [] }, picked: new Set(), renderList() {},
@@ -123,6 +123,35 @@ async function main() {
}
}
}
+ // A project sweep has separate start and long-running follow phases, including page navigation.
+ for (const failed of [false, true]) {
+ const test = setup("steering", '$("deepSweepStart").onclick = async () => {', '\n};', "deepSweepLog", "deepSweepStart");
+ const followed = [];
+ test.ctx.followDeepSweep = (...args) => followed.push(args);
+ const running = test.ctx.run();
+ assert.equal(test.button.disabled, true);
+ assert.ok(test.log.bar && !test.log.bar.removed);
+ if (failed) test.start.reject(new Error("Cannot start"));
+ else test.start.resolve({job:"sweep"});
+ await running;
+ assert.equal(test.log.bar.removed, true);
+ assert.equal(test.button.disabled, false);
+ assert.equal(followed.length, failed ? 0 : 1);
+ }
+ for (const outcome of ["done", "error", "cancelled", "detached"]) {
+ const test = setup("steering", 'async function followDeepSweep(slug, job) {', '\n}\n', "deepSweepLog", "deepSweepStart");
+ Object.assign(test.ctx, {slug:"test", job:"sweep", deepSweepFollowing:null, renderDeepSweep:()=>{}});
+ const running = test.ctx.run();
+ await tick();
+ const log = test.log.children[0];
+ assert.ok(log.bar && !log.bar.removed);
+ if (outcome === "detached") test.ctx.S.slug = "other";
+ test.poll.resolve({state:outcome === "detached" ? "done" : outcome, log:["Working"], error:"Stopped", progress:{function:"f", phase:"local repairs"}, result:{}});
+ await running;
+ assert.equal(log.bar.removed, true);
+ if (outcome === "detached") assert.equal(log.textContent, "");
+ else if (outcome !== "done") assert.equal(log.error, "Stopped");
+ }
// SDK indexing includes optional local C and always clears its foreground progress bar.
for (const fail of [false, true]) {
const test = setup("steering", '$("sdkImport").onclick = async () => {', '\n};', "sdkLog", "sdkImport");
diff --git a/tests/test_app_server.py b/tests/test_app_server.py
index 53d21b63..38ce6f8b 100644
--- a/tests/test_app_server.py
+++ b/tests/test_app_server.py
@@ -946,3 +946,17 @@ def run(net, slug, name, minutes, variants, log):
assert result['state'] == 'done'
assert result['result']['function'] == 'selected_function'
assert checked == [('selected-game','selected_function',10,99)]
+
+
+def test_deep_sweep_routes_validate_start_stop_and_status(app, monkeypatch):
+ client, net, _ = app
+ calls = []
+ monkeypatch.setattr(net.deep_sweeps, 'start', lambda *args: calls.append(args) or {'job':'sweep','enabled':True})
+ monkeypatch.setattr(net.deep_sweeps, 'stop', lambda slug: {'enabled':False,'project':slug})
+ monkeypatch.setattr(net.deep_sweeps, 'status', lambda slug: {'project':slug,'completed':2})
+ assert client.get('/api/permuter/pass?slug=example').json()['completed'] == 2
+ response = client.post('/api/permuter/pass', json={'slug':'example','minutes':7,'max_variants':99,'repeat':True}, headers={'X-Decomp':'1'})
+ assert response.json()['job'] == 'sweep' and calls == [('example',7,99,True)]
+ assert not client.post('/api/permuter/pass', json={'slug':'example','action':'stop'}, headers={'X-Decomp':'1'}).json()['enabled']
+ assert client.post('/api/permuter/pass', json={'slug':'example','repeat':'yes'}, headers={'X-Decomp':'1'}).status_code == 400
+ assert client.post('/api/permuter/pass', json={'slug':'example','action':'unknown'}, headers={'X-Decomp':'1'}).status_code == 400
diff --git a/tests/test_deep_sweep.py b/tests/test_deep_sweep.py
new file mode 100644
index 00000000..0a67e49b
--- /dev/null
+++ b/tests/test_deep_sweep.py
@@ -0,0 +1,190 @@
+"""A durable deep pass visits all eligible candidates without claiming unfinished work is complete."""
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from decompnet.app import deep_permuter, deep_sweep
+from decompnet.app.deep_sweep import Sweeps
+from decompnet.app.engine import RomNetwork
+from decompnet.app.support import Jobs
+from tests.test_deep_permuter import Log
+
+
+@pytest.fixture
+def sweep(tmp_path, monkeypatch):
+ net = RomNetwork(tmp_path / 'db.sqlite', tmp_path)
+ with net.store.tx() as db:
+ pid = db.execute('INSERT INTO projects(slug,name,arch,root,config) VALUES(?,?,?,?,?)',
+ ('game', 'Example', 'mips', str(tmp_path), json.dumps({'platform':'n64',
+ 'compiler':{'toolchain':'gcc-2.7.2-kmc','flags':['-O2']}}))).lastrowid
+ for i, status in enumerate(['open', 'open', 'matched', 'asm']):
+ db.execute('INSERT INTO functions(project_id,name,status,size,vram,best_source,best_score) VALUES(?,?,?,?,?,?,?)',
+ (pid, f'f{i}', status, 16+i, 0x80001000+i*32, f'int f{i}(void) {{return 1;}}', 50))
+ obj = Sweeps(net, Jobs())
+ net.deep_sweeps = obj
+ monkeypatch.setattr(deep_sweep, 'POLL_SECONDS', 0)
+ calls = []
+ def search(net, slug, name, minutes, variants, log):
+ calls.append(name)
+ with net.store.tx() as db:
+ db.execute('UPDATE functions SET best_score=25,best_source=? WHERE name=?',
+ (f'int {name}(void) {{return 2;}}', name))
+ return {'tested':3,'matched':False,'reason':'Time limit reached','best_score':25}
+ monkeypatch.setattr(deep_permuter, 'run', search)
+ return obj, net, calls, tmp_path
+
+
+@pytest.mark.parametrize('imported', [False, True])
+def test_persists_bounded_completion_and_resumes_after_restart(sweep, monkeypatch, imported):
+ obj, net, calls, path = sweep
+ if imported:
+ monkeypatch.setattr(net, 'fork_slug', lambda _: 'game')
+ net.forks = SimpleNamespace(get=lambda _: (None,SimpleNamespace(root=path,unit=lambda _:SimpleNamespace(source=None,target=None)),{}))
+ result = obj.run('game', 1, 10, Log())
+ assert result['functions_tested'] == 2 and calls == ['f0','f1']
+ assert obj.history('game', net._function('f0','game'))['completed']
+ # The pass's own improvements must not make it immediately eligible again.
+ assert obj.run('game', 1, 10, Log())['functions_tested'] == 0
+ reopened = RomNetwork(path/'db.sqlite', path)
+ if imported:
+ monkeypatch.setattr(reopened, 'fork_slug', net.fork_slug)
+ reopened.forks = net.forks
+ again = Sweeps(reopened, Jobs())
+ assert again.run('game',1,10,Log())['functions_tested'] == 0
+ with reopened.store.tx() as db:
+ db.execute("UPDATE functions SET best_source='int f0(void){return 3;}' WHERE name='f0'")
+ assert again.history('game',reopened._function('f0','game'))['stale']
+ assert again.run('game',1,10,Log())['functions_tested'] == 1
+ assert calls == ['f0','f1','f0']
+
+
+def test_budget_or_context_changes_retry_completed_passes(sweep, monkeypatch):
+ obj, net, calls, _ = sweep
+ obj.run('game',1,10,Log())
+ assert obj.run('game',2,10,Log())['functions_tested'] == 2
+ old = net._context
+ monkeypatch.setattr(net,'_context',lambda *args:dict(old(*args) or {}) | {'header':'typedef int changed;'})
+ assert obj.history('game',net._function('f0','game'))['stale']
+ assert obj.run('game',2,10,Log())['functions_tested'] == 2
+
+
+def test_cancelled_function_is_not_complete_and_can_resume(sweep, monkeypatch):
+ obj, net, calls, _ = sweep
+ def cancel(*args):
+ raise RuntimeError('cancelled')
+ original = deep_permuter.run
+ monkeypatch.setattr(deep_permuter,'run',cancel)
+ with pytest.raises(RuntimeError,match='cancelled'):
+ obj.run('game',1,10,Log())
+ assert not obj.history('game',net._function('f0','game'))['completed']
+ monkeypatch.setattr(deep_permuter,'run',original)
+ assert obj.run('game',1,10,Log())['functions_tested'] == 2
+
+
+def test_busy_and_unusable_candidates_wait_without_completion(sweep, monkeypatch):
+ obj, net, calls, _ = sweep
+ import time
+ with net.store.tx() as db:
+ db.execute("INSERT INTO claims(function_id,agent_id,expires_at) VALUES(?,?,?)",(net._function('f0','game')['id'],1,time.time()+60))
+ db.execute("UPDATE functions SET best_source=NULL,best_score=NULL WHERE name='f1'")
+ log=Log()
+ def pause(seconds):
+ log.cancelled=True
+ monkeypatch.setattr(deep_sweep,'POLL_SECONDS',1)
+ monkeypatch.setattr(deep_sweep.time,'sleep',pause)
+ with pytest.raises(RuntimeError,match='cancelled'):
+ obj.run('game',1,10,log)
+ assert calls == []
+ assert net.store.one('SELECT COUNT(*) n FROM deep_permutation_passes')['n'] == 0
+ assert log.updates[-1]['waiting'] == 2
+
+
+def test_bad_candidates_are_errors_not_completed_and_changed_inputs_retry(sweep, monkeypatch):
+ obj,net,calls,_=sweep
+ def bad(*args):
+ from decompnet.network import NetworkError
+ raise NetworkError('Saved C no longer compiles')
+ monkeypatch.setattr(deep_permuter,'run',bad)
+ assert obj.run('game',1,10,Log())['functions_tested'] == 0
+ row=obj.history('game',net._function('f0','game'))
+ assert row['state']=='error' and not row['completed'] and row['completed_at'] is None
+ assert obj.status('game')['errors']==2
+
+
+def test_changed_build_during_search_does_not_finish_the_new_build(sweep, monkeypatch):
+ obj,net,calls,_=sweep
+ original=deep_permuter.run
+ def change(*args):
+ result=original(*args)
+ # Cancel before another turn; only inspect the completed first invocation.
+ with net.store.tx() as db:
+ db.execute("UPDATE projects SET config=?",(json.dumps({'platform':'n64','compiler':{'toolchain':'gcc-2.7.2-kmc','flags':['-O3']}}),))
+ return result
+ monkeypatch.setattr(deep_permuter,'run',change)
+ log=Log()
+ original_record=obj.record
+ def record(fn,scope,state,*args,**kwargs):
+ original_record(fn,scope,state,*args,**kwargs)
+ if state=='pending':log.cancelled=True
+ monkeypatch.setattr(obj,'record',record)
+ with pytest.raises(RuntimeError,match='cancelled'):obj.run('game',1,10,log)
+ assert not obj.history('game',net._function('f0','game'))['completed']
+
+
+def test_start_stop_and_restart_keep_enabled_intent_and_prevent_parallel_sweeps(sweep, monkeypatch):
+ import threading
+ import time
+
+ from decompnet.network import NetworkError
+ obj,net,calls,_=sweep
+ entered,finish=threading.Event(),threading.Event()
+ def block(*args):
+ entered.set()
+ finish.wait(2)
+ args[-1].check_cancel()
+ return {'tested':0,'matched':False}
+ monkeypatch.setattr(deep_permuter,'run',block)
+ first=obj.start('game',1,10)
+ assert entered.wait(2)
+ with pytest.raises(NetworkError,match='already running'):obj.start('game',1,10)
+ obj.shutdown()
+ assert obj.config('game')['enabled'] # normal app exit resumes next launch
+ finish.set()
+ for _ in range(100):
+ if obj.jobs.jobs[first['job']]['state']!='running':break
+ time.sleep(.01)
+ assert obj.jobs.jobs[first['job']]['state']=='cancelled'
+ assert not obj.history('game',net._function('f0','game'))['completed']
+ starts=[]
+ restarted=Sweeps(net,Jobs())
+ monkeypatch.setattr(restarted,'start',lambda *args:starts.append(args))
+ restarted.resume()
+ assert starts==[('game',1,10)]
+ obj.stop('game')
+ assert not obj.config('game')['enabled']
+
+
+def test_claim_race_is_retried_even_if_other_worker_already_released_it(sweep, monkeypatch):
+ from decompnet.network import NetworkError
+ obj,net,calls,_=sweep
+ original=deep_permuter.run
+ first=[True]
+ def race(*args):
+ if first.pop() if first else False:
+ raise NetworkError('This function was claimed meanwhile; try again after it is released.')
+ return original(*args)
+ monkeypatch.setattr(deep_permuter,'run',race)
+ assert obj.run('game',1,10,Log())['functions_tested']==2
+ assert obj.history('game',net._function('f0','game'))['completed']
+
+
+def test_single_function_search_also_records_a_completed_pass(monkeypatch):
+ from decompnet.app.permute import Permuter
+ from tests.test_deep_permuter import setup
+ net=setup(monkeypatch)
+ obj=Sweeps(net,Jobs());net.deep_sweeps=obj
+ monkeypatch.setattr(Permuter,'_score',lambda *args:40)
+ result=deep_permuter.run(net,'game','f',1,1,Log())
+ assert result['reason']=='Variant limit reached'
+ assert obj.history('game',net.function())['completed']
From 666845aaff6b3e437ad8f5cca140584e5e40e8a3 Mon Sep 17 00:00:00 2001
From: RetroGameDeveloper
<40120498+RetroGameDeveloper@users.noreply.github.com>
Date: Wed, 28 Oct 2026 23:24:03 +0000
Subject: [PATCH 2/5] Show persistent deep-pass improvement counts and
before-and-after scores
---
src/decompnet/app/deep_permuter.py | 7 +++
src/decompnet/app/deep_sweep.py | 35 +++++++++++++-
src/decompnet/app/static/index.html | 1 +
src/decompnet/app/static/js/steering.js | 16 +++++++
tests/gui_progress.js | 17 +++++++
tests/test_deep_sweep.py | 61 +++++++++++++++++++++++++
6 files changed, 136 insertions(+), 1 deletion(-)
diff --git a/src/decompnet/app/deep_permuter.py b/src/decompnet/app/deep_permuter.py
index b7b126b5..17af9742 100644
--- a/src/decompnet/app/deep_permuter.py
+++ b/src/decompnet/app/deep_permuter.py
@@ -44,6 +44,8 @@ def __init__(self, net, minutes, max_variants, log):
self.best = None
self.phase = 'local repairs'
self.measuring_baseline = False
+ self.record_improvement = lambda score: None
+ self.recorded_best = None
def check(self):
self.log.check_cancel()
@@ -53,6 +55,9 @@ def check(self):
raise SearchLimit('Variant limit reached')
def report(self):
+ if self.best is not None and (self.recorded_best is None or self.best < self.recorded_best):
+ self.record_improvement(self.best)
+ self.recorded_best = self.best
self.log.progress({'phase': self.phase, 'tested': self.tested, 'skipped': self.skipped,
'best_score': self.best, 'variant_limit': self.limit, 'rejected': self.rejected,
'seconds_left': max(0, round(self.deadline-time.monotonic()))})
@@ -105,6 +110,8 @@ def run(net, slug, name, minutes, max_variants, log):
from ..candidate_scope import key
build_scope = key(net, proj, fn)
search = Search(net, minutes, max_variants, log)
+ if sweeps:
+ search.record_improvement = lambda score: sweeps.improvement(fn, score)
state = State()
agent, _ = net.local_agent('permuter')
reason = 'Current candidate families finished'
diff --git a/src/decompnet/app/deep_sweep.py b/src/decompnet/app/deep_sweep.py
index ee261802..30ccd9f4 100644
--- a/src/decompnet/app/deep_sweep.py
+++ b/src/decompnet/app/deep_sweep.py
@@ -4,7 +4,7 @@
import threading
import time
-from .. import candidate_scope
+from .. import candidate_scope, scoring
from ..network import NetworkError
from . import agent_tokens, deep_permuter
@@ -27,6 +27,10 @@ def __init__(self, net, jobs):
self.lock = threading.RLock()
with net.store.tx() as db:
db.execute(SCHEMA)
+ db.execute('''CREATE TABLE IF NOT EXISTS deep_permutation_improvements(
+ function_id INTEGER PRIMARY KEY, starting_score INTEGER NOT NULL,
+ starting_percent REAL, new_score INTEGER NOT NULL,
+ new_percent REAL, updated_at REAL NOT NULL)''')
def config(self, slug):
proj = self.net.project_row(slug)
@@ -50,6 +54,32 @@ def history(self, slug, fn):
result['result'] = json.loads(row['result'])
return result
+ def improvement(self, before, score):
+ """Keep only scores measured by this search, including wins before cancellation."""
+ start = before['best_score']
+ if score is None or start is None or score >= start:
+ return
+ def percent(value):
+ if value == 0:
+ return 100.0
+ if not before['n_insns']:
+ return None
+ return max(0.0, min(99.9 if value else 100.0,
+ 100.0 * (1 - value / max(1, before['n_insns'] * 100))))
+ with self.net.store.tx() as db:
+ db.execute('''INSERT INTO deep_permutation_improvements
+ (function_id,starting_score,starting_percent,new_score,new_percent,updated_at)
+ VALUES(?,?,?,?,?,?) ON CONFLICT(function_id) DO UPDATE SET
+ new_score=excluded.new_score,new_percent=excluded.new_percent,updated_at=excluded.updated_at
+ WHERE excluded.new_score < deep_permutation_improvements.new_score''',
+ (before['id'], start, percent(start), score, percent(score), time.time()))
+
+ def improvements(self, project_id):
+ rows = self.net.store.all('''SELECT i.*, f.name AS function FROM deep_permutation_improvements i
+ JOIN functions f ON f.id=i.function_id WHERE f.project_id=? ORDER BY i.updated_at DESC,f.id''', project_id)
+ return [dict(row) | {'starting_percent_text': scoring.percent_text(row['starting_percent']),
+ 'new_percent_text': scoring.percent_text(row['new_percent'])} for row in rows]
+
def status(self, slug):
proj = self.net.project_row(slug)
counts = self.net.store.one("SELECT COUNT(*) total, SUM(best_source IS NOT NULL AND best_score IS NOT NULL) ready "
@@ -57,10 +87,12 @@ def status(self, slug):
completed = self.net.store.one("SELECT COUNT(*) n FROM deep_permutation_passes d JOIN functions f ON f.id=d.function_id "
"WHERE f.project_id=? AND d.state='completed'", proj['id'])['n']
errors = self.net.store.one("SELECT COUNT(*) n FROM deep_permutation_passes d JOIN functions f ON f.id=d.function_id WHERE f.project_id=? AND d.state='error'", proj['id'])['n']
+ improvements = self.improvements(proj['id'])
with self.lock:
current = dict(self.active.get(slug, {}))
job = self.jobs.jobs.get(current.get('job'), {})
return self.config(slug) | {'total': counts['total'], 'ready': counts['ready'] or 0,
+ 'improved': len(improvements), 'improvements': improvements,
'completed': completed, 'errors': errors, 'state': job.get('state', 'idle'), **current,
'progress': job.get('progress', {}), 'error': job.get('error')}
@@ -113,6 +145,7 @@ def shutdown(self):
self.jobs.cancel(current['job'])
def completed(self, slug, before, minutes, variants, result, build_scope):
+ self.improvement(before, result.get('best_score'))
fresh = self.net._function(before['name'], slug)
proj = self.net.project_row(slug)
stable = build_scope == candidate_scope.key(self.net, proj, before)
diff --git a/src/decompnet/app/static/index.html b/src/decompnet/app/static/index.html
index 4f011f98..8bb05fed 100644
--- a/src/decompnet/app/static/index.html
+++ b/src/decompnet/app/static/index.html
@@ -394,6 +394,7 @@
ROM map
One function at a time. Busy functions and those without scored C wait for their agent or draft. Only the ordinary byte judge can save a match. A completed pass means its bounded turn finished, not that every possible rewrite was tried.
${status.improved || 0} functions improved by deep permutation
`
+ + (rows.length ? `
Function
Starting score
Starting match
New score
New match
${rows.map((r, i) => `
${esc(r.starting_score)}
${esc(r.starting_percent_text)}
${esc(r.new_score)}
${esc(r.new_percent_text)}
`).join("")}
Lower scores and higher match percentages are better. Saved across restarts and repeat passes; each function appears once, from its first recorded improvement to its best deep-pass result.