Self learning design

AutoCua/data_extraction.py

@@ -4,17 +4,25 @@
 its clean frame, its element mapping), so nothing here knows which OS it runs
 on. Wired into the mac driver today.
 
-With ENABLED on, every run appends one episode to AutoCua_data/dataset/. Runs
-only ever add to it, so the dataset grows with each run and nothing already
-recorded is touched:
+With ENABLED on, every run records into AutoCua_data/dataset/. Runs only ever
+add to it, so the dataset grows with each run and nothing already recorded is
+touched:
 
     dataset/
-        index.jsonl                 one line per finished run
+        index.jsonl                 one line per finished run (an episode with
+                                    several requests repeats; its LAST line is current)
         episodes/<episode>/
-            episode.json            task, model, screen, outcome
+            episode.json            task, model, screen, outcome, and one entry per request
             steps.jsonl             one line per step, written as the run goes
             frames/step_NNN.png     the CLEAN screenshot the step was decided on
 
+An episode is one continuous trajectory. A main.py run is one episode. A
+desktop-app chat is ONE episode too: every task asked in that chat APPENDS to
+the same folder (steps and frames keep counting, each step carries its request
+number, the header lists every request with its task text and first step), so
+the record has the same shape as the history the model sees at run time - the
+earlier steps, then a new request, then the steps that answer it.
+
 A step keeps both readings of one decision: what the teacher saw (the element
 tree and its ids) and where each action really landed (the controller's own
 click point, in screen points and on the 0-1000 grid of the frame). So one step
@@ -32,6 +40,38 @@
 # Record every run into the dataset. False: runs behave exactly as before.
 ENABLED = False
 
+# The desktop app's chat this run belongs to (its chat_<id>), set by the app
+# around each computer-use run; None for main.py runs. The chat IS the episode:
+# its folder is episodes/<chat_id>, and every later task in the chat reopens
+# it and appends (see Episode).
+chat_id = None
+
+# Tools the agent does WITHOUT while recording. A recorded run is a lesson in
+# GUI work, so a tool whose work leaves no trace on screen is not sent at all
+# rather than sent and ignored - the model only ever sees tools it may use:
+#   open_app   an app opened by name teaches nothing; let it be opened the way
+#              a person does it, on screen.
+#   sub_agent / agent_wait   a sub-agent (coder, browser, iPhone) works off
+#              this screen, so no frame explains what happened. With these
+#              gone no sub-agent can start, and <sub_agents> says offline.
+#   web        an answer fetched by another model, nothing on screen.
+#   applescript  drives an app from the outside; the same work must be
+#              done on screen instead. (mac only; harmless elsewhere.)
+# shell is kept on purpose (owner decision, 2026-10-11).
+# Recording off: every tool is sent, exactly as before.
+OFF_TOOLS = frozenset(("open_app", "sub_agent", "agent_wait", "web", "applescript"))
+
+
+def live_tools(tools):
+    """The main driver's registry minus OFF_TOOLS while recording.
+
+    Each platform's tool_registry calls this once, where its MAIN_TOOLS are
+    built, so the tools the model is GIVEN and the names a call is CHECKED
+    against (MAIN_TOOL_NAMES*, derived from the same lists) can never
+    disagree: a withheld tool is simply not a tool this run."""
+    return tools if not ENABLED else [t for t in tools if t.get("name") not in OFF_TOOLS]
+
+
 # Tools that act at one element's point. drag_drop acts at two ("8 to 15").
 _POINTER_TOOLS = ("left_click", "right_click", "input", "scroll")
 
@@ -53,24 +93,44 @@ def begin(task, platform, provider, model, speed, screen, original_task=None, re
 
 
 class Episode:
-    """One run of the agent: a header, then its steps as they happen."""
+    """One trajectory: a header, then its steps as they happen.
+
+    A chat's episode is reopened by every later request of that chat: the
+    step and frame numbers carry on from what is already on disk (counted
+    from the files, so a run that crashed before finish() cannot make the next
+    one overwrite anything), and the header's `requests` list gets one more
+    entry. A chat whose folder does not exist yet (recording switched on
+    mid-chat, or the dataset cleared) starts a fresh episode with
+    `resumed: true`, so the converter can see its history is cut off."""
 
     def __init__(self, task, platform, provider, model, speed, screen,
                  original_task=None, resumed=False):
-        self.id = datetime.now().strftime("%Y-%m-%d_%H-%M-%S_") + secrets.token_hex(3)
         self.root = data_root() / "dataset"
+        self.id = chat_id or datetime.now().strftime("%Y-%m-%d_%H-%M-%S_") + secrets.token_hex(3)
         self.dir = self.root / "episodes" / self.id
-        (self.dir / "frames").mkdir(parents=True, exist_ok=True)
-        self.steps = 0
-        self.frames = 0
-        self.header = {
-            "episode": self.id, "task": task,
-            "original_task": original_task or task, "resumed": bool(resumed),
-            "platform": platform, "provider": provider, "model": model, "speed": speed,
-            "screen": screen, "started_at": _now(),
-        }
-        _write_json(self.dir / "episode.json", self.header)
-        print(f"Dataset: recording episode {self.id}")
+        header_path = self.dir / "episode.json"
+        self.header = _read_json(header_path) if chat_id and header_path.exists() else None
+        if self.header is None:
+            (self.dir / "frames").mkdir(parents=True, exist_ok=True)
+            self.header = {
+                "episode": self.id, "task": task,
+                "original_task": original_task or task, "resumed": bool(resumed), "chat": chat_id,
+                "platform": platform, "provider": provider, "model": model, "speed": speed,
+                "screen": screen, "started_at": _now(), "requests": [],
+            }
+        steps_path = self.dir / "steps.jsonl"
+        self.steps = sum(1 for _ in steps_path.open("rb")) if steps_path.exists() else 0
+        self.frames = sum(1 for p in (self.dir / "frames").iterdir() if p.suffix == ".png")
+        self._base = self.steps          # this request's steps count on from here
+        self.request = len(self.header["requests"]) + 1
+        self.header["requests"].append({
+            "no": self.request, "task": task, "resumed": bool(resumed),
+            "first_step": self.steps + 1, "provider": provider, "model": model,
+            "speed": speed, "started_at": _now(),
+        })
+        _write_json(header_path, self.header)
+        print(f"Dataset: recording episode {self.id}"
+              + (f", request {self.request} (from step {self.steps + 1})" if self.request > 1 else ""))
 
     def add_step(self, step, screen, frame_image, app, element_tree, elements, clicks,
                  reasoning, text, tool_calls, actions, result, context=None):
@@ -87,6 +147,9 @@ def add_step(self, step, screen, frame_image, app, element_tree, elements, click
                      (e.g. todo, scratchpad, skills), as it was BEFORE the
                      step's actions ran."""
         try:
+            # The driver counts steps from 1 in every run; the episode counts
+            # on from the requests before this one.
+            step = self._base + int(step)
             frame = None
             if frame_image is not None:
                 frame = f"frames/step_{step:03d}.png"
@@ -96,6 +159,7 @@ def add_step(self, step, screen, frame_image, app, element_tree, elements, click
                 self.frames += 1
             _append(self.dir / "steps.jsonl", {
                 "step": step,
+                "request": self.request,
                 "time": _now(),
                 "frame": frame,
                 "screen": screen,
@@ -114,15 +178,22 @@ def add_step(self, step, screen, frame_image, app, element_tree, elements, click
             print(f"Dataset: step {step} not recorded ({e})")
 
     def finish(self, status, message):
-        """Close the episode and add it to the dataset index."""
+        """Close this request: outcome on its entry and on the episode, totals
+        refreshed, one more index line (the last line of an episode is current)."""
         try:
-            self.header.update({"finished_at": _now(), "status": status, "message": message,
+            now = _now()
+            self.header["requests"][-1].update({"finished_at": now, "status": status,
+                                                "message": message, "steps": self.steps - self._base})
+            self.header.update({"finished_at": now, "status": status, "message": message,
                                 "steps": self.steps, "frames": self.frames})
             _write_json(self.dir / "episode.json", self.header)
-            _append(self.root / "index.jsonl", {k: self.header[k] for k in (
-                "episode", "task", "original_task", "resumed", "platform", "model",
-                "status", "steps", "frames", "started_at", "finished_at")})
-            print(f"Dataset: episode {self.id} saved ({self.steps} steps, {self.frames} frames)")
+            line = {k: self.header[k] for k in (
+                "episode", "task", "original_task", "resumed", "chat", "platform", "model",
+                "status", "steps", "frames", "started_at", "finished_at")}
+            line["requests"] = len(self.header["requests"])
+            _append(self.root / "index.jsonl", line)
+            print(f"Dataset: episode {self.id} saved ({self.steps} steps, {self.frames} frames, "
+                  f"{len(self.header['requests'])} request(s))")
         except Exception as e:
             print(f"Dataset: episode {self.id} not closed ({e})")
 
@@ -188,6 +259,10 @@ def _now():
     return datetime.now().isoformat(timespec="seconds")
 
 
+def _read_json(path):
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
 def _write_json(path, obj):
     # Path writes, not open(): compiled builds patch builtins.open (see
     # AutoCua/__init__.py). Written aside and swapped in, never half-written.