From f9e770d87f95dd8cff6d6a845563c77c18897e48 Mon Sep 17 00:00:00 2001 From: yadavchiragg Date: Sat, 14 Nov 2026 08:36:43 +0530 Subject: [PATCH] feat(cli): add concore run to start a built study --- README.md | 3 + concore_cli/README.md | 24 ++++ concore_cli/cli.py | 28 +++++ concore_cli/commands/run.py | 184 ++++++++++++++++++++++++++++++ tests/test_run_command.py | 220 ++++++++++++++++++++++++++++++++++++ 5 files changed, 459 insertions(+) create mode 100644 concore_cli/commands/run.py create mode 100644 tests/test_run_command.py diff --git a/README.md b/README.md index 1d59a862..04a60e10 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ concore validate workflow.graphml # Compile your workflow concore build workflow.graphml --auto-build +# Start the built study +concore run out --wait + # Monitor running processes concore status diff --git a/concore_cli/README.md b/concore_cli/README.md index 78f98b1a..7dcf808e 100644 --- a/concore_cli/README.md +++ b/concore_cli/README.md @@ -23,6 +23,9 @@ concore validate workflow.graphml # Build your workflow concore build workflow.graphml +# Start the built study and wait for it to finish +concore run out --wait + # Check running processes concore status @@ -103,6 +106,27 @@ concore validate workflow.graphml concore validate workflow.graphml --source ./src ``` +### `concore run ` + +Starts a built study (the folder `concore build` created) by running its generated `run` script, so you do not have to `cd` into it. + +**Options:** +- `--build` - Run the study's `build` script first (stops if the build fails) +- `--wait` - Wait until every node has exited. `Ctrl+C` stops the study's nodes +- `--timeout ` - With `--wait`, stop the study after this many seconds (exit code 124) + +**Examples:** +```bash +concore run out +concore run out --build --wait +concore run out --wait --timeout 300 +``` + +**Notes:** +- `--wait` needs the node process ids that POSIX and macOS builds record; for Docker and Windows builds the study is started and a warning explains there is nothing to wait for. +- `--wait` reports when the nodes have exited, not whether they succeeded. Check each node's `concoreout.txt` for errors. +- Exit codes: `0` success, `124` timeout, `130` interrupted, or the exit code of the failing script. + ### `concore status` Shows all currently running concore processes with details: diff --git a/concore_cli/cli.py b/concore_cli/cli.py index dc8fca34..58661c75 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -8,6 +8,7 @@ from .commands.validate import validate_workflow from .commands.status import show_status from .commands.stop import stop_all +from .commands.run import run_study from .commands.inspect import inspect_workflow from .commands.watch import watch_study from .commands.doctor import doctor_check @@ -152,6 +153,33 @@ def status(): sys.exit(1) +@cli.command() +@click.argument("study_dir", type=click.Path(exists=True)) +@click.option( + "--build", "build_first", is_flag=True, help="Run the study's build script first" +) +@click.option( + "--wait", is_flag=True, help="Wait until every node exits (Ctrl+C stops the study)" +) +@click.option( + "--timeout", + type=click.FloatRange(min=0, min_open=True), + default=None, + help="With --wait: stop the study after this many seconds", +) +def run(study_dir, build_first, wait, timeout): + """Start a built study (the folder created by concore build)""" + try: + code = run_study( + study_dir, console, build=build_first, wait=wait, timeout=timeout + ) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + if code: + sys.exit(code) + + @cli.command() @click.confirmation_option(prompt="Stop all running concore processes?") def stop(): diff --git a/concore_cli/commands/run.py b/concore_cli/commands/run.py new file mode 100644 index 00000000..ea32e218 --- /dev/null +++ b/concore_cli/commands/run.py @@ -0,0 +1,184 @@ +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +import psutil + +EXIT_TIMEOUT = 124 +EXIT_INTERRUPTED = 130 +PID_WAIT_SECONDS = 10.0 +POLL_SECONDS = 0.5 + + +def _find_script(study_path, name): + """Return the generated `` script for this platform, or None.""" + if sys.platform == "win32": + candidate = study_path / f"{name}.bat" + if candidate.is_file(): + return candidate + candidate = study_path / name + if candidate.is_file(): + return candidate + return None + + +def _command(script): + if script.suffix.lower() == ".bat": + return ["cmd", "/c", str(script)] + if os.access(script, os.X_OK): + return [str(script)] + return ["bash", str(script)] + + +def _node_dirs(run_script): + """Node folders started by a POSIX run script: lines like `(cd CZ; ... &`.""" + try: + text = run_script.read_text() + except OSError: + return [] + dirs = [] + for match in re.finditer(r"\(cd\s+([^;\s]+)\s*;", text): + folder = match.group(1).strip("'\"") + if folder not in dirs: + dirs.append(folder) + return dirs + + +def _read_pid(pid_file, newer_than): + """Read a pid file written by this run (older files are stale).""" + try: + if pid_file.stat().st_mtime < newer_than: + return None + return int(pid_file.read_text().strip()) + except (OSError, ValueError): + return None + + +def _alive(pid): + try: + return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE + except psutil.NoSuchProcess: + return False + except psutil.AccessDenied: + return True + + +def _collect_pids(study_path, node_dirs, started, console): + """Wait briefly for each node's pid file, then return {folder: pid}.""" + pids = {} + deadline = time.monotonic() + PID_WAIT_SECONDS + while True: + for folder in node_dirs: + if folder not in pids: + pid = _read_pid(study_path / folder / "concorepid", started) + if pid is not None: + pids[folder] = pid + if len(pids) == len(node_dirs) or time.monotonic() > deadline: + break + time.sleep(0.2) + for folder in node_dirs: + if folder not in pids: + console.print( + f"[yellow]Warning:[/yellow] no process id found for node '{folder}'" + ) + return pids + + +def _stop_study(study_path, pids): + """Stop one study: run its own stop script, then make sure its pids are gone.""" + stop_script = _find_script(study_path, "stop") + if stop_script is not None: + subprocess.run( + _command(stop_script), + cwd=study_path, + capture_output=True, + text=True, + ) + for pid in pids.values(): + if _alive(pid): + try: + psutil.Process(pid).kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + +def run_study(study_dir, console, build=False, wait=False, timeout=None): + """Start a built study. Returns the process exit code for the command.""" + study_path = Path(study_dir).resolve() + if not study_path.is_dir(): + console.print(f"[red]Error:[/red] '{study_dir}' is not a directory") + return 1 + if timeout is not None and not wait: + console.print("[red]Error:[/red] --timeout requires --wait") + return 1 + + run_script = _find_script(study_path, "run") + if run_script is None: + console.print( + f"[red]Error:[/red] No run script found in {study_path}\n" + "[dim]Build the workflow first: concore build " + "--output [/dim]" + ) + return 1 + + if build: + build_script = _find_script(study_path, "build") + if build_script is None: + console.print(f"[red]Error:[/red] No build script found in {study_path}") + return 1 + console.print("[cyan]Building study...[/cyan]") + build_result = subprocess.run(_command(build_script), cwd=study_path) + if build_result.returncode != 0: + console.print( + f"[red]Build failed[/red] (exit code {build_result.returncode})" + ) + return build_result.returncode + + node_dirs = _node_dirs(run_script) + console.print(f"[cyan]Starting study:[/cyan] {study_path}") + started = time.time() - 1 + run_result = subprocess.run(_command(run_script), cwd=study_path) + if run_result.returncode != 0: + console.print( + f"[red]Run script failed[/red] (exit code {run_result.returncode})" + ) + return run_result.returncode + + if not wait: + console.print( + "[green]✓[/green] Study started\n" + "[dim]Monitor it with: concore watch " + "Stop it with: concore stop[/dim]" + ) + return 0 + + if not node_dirs: + console.print( + "[yellow]Warning:[/yellow] --wait needs a study that records node " + "process ids (a posix/macOS build). The study was started, but " + "there is nothing to wait for." + ) + return 0 + + pids = _collect_pids(study_path, node_dirs, started, console) + console.print("[dim]Waiting for the study to finish (Ctrl+C to stop it)...[/dim]") + start = time.monotonic() + try: + while any(_alive(pid) for pid in pids.values()): + if timeout is not None and time.monotonic() - start > timeout: + console.print( + f"[yellow]Timeout:[/yellow] study still running after {timeout}s, stopping it" + ) + _stop_study(study_path, pids) + return EXIT_TIMEOUT + time.sleep(POLL_SECONDS) + except KeyboardInterrupt: + console.print("\n[yellow]Interrupted:[/yellow] stopping the study...") + _stop_study(study_path, pids) + return EXIT_INTERRUPTED + + console.print("[green]✓[/green] Study finished") + return 0 diff --git a/tests/test_run_command.py b/tests/test_run_command.py new file mode 100644 index 00000000..c9b0a760 --- /dev/null +++ b/tests/test_run_command.py @@ -0,0 +1,220 @@ +"""Tests for `concore run` (concore_cli/commands/run.py).""" + +import os +import subprocess +import sys +import time + +import psutil +import pytest +from click.testing import CliRunner + +from concore_cli.cli import cli +from concore_cli.commands import run as run_mod +from concore_cli.commands.run import ( + EXIT_INTERRUPTED, + EXIT_TIMEOUT, + _node_dirs, + run_study, +) + +pytestmark = pytest.mark.skipif( + sys.platform == "win32", reason="uses generated POSIX shell scripts" +) + + +class FakeConsole: + def __init__(self): + self.lines = [] + + def print(self, *args, **kwargs): + self.lines.append(" ".join(str(a) for a in args)) + + @property + def text(self): + return "\n".join(self.lines) + + +def _write_script(path, body): + path.write_text("#!/bin/bash\n" + body) + path.chmod(0o700) + + +def _make_study(tmp_path, seconds="0.5", nodes=("A", "B")): + """A tiny study shaped like mkconcore's POSIX output.""" + study = tmp_path / "study" + study.mkdir() + run_lines = "" + stop_lines = "" + for node in nodes: + (study / node).mkdir() + run_lines += f"(cd {node}; sleep {seconds} & echo $! >concorepid) &\n" + stop_lines += f"kill -9 `cat {node}/concorepid`\nrm {node}/concorepid\n" + _write_script(study / "run", run_lines) + _write_script(study / "stop", stop_lines) + return study + + +def _pids(study, nodes=("A", "B")): + return [int((study / n / "concorepid").read_text().strip()) for n in nodes] + + +def test_node_dirs_parses_generated_run_script(tmp_path): + script = tmp_path / "run" + script.write_text( + "#!/bin/bash\n" + "(cd CZ; python3 controller.py >concoreout.txt & echo $! >concorepid) &\n" + "(cd PZ; python3 pm.py >concoreout.txt & echo $! >concorepid) &\n" + "(cd CZ; echo again) &\n" + ) + assert _node_dirs(script) == ["CZ", "PZ"] + + +def test_node_dirs_empty_for_docker_style_script(tmp_path): + script = tmp_path / "run" + script.write_text("docker run --name=CZ -v CU:/out1 docker-controller&\n") + assert _node_dirs(script) == [] + + +def test_missing_run_script_fails(tmp_path): + console = FakeConsole() + assert run_study(tmp_path, console) == 1 + assert "No run script" in console.text + + +def test_timeout_requires_wait(tmp_path): + study = _make_study(tmp_path) + console = FakeConsole() + assert run_study(study, console, timeout=5) == 1 + assert "--timeout requires --wait" in console.text + + +def test_start_without_wait_returns_immediately(tmp_path): + study = _make_study(tmp_path, seconds="30") + console = FakeConsole() + began = time.monotonic() + try: + assert run_study(study, console) == 0 + assert time.monotonic() - began < 5 + time.sleep(0.5) + assert all(psutil.pid_exists(pid) for pid in _pids(study)) + finally: + subprocess.run([str(study / "stop")], cwd=study, capture_output=True) + + +def test_wait_returns_zero_when_nodes_finish(tmp_path): + study = _make_study(tmp_path, seconds="1") + console = FakeConsole() + assert run_study(study, console, wait=True) == 0 + assert "Study finished" in console.text + + +def test_wait_timeout_stops_the_study(tmp_path): + study = _make_study(tmp_path, seconds="60") + console = FakeConsole() + began = time.monotonic() + code = run_study(study, console, wait=True, timeout=1) + assert code == EXIT_TIMEOUT + assert time.monotonic() - began < 30 + assert "Timeout" in console.text + time.sleep(0.5) + # the stop script removed the pid files and the nodes are gone + assert not (study / "A" / "concorepid").exists() + + +def test_interrupt_stops_the_study(tmp_path, monkeypatch): + study = _make_study(tmp_path, seconds="60") + console = FakeConsole() + calls = {"n": 0} + real_sleep = time.sleep + + def interrupting_sleep(seconds): + calls["n"] += 1 + if calls["n"] > 3: + raise KeyboardInterrupt + real_sleep(0.05) + + monkeypatch.setattr(run_mod.time, "sleep", interrupting_sleep) + assert run_study(study, console, wait=True) == EXIT_INTERRUPTED + assert "Interrupted" in console.text + assert not (study / "A" / "concorepid").exists() + + +def test_run_script_failure_exit_code_is_returned(tmp_path): + study = tmp_path / "study" + study.mkdir() + _write_script(study / "run", "exit 3\n") + console = FakeConsole() + assert run_study(study, console) == 3 + assert "Run script failed" in console.text + + +def test_build_runs_before_run(tmp_path): + study = _make_study(tmp_path, seconds="0.2") + _write_script(study / "build", "touch built\n") + console = FakeConsole() + assert run_study(study, console, build=True, wait=True) == 0 + assert (study / "built").exists() + + +def test_failed_build_stops_before_running(tmp_path): + study = tmp_path / "study" + study.mkdir() + _write_script(study / "build", "exit 4\n") + _write_script(study / "run", "touch ran\n") + console = FakeConsole() + assert run_study(study, console, build=True) == 4 + assert not (study / "ran").exists() + assert "Build failed" in console.text + + +def test_wait_ignores_stale_pid_files(tmp_path, monkeypatch): + monkeypatch.setattr(run_mod, "PID_WAIT_SECONDS", 0.5) + study = tmp_path / "study" + (study / "A").mkdir(parents=True) + # a pid file left by an earlier run, pointing at a live process + pid_file = study / "A" / "concorepid" + pid_file.write_text(str(os.getpid())) + old = time.time() - 3600 + os.utime(pid_file, (old, old)) + # this run's script never writes a pid file for A + _write_script(study / "run", "(cd A; true) &\n") + console = FakeConsole() + began = time.monotonic() + assert run_study(study, console, wait=True) == 0 + assert time.monotonic() - began < 10 + assert "no process id found for node 'A'" in console.text + + +def test_wait_on_study_without_pid_files_only_warns(tmp_path): + study = tmp_path / "study" + study.mkdir() + _write_script(study / "run", "true\n") + console = FakeConsole() + assert run_study(study, console, wait=True) == 0 + assert "nothing to wait for" in console.text + + +def test_cli_run_starts_a_study(tmp_path): + study = _make_study(tmp_path, seconds="0.3") + result = CliRunner().invoke(cli, ["run", str(study), "--wait"]) + assert result.exit_code == 0, result.output + + +def test_cli_run_propagates_exit_code(tmp_path): + study = tmp_path / "study" + study.mkdir() + _write_script(study / "run", "exit 3\n") + result = CliRunner().invoke(cli, ["run", str(study)]) + assert result.exit_code == 3 + + +def test_cli_run_rejects_missing_directory(tmp_path): + result = CliRunner().invoke(cli, ["run", str(tmp_path / "nope")]) + assert result.exit_code != 0 + + +def test_cli_run_rejects_non_positive_timeout(tmp_path): + study = _make_study(tmp_path) + result = CliRunner().invoke(cli, ["run", str(study), "--wait", "--timeout", "0"]) + assert result.exit_code != 0