Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions concore_cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,21 @@ Stops all running concore processes. Prompts for confirmation before proceeding.
concore stop
```

### `concore clean <study_dir>`

Removes the leftovers of a run from a built study: each node's `concoreout.txt` and `concorepid`, and the data files inside the edge directories (the same files the generated `clear` script removes). Sources, `src/`, node scripts and any other directory are never touched. Refuses to run while a node is still running.

**Options:**
- `--dry-run` - List what would be removed, change nothing
- `-y, --yes` - Do not ask for confirmation
- `--force` - Clean even if a node looks like it is still running

**Example:**
```bash
concore clean out --dry-run
concore clean out -y
```

## Development Workflow

1. **Create a new project**
Expand Down
19 changes: 19 additions & 0 deletions concore_cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from .commands.stop import stop_all
from .commands.inspect import inspect_workflow
from .commands.watch import watch_study
from .commands.clean import clean_study
from .commands.doctor import doctor_check
from .commands.setup import setup_concore
from . import __version__
Expand Down Expand Up @@ -181,6 +182,24 @@ def watch(study_dir, interval, once):
sys.exit(1)


@cli.command()
@click.argument("study_dir", type=click.Path(exists=True))
@click.option("--dry-run", is_flag=True, help="List what would be removed")
@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation")
@click.option(
"--force", is_flag=True, help="Clean even if nodes look like they are still running"
)
def clean(study_dir, dry_run, yes, force):
"""Remove run leftovers (node output, pid files, edge data) from a study"""
try:
ok = clean_study(study_dir, dry_run, yes, force, console)
if not ok:
sys.exit(1)
except Exception as e:
console.print(f"[red]Error:[/red] {str(e)}")
sys.exit(1)


@cli.command()
def doctor():
"""Check system readiness for running concore studies"""
Expand Down
109 changes: 109 additions & 0 deletions concore_cli/commands/clean.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import re
from pathlib import Path

import click
import psutil

from .watch import _find_nodes

PORT_RE = re.compile(r"^(in|out)\d+$")
NODE_FILES = ("concoreout.txt", "concorepid")


def _edge_dirs(study_path, nodes):
"""Edge directories: what each node's in*/out* ports point at."""
node_names = {name for name, _ in nodes}
edges = {}
for _, node_path in nodes:
for port in node_path.iterdir():
if not PORT_RE.match(port.name):
continue
target = port.resolve()
if (
target.is_dir()
and target.parent == study_path
and target.name not in node_names
and target.name != "src"
):
edges[target.name] = target
return sorted(edges.values())


def _files_to_remove(study_path, nodes):
found = []
for _, node_path in nodes:
for name in NODE_FILES:
f = node_path / name
if f.is_file() and not f.is_symlink():
found.append(f)
for edge in _edge_dirs(study_path, nodes):
for f in sorted(edge.iterdir()):
if f.is_file() and not f.is_symlink():
found.append(f)
return found


def _running_nodes(nodes):
running = []
for name, node_path in nodes:
pidfile = node_path / "concorepid"
try:
pid = int(pidfile.read_text().strip())
except (OSError, ValueError):
continue
if psutil.pid_exists(pid):
try:
if psutil.Process(pid).status() == psutil.STATUS_ZOMBIE:
continue
except psutil.Error:
continue
running.append((name, pid))
return running


def clean_study(study_dir, dry_run, yes, force, console):
study_path = Path(study_dir).resolve()
if not study_path.is_dir():
console.print(f"[red]Error:[/red] '{study_dir}' is not a directory")
return False

nodes = _find_nodes(study_path)
if not nodes:
console.print(
"[red]Error:[/red] no nodes found. "
"Point to a built study directory (run build first)."
)
return False

running = _running_nodes(nodes)
if running and not force and not dry_run:
names = ", ".join(f"{n} (pid {p})" for n, p in running)
console.print(f"[red]Error:[/red] study is still running: {names}")
console.print("[dim]Stop it first (concore stop), or use --force.[/dim]")
return False

files = _files_to_remove(study_path, nodes)
if not files:
console.print("[green]Nothing to clean.[/green]")
return True

for f in files:
console.print(f.relative_to(study_path).as_posix(), markup=False)

if dry_run:
console.print(f"\n[cyan]Dry run:[/cyan] {len(files)} file(s) would be removed.")
return True

if not yes and not click.confirm(f"\nRemove {len(files)} file(s)?", default=False):
console.print("[yellow]Cancelled.[/yellow]")
return True

removed = 0
for f in files:
try:
f.unlink()
removed += 1
except OSError as e:
console.print(f"[red]Could not remove[/red] {f.name}: {e}", markup=True)
console.print(f"[green]Removed {removed} file(s).[/green]")
return removed == len(files)
114 changes: 114 additions & 0 deletions tests/test_clean_command.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import os

import pytest
from click.testing import CliRunner

from concore_cli.cli import cli

pytestmark = pytest.mark.skipif(
os.name == "nt", reason="study layout uses symlinks created by POSIX build"
)


def _study(tmp_path):
study = tmp_path / "study"
for name in ("CZ", "PZ", "CU", "PYM", "src"):
(study / name).mkdir(parents=True)
os.symlink("../CU", study / "CZ" / "out1")
os.symlink("../PYM", study / "CZ" / "in1")
os.symlink("../PYM", study / "PZ" / "out1")
os.symlink("../CU", study / "PZ" / "in1")
(study / "CZ" / "controller.py").write_text("code\n")
(study / "CZ" / "concoreout.txt").write_text("log\n")
(study / "PZ" / "concoreout.txt").write_text("log\n")
(study / "PZ" / "concorepid").write_text("99999999\n") # not a live pid
(study / "CU" / "u").write_text("[0.0, 1]\n")
(study / "PYM" / "ym").write_text("[0.0, 2]\n")
(study / "src" / "controller.py").write_text("src\n")
(study / "results").mkdir()
(study / "results" / "keep.csv").write_text("keep\n")
return study


def test_dry_run_lists_but_removes_nothing(tmp_path):
study = _study(tmp_path)
result = CliRunner().invoke(cli, ["clean", str(study), "--dry-run"])
assert result.exit_code == 0
assert "CZ/concoreout.txt" in result.output
assert "CU/u" in result.output
assert (study / "CU" / "u").exists()
assert (study / "CZ" / "concoreout.txt").exists()


def test_clean_removes_only_run_leftovers(tmp_path):
study = _study(tmp_path)
result = CliRunner().invoke(cli, ["clean", str(study), "-y"])
assert result.exit_code == 0
for gone in (
"CZ/concoreout.txt",
"PZ/concoreout.txt",
"PZ/concorepid",
"CU/u",
"PYM/ym",
):
assert not (study / gone).exists(), gone
for kept in (
"CZ/controller.py",
"src/controller.py",
"results/keep.csv",
):
assert (study / kept).exists(), kept
assert (study / "CZ" / "in1").is_symlink()
assert (study / "CU").is_dir()


def test_clean_asks_for_confirmation(tmp_path):
study = _study(tmp_path)
result = CliRunner().invoke(cli, ["clean", str(study)], input="n\n")
assert result.exit_code == 0
assert "Cancelled" in result.output
assert (study / "CU" / "u").exists()


def test_clean_refuses_when_node_is_running(tmp_path):
study = _study(tmp_path)
(study / "CZ" / "concorepid").write_text(f"{os.getpid()}\n")
result = CliRunner().invoke(cli, ["clean", str(study), "-y"])
assert result.exit_code == 1
assert "still running" in result.output
assert (study / "CU" / "u").exists()


def test_clean_force_overrides_running_check(tmp_path):
study = _study(tmp_path)
(study / "CZ" / "concorepid").write_text(f"{os.getpid()}\n")
result = CliRunner().invoke(cli, ["clean", str(study), "-y", "--force"])
assert result.exit_code == 0
assert not (study / "CU" / "u").exists()


def test_clean_nothing_to_do(tmp_path):
study = _study(tmp_path)
CliRunner().invoke(cli, ["clean", str(study), "-y"])
result = CliRunner().invoke(cli, ["clean", str(study), "-y"])
assert result.exit_code == 0
assert "Nothing to clean" in result.output


def test_clean_rejects_non_study_directory(tmp_path):
plain = tmp_path / "plain"
plain.mkdir()
result = CliRunner().invoke(cli, ["clean", str(plain), "-y"])
assert result.exit_code == 1
assert "no nodes found" in result.output


def test_clean_does_not_follow_symlink_out_of_study(tmp_path):
study = _study(tmp_path)
outside = tmp_path / "outside"
outside.mkdir()
(outside / "precious.txt").write_text("x\n")
os.symlink(str(outside), study / "CZ" / "out2")
result = CliRunner().invoke(cli, ["clean", str(study), "-y"])
assert result.exit_code == 0
assert (outside / "precious.txt").exists()
Loading