Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions concore_cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ Compiles a concore workflow GraphML file into executable scripts (POSIX, Windows
concore build workflow.graphml --source ./src --output ./build --auto-build
```

Build stops before generating anything if a node's source file (the part after `:` in its label, such as `controller.py`) is missing from the source directory, and lists the missing files. Labels with no extension (pulled containers) and port-only nodes (`IN:`) are not checked.

Docker compose example:

```bash
Expand Down
33 changes: 33 additions & 0 deletions concore_cli/commands/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import sys
import shutil
from pathlib import Path
from bs4 import BeautifulSoup
from rich.panel import Panel
from rich.progress import Progress, SpinnerColumn, TextColumn

Expand All @@ -19,6 +20,32 @@ def _find_mkconcore_path():
return None


def _missing_sources(workflow_path, source_path):
"""Source files named by node labels ('ID:file.ext') that are not in source_path.

Labels without an extension (pulled containers) and labels without a file
part (port-only nodes such as 'IN:') have nothing to check.
"""
try:
soup = BeautifulSoup(workflow_path.read_text(encoding="utf-8"), "xml")
except (OSError, UnicodeDecodeError):
return []
missing = set()
for node in soup.find_all("node"):
tag = node.find("y:NodeLabel")
if tag is None:
tag = node.find("NodeLabel")
if tag is None or not tag.text:
continue
parts = tag.text.strip().split(":")
if len(parts) != 2:
continue
name = parts[1]
if "." in name and not (source_path / name).exists():
missing.add(name)
return sorted(missing)


def _yaml_quote(value):
return "'" + value.replace("'", "''") + "'"

Expand Down Expand Up @@ -187,6 +214,12 @@ def build_workflow(
if compose and exec_type != "docker":
raise ValueError("--compose can only be used with --type docker")

missing = _missing_sources(workflow_path, source_path)
if missing:
raise FileNotFoundError(
f"Node source file(s) not found in '{source_path}': {', '.join(missing)}"
)

mkconcore_path = _find_mkconcore_path()
if mkconcore_path is None:
raise FileNotFoundError(
Expand Down
59 changes: 59 additions & 0 deletions tests/test_build_missing_sources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
from click.testing import CliRunner

from concore_cli.cli import cli
from concore_cli.commands.build import _missing_sources

GRAPH = """<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:y="http://www.yworks.com/xml/graphml">
<graph id="G" edgedefault="directed">
{nodes}
</graph>
</graphml>
"""


def _node(node_id, label):
return (
f'<node id="{node_id}"><data key="d0">'
f"<y:NodeLabel>{label}</y:NodeLabel></data></node>"
)


def _write_graph(tmp_path, *labels):
nodes = "\n".join(_node(f"n{i}", label) for i, label in enumerate(labels))
path = tmp_path / "wf.graphml"
path.write_text(GRAPH.format(nodes=nodes), encoding="utf-8")
return path


def test_missing_sources_lists_absent_files(tmp_path):
src = tmp_path / "src"
src.mkdir()
(src / "controller.py").write_text("x\n")
wf = _write_graph(tmp_path, "CZ:controller.py", "PZ:pm.py", "OZ:pm.py", "A:b.m")
assert _missing_sources(wf, src) == ["b.m", "pm.py"]


def test_missing_sources_ignores_pulled_containers_and_port_nodes(tmp_path):
src = tmp_path / "src"
src.mkdir()
wf = _write_graph(tmp_path, "CZ:cpymat", "IN:", "OUT:")
assert _missing_sources(wf, src) == []


def test_missing_sources_tolerates_unreadable_workflow(tmp_path):
assert _missing_sources(tmp_path / "nope.graphml", tmp_path) == []


def test_build_fails_early_when_source_file_is_missing(tmp_path):
src = tmp_path / "src"
src.mkdir()
(src / "controller.py").write_text("x\n")
wf = _write_graph(tmp_path, "CZ:controller.py", "PZ:pm.py")
out = tmp_path / "out"
result = CliRunner().invoke(
cli, ["build", str(wf), "--source", str(src), "--output", str(out)]
)
assert result.exit_code == 1
assert "pm.py" in result.output
assert not out.exists()
Loading