From d7a5b4d221d534070d6bd777cb043323dca6d7c7 Mon Sep 17 00:00:00 2001 From: AmirNCI <“x25188984@student.ncirl.ie”> Date: Tue, 29 Sep 2026 21:25:01 +0100 Subject: [PATCH] feat: add Factory Droid and Devin CLI agent adapters Adds two new agent adapters following the existing AgentAdapter pattern (build_command()/extract_report()/extract_usage()), registered in agents/registry.py and cli.py's AgentId enum: - droid.py: Factory CLI (`droid exec`). Authenticates via FACTORY_API_KEY (env var, so it can run headlessly in CI once configured). Defaults to read-only mode with no extra flag needed. Flag parsing (`--`, `--cwd`, `--output-format json`) was verified directly against the real CLI. - devin.py: Devin CLI (`devin -p`). Authenticates only via a one-time interactive `devin auth login` -- no env-var/API-key path exists, so unlike Droid it can't be authenticated from a bare CI runner without a human logging in first. Requires --respect-workspace-trust false, or non-interactive runs fail on any directory Devin hasn't seen before. 15 new unit tests (tests/test_agents/test_droid.py, test_devin.py) cover build_command()/extract_report()/extract_usage(), following the pattern in test_codex.py. Both adapters run cleanly through the full `secfoo run` pipeline (prompt rendered, real CLI invoked, clean status returned) but have NOT been verified against a live account -- no FACTORY_API_KEY or Devin login was available while writing this. See each file's docstring for exactly what was and wasn't verified. Testing: - pytest tests: 700 passed, 0 failed - ruff check on the new/changed files: clean (cli.py's pre-existing Optional[X]-style findings are unrelated to this change) Ref: SECFOO-6 Co-Authored-By: Claude Sonnet 5 --- src/secfoo/agents/devin.py | 95 ++++++++++++++++++++++++++++++++ src/secfoo/agents/droid.py | 97 +++++++++++++++++++++++++++++++++ src/secfoo/agents/registry.py | 4 ++ src/secfoo/cli.py | 2 + tests/test_agents/test_devin.py | 35 ++++++++++++ tests/test_agents/test_droid.py | 75 +++++++++++++++++++++++++ 6 files changed, 308 insertions(+) create mode 100644 src/secfoo/agents/devin.py create mode 100644 src/secfoo/agents/droid.py create mode 100644 tests/test_agents/test_devin.py create mode 100644 tests/test_agents/test_droid.py diff --git a/src/secfoo/agents/devin.py b/src/secfoo/agents/devin.py new file mode 100644 index 0000000..33b5f60 --- /dev/null +++ b/src/secfoo/agents/devin.py @@ -0,0 +1,95 @@ +"""Devin CLI adapter. + +Answers the open question the Q4 sprint plan flagged for this integration +("check whether Devin offers a true local CLI vs. only a cloud session +API"): as of Devin CLI 3000.11.3 (Sep 2026, official `devin-cli` Homebrew +cask -- *not* the unofficial `devin-cli`/`devin` packages that also exist +on PyPI under the same-sounding name, which are unrelated third-party +projects and were deliberately not used here), it is a real local-first +agent binary with its own non-interactive mode, so this is a normal +CLI adapter, not the API-based shape Initiative 2 uses. + +Flags verified against a real `devin --help` / `devin auth status` +(no Devin account was available while writing this, so every flag below +was confirmed by checking it parses through to devin's own runtime error -- +"Error: Login canceled" -- rather than a CLI parse error, the same +verification standard codex.py and droid.py use): + +- `-p `: non-interactive mode, prompt passed as the flag's own + value (same convention as claude.py's `-p`) -- verified directly. +- `--respect-workspace-trust false`: REQUIRED. Devin's own --help states + print/non-interactive mode "cannot show the trust prompt and fails in + an untrusted directory" unless this is passed; secfoo always runs + against a target Devin has never seen before, so omitting this would + make every single run fail on a fresh machine. The flag's presence and + value syntax were verified directly (parses through to the same + "Login canceled" runtime error); the trust-prompt behavior itself is + from Devin's documented help text, not independently reproduced here, + since no run got past the auth step. +- `--permission-mode auto`: passed explicitly even though Devin's own + --help lists "auto" as the default (auto-approves read-only tools) -- + same belt-and-suspenders reasoning as claude.py's explicit + `--permission-mode default`. NOT independently verified in combination + with the other two flags in this session (a three-flag combined probe + was blocked by this session's own tooling safety policy, which read the + combination as looking like an autonomous/"unsafe agent" configuration + attempt) -- re-verify this exact command once a real account is + available, before relying on it in CI. +- No `--cwd`-equivalent flag exists for Devin (checked the full --help + output -- there is none). Devin's print mode is assumed to operate on + the process's own working directory instead, which base.py already + sets via `subprocess.Popen(cwd=workdir)`. Not independently confirmed + end-to-end. +- No `--output-format`/JSON flag exists either (only `--export`, which + writes a separate file rather than emitting to stdout) -- so, like + codex.py, extract_report() is left at the default stdout-passthrough + and extract_usage() is not overridden (cost/tokens stay unknown, not + guessed). + +Auth: unlike codex/api/droid (env-var or API-key based, so a CI runner can +authenticate non-interactively from zero), Devin only supports +`devin auth login` -- a one-time interactive OAuth flow, or +`--force-manual-token-flow` to paste a token by hand. Once done, the +resulting `~/.local/share/devin/credentials.toml` is presumably reused by +later non-interactive `-p` calls, the same operational model claude/gemini +already use in secfoo (a human logs in once, outside of secfoo, before any +`secfoo run` invocation). This means Devin can never be authenticated from +a bare CI runner the way `--agent api`/`--agent droid` (with an env var +key) can -- worth flagging back to whoever owns Initiative 1's sequencing +note about CI testing. + +NOT yet verified end-to-end: no Devin account was available while writing +this. Every flag above parses correctly against the real CLI, and +`secfoo run --agent devin` completes the full pipeline and reports a clean +`failed` status rather than crashing (same shape confirmed for the droid +adapter) -- but no real report has been produced or inspected. Re-run +the adapter tests plus one real assessment once an account is available. +""" + +from __future__ import annotations + +from pathlib import Path + +from secfoo.agents.base import AgentAdapter + + +class DevinAdapter(AgentAdapter): + name = "devin" + binary = "devin" + default_timeout_seconds = 1800 + + def build_command(self, prompt: str, *, workdir: Path) -> list[str]: + return [ + self.binary, + "-p", + prompt, + # Required: non-interactive mode fails in a directory Devin + # hasn't seen before unless this is explicitly disabled (see + # module docstring). + "--respect-workspace-trust", + "false", + # Explicit even though it's the documented default -- a + # review-only skill never needs "accept-edits"/"dangerous". + "--permission-mode", + "auto", + ] diff --git a/src/secfoo/agents/droid.py b/src/secfoo/agents/droid.py new file mode 100644 index 0000000..b6fe50e --- /dev/null +++ b/src/secfoo/agents/droid.py @@ -0,0 +1,97 @@ +"""Factory Droid CLI adapter. + +Flags verified against a real `droid exec --help` (Factory CLI 0.227.0, +Sep 2026) and confirmed by probing until each one reached droid's own auth +check (no FACTORY_API_KEY/login available while writing this) rather than a +flag-parsing error: + +- `--` before the prompt is genuinely supported (verified: + `droid exec -- "--looks-like-a-flag"` reaches the "Authentication failed" + auth check, not an "unknown option" parse error) -- unlike + agents/cursor.py and agents/gemini.py, which pass the prompt to + `-p`/positional with no such separator. +- `--cwd` is accepted (belt and suspenders -- base.py's subprocess.Popen + already sets cwd=workdir; passing both keeps droid's own notion of the + project root in sync with the OS-level cwd). +- `--output-format json` wraps the answer in an envelope with `result` + (the final message) and `usage` (input_tokens/output_tokens plus a + `factory_credits` figure -- not a dollar amount, so `cost_usd` is left + unknown here rather than guessing a conversion rate). + +`droid exec` defaults to read-only mode (no file/system modification) with +no extra flag needed -- exactly what a review-only skill wants, so no +`--auto` flag is passed; `--auto`/`--skip-permissions-unsafe` would only +be relevant for a future skill that's allowed to write, which none of +secfoo's current skills are. + +Auth: droid needs either an interactive `/login` session or a +FACTORY_API_KEY environment variable. secfoo always invokes non- +interactively, so only FACTORY_API_KEY works here; if it's unset, every +run fails with droid's own "Authentication failed" message rather than +secfoo's cleaner `binary_not_found` status -- same tradeoff codex.py makes +(is_available() only checks the binary is on PATH, not that it's actually +authenticated). + +NOT yet verified end-to-end: no FACTORY_API_KEY was available while +writing this, so build_command()'s flags are confirmed by direct probing +against the real CLI (see above) but a full `secfoo run --agent droid` +against a live account has not been exercised. Re-run the adapter tests +plus one real assessment once a key is available, before relying on this +in CI. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from secfoo.agents.base import AgentAdapter, Usage + + +class DroidAdapter(AgentAdapter): + name = "droid" + binary = "droid" + default_timeout_seconds = 1800 + + def build_command(self, prompt: str, *, workdir: Path) -> list[str]: + return [ + self.binary, + "exec", + "--output-format", + "json", + "--cwd", + str(workdir), + # `--` so a prompt that happens to start with "-" is never + # parsed as a flag (verified directly against the real CLI -- + # see module docstring). + "--", + prompt, + ] + + def extract_report(self, stdout: str) -> str: + try: + payload = json.loads(stdout) + except json.JSONDecodeError: + return stdout + return payload.get("result", stdout) + + def extract_usage(self, stdout: str) -> Usage: + try: + payload = json.loads(stdout) + except json.JSONDecodeError: + return Usage() + if not isinstance(payload, dict): + return Usage() + usage = payload.get("usage") or {} + input_tokens = sum( + usage.get(key) or 0 + for key in ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens") + ) + return Usage( + input_tokens=input_tokens if usage else None, + output_tokens=usage.get("output_tokens"), + # factory_credits (also in the usage block) is not a dollar + # figure and we have no confirmed conversion rate -- leave + # cost_usd unknown rather than report a made-up number. + cost_usd=None, + ) diff --git a/src/secfoo/agents/registry.py b/src/secfoo/agents/registry.py index 79a2517..3d276b0 100644 --- a/src/secfoo/agents/registry.py +++ b/src/secfoo/agents/registry.py @@ -6,6 +6,8 @@ from secfoo.agents.claude import ClaudeAdapter from secfoo.agents.codex import CodexAdapter from secfoo.agents.cursor import CursorAdapter +from secfoo.agents.devin import DevinAdapter +from secfoo.agents.droid import DroidAdapter from secfoo.agents.gemini import GeminiAdapter from secfoo.agents.secfoo import SecFooAdapter @@ -17,6 +19,8 @@ ApiAdapter.name: ApiAdapter, SecFooAdapter.name: SecFooAdapter, CodexAdapter.name: CodexAdapter, + DroidAdapter.name: DroidAdapter, + DevinAdapter.name: DevinAdapter, } diff --git a/src/secfoo/cli.py b/src/secfoo/cli.py index 324f34a..8cc8165 100644 --- a/src/secfoo/cli.py +++ b/src/secfoo/cli.py @@ -98,6 +98,8 @@ class AgentId(str, Enum): API = "api" SECFOO = "secfoo" CODEX = "codex" + DROID = "droid" + DEVIN = "devin" class DepthId(str, Enum): diff --git a/tests/test_agents/test_devin.py b/tests/test_agents/test_devin.py new file mode 100644 index 0000000..7609e8f --- /dev/null +++ b/tests/test_agents/test_devin.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from secfoo.agents import ADAPTERS, get_adapter +from secfoo.agents.devin import DevinAdapter + + +def test_registered_under_devin_id(): + assert ADAPTERS["devin"] is DevinAdapter + assert isinstance(get_adapter("devin"), DevinAdapter) + + +def test_build_command_is_non_interactive_print_mode(tmp_path): + cmd = DevinAdapter().build_command("hello", workdir=tmp_path) + assert cmd[0] == "devin" + assert cmd[cmd.index("-p") + 1] == "hello" + + +def test_build_command_disables_workspace_trust_check(tmp_path): + # Required: devin's own docs say non-interactive mode fails in a + # directory it hasn't seen before unless this is explicitly disabled. + cmd = DevinAdapter().build_command("hello", workdir=tmp_path) + assert cmd[cmd.index("--respect-workspace-trust") + 1] == "false" + + +def test_build_command_does_not_grant_edit_or_dangerous_permissions(tmp_path): + cmd = DevinAdapter().build_command("hello", workdir=tmp_path) + assert cmd[cmd.index("--permission-mode") + 1] == "auto" + assert "accept-edits" not in cmd + assert "dangerous" not in cmd + + +def test_extract_report_is_stdout_passthrough(): + # No JSON/output-format flag exists for devin (see module docstring), + # so the default base-class behavior (stdout is the report) applies. + assert DevinAdapter().extract_report("final report text") == "final report text" diff --git a/tests/test_agents/test_droid.py b/tests/test_agents/test_droid.py new file mode 100644 index 0000000..83a6729 --- /dev/null +++ b/tests/test_agents/test_droid.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import json + +from secfoo.agents import ADAPTERS, get_adapter +from secfoo.agents.droid import DroidAdapter + + +def test_registered_under_droid_id(): + assert ADAPTERS["droid"] is DroidAdapter + assert isinstance(get_adapter("droid"), DroidAdapter) + + +def test_build_command_is_non_interactive_exec(tmp_path): + cmd = DroidAdapter().build_command("hello", workdir=tmp_path) + assert cmd[:2] == ["droid", "exec"] + + +def test_build_command_uses_json_output(tmp_path): + cmd = DroidAdapter().build_command("hello", workdir=tmp_path) + assert cmd[cmd.index("--output-format") + 1] == "json" + + +def test_build_command_sets_working_root(tmp_path): + cmd = DroidAdapter().build_command("hello", workdir=tmp_path) + assert cmd[cmd.index("--cwd") + 1] == str(tmp_path) + + +def test_build_command_does_not_grant_write_access(tmp_path): + # `droid exec` defaults to read-only mode with no flag -- a review-only + # skill must never pass --auto or --skip-permissions-unsafe. + cmd = DroidAdapter().build_command("hello", workdir=tmp_path) + assert "--auto" not in cmd + assert "--skip-permissions-unsafe" not in cmd + + +def test_prompt_is_last_arg_after_double_dash(tmp_path): + cmd = DroidAdapter().build_command("--looks-like-a-flag", workdir=tmp_path) + assert cmd[-2:] == ["--", "--looks-like-a-flag"] + + +def test_extract_report_pulls_result_field(): + stdout = json.dumps({"result": "final report text", "usage": {}}) + assert DroidAdapter().extract_report(stdout) == "final report text" + + +def test_extract_report_falls_back_to_raw_stdout_on_bad_json(): + assert DroidAdapter().extract_report("not json") == "not json" + + +def test_extract_usage_sums_token_fields(): + stdout = json.dumps( + { + "result": "ok", + "usage": { + "input_tokens": 100, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 5, + "output_tokens": 50, + "factory_credits": 3, + }, + } + ) + usage = DroidAdapter().extract_usage(stdout) + assert usage.input_tokens == 115 + assert usage.output_tokens == 50 + # factory_credits is not a dollar figure -- must not be reported as cost. + assert usage.cost_usd is None + + +def test_extract_usage_unknown_on_bad_json(): + usage = DroidAdapter().extract_usage("not json") + assert usage.input_tokens is None + assert usage.output_tokens is None + assert usage.cost_usd is None