diff --git a/README.md b/README.md index 6a3aaf5..86ec7f8 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,34 @@ You need an agent CLI already installed and authenticated (`--agent claude|agent|agy|gemini|codex`), or an API key with the built-in `api` agent — see [Install](#install). +## Repository targets + +`--target` accepts local directories, GitHub URLs, GitLab (including nested +namespaces), and Bitbucket Cloud repositories: + +```bash +secfoo run --skill sast --target https://gitlab.com/group/subgroup/repo --agent claude +secfoo run --skill sast --target https://bitbucket.org/workspace/repo --agent claude +``` + +GitLab and Bitbucket support HTTPS and `git@host:namespace/repo.git` SSH URLs; +`.git` is optional. SSH uses your existing SSH authentication. For self-hosted +GitLab, set `SECFOO_GITLAB_HOST` to its hostname, optionally with an HTTPS port +(e.g. `git.example.org:8443`), then pass that host's repository URL. The default +`gitlab.com` remains supported. URL credentials, queries, fragments, HTTP, and +`ssh://` forms are rejected for these new targets. + +For private HTTPS repositories, provide `SECFOO_GITLAB_TOKEN` (a GitLab token +with repository read access) or `SECFOO_BITBUCKET_TOKEN` (a Bitbucket repository, +project, or workspace access token with repository read permission) through your +environment or CI secret store. Bitbucket user API tokens/app passwords are not +supported by this token mode. Tokens are supplied to Git through a temporary +askpass helper, never placed in clone arguments, repository URLs, reports, or +Git config. Git errors omit raw server output to keep credentials out of logs. +Authenticated HTTPS clones disable redirects to avoid forwarding credentials to +another host. Without a token, Git uses your existing noninteractive authentication. Clones are shallow and +removed after the run, including failures. + ## A 30-second look at the dashboard

diff --git a/src/secfoo/runner.py b/src/secfoo/runner.py index a7e194b..3f30268 100644 --- a/src/secfoo/runner.py +++ b/src/secfoo/runner.py @@ -37,6 +37,7 @@ from secfoo.skills.renderer import TargetContext, render_prompt from secfoo.stack import Stack, detect_stack, language_for_ecosystem, language_for_path from secfoo.storage.repository import RunRepository +from secfoo.targets.forge import clone_forge from secfoo.targets.github import clone_shallow from secfoo.targets.resolver import ResolvedTarget, TargetKind, resolve_target @@ -126,11 +127,11 @@ class RunOutcome: def _project_display_name(resolved: ResolvedTarget) -> str: - if resolved.kind is TargetKind.GITHUB: + if resolved.is_temporary: cleaned = resolved.display_name.rstrip("/") if cleaned.endswith(".git"): cleaned = cleaned[: -len(".git")] - match = re.search(r"github\.com[/:]([^/]+/[^/]+)$", cleaned) + match = re.search(r"(?:https?://[^/]+/|git@[^:]+:)(.+)$", cleaned) return match.group(1) if match else cleaned return Path(resolved.display_name).name or resolved.display_name @@ -393,6 +394,9 @@ def _target_workdir(resolved: ResolvedTarget) -> Iterator[Path]: if resolved.kind is TargetKind.GITHUB: with clone_shallow(resolved.display_name) as tmp_dir: yield tmp_dir + elif resolved.kind in {TargetKind.GITLAB, TargetKind.BITBUCKET}: + with clone_forge(resolved.display_name, kind=resolved.kind) as tmp_dir: + yield tmp_dir else: assert resolved.path is not None yield resolved.path diff --git a/src/secfoo/targets/forge.py b/src/secfoo/targets/forge.py new file mode 100644 index 0000000..a6c0e56 --- /dev/null +++ b/src/secfoo/targets/forge.py @@ -0,0 +1,64 @@ +"""Ephemeral clones for GitLab and Bitbucket, with environment-only credentials.""" +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +from secfoo.targets.resolver import TargetKind, TargetResolutionError, forge_kind + + +@contextmanager +def clone_forge(url: str, *, kind: TargetKind, timeout: int = 600) -> Iterator[Path]: + if kind not in {TargetKind.GITLAB, TargetKind.BITBUCKET} or forge_kind(url) is not kind: + raise TargetResolutionError("Invalid forge target") + token_var = "SECFOO_GITLAB_TOKEN" if kind is TargetKind.GITLAB else "SECFOO_BITBUCKET_TOKEN" + username = "oauth2" if kind is TargetKind.GITLAB else "x-token-auth" + with tempfile.TemporaryDirectory(prefix="secfoo-clone-") as directory: + root = Path(directory) + checkout = root / "repo" + env = os.environ.copy() + # Limit token environment variables passed to the clone subprocess. + token = env.pop(token_var, "") + env.pop("SECFOO_GITLAB_TOKEN", None) + env.pop("SECFOO_BITBUCKET_TOKEN", None) + env["GIT_TERMINAL_PROMPT"] = "0" + cmd = ["git"] + if token and url.lower().startswith("https://"): + # The helper contains no credentials; Git sees only its temporary path. + # -c applies to this process, so credentials are not saved in .git/config. + helper = root / "askpass.py" + helper.write_text( + "import os, sys\n" + "print(os.environ['SECFOO_CLONE_USER'] if 'username' in sys.argv[1].lower() " + "else os.environ['SECFOO_CLONE_TOKEN'])\n", encoding="utf-8", + ) + # Git runs askpass commands through its own shell (also bundled + # with Git for Windows); Python paths containing spaces are quoted. + askpass = shlex.quote(sys.executable) + " " + shlex.quote(str(helper)) + env.update(GIT_ASKPASS=askpass, SECFOO_CLONE_TOKEN=token, SECFOO_CLONE_USER=username) + # Disable configured credential helpers so a successful clone cannot + # store the token through a user's credential manager. + cmd += ["-c", "credential.helper="] + if token and url.lower().startswith("https://"): + cmd += ["-c", "http.followRedirects=false"] + cmd += ["clone", "--depth", "1", "--", url, str(checkout)] + try: + subprocess.run(cmd, check=True, capture_output=True, text=True, timeout=timeout, env=env) + except subprocess.TimeoutExpired: + raise TargetResolutionError(f"git clone timed out after {timeout}s") from None + except subprocess.CalledProcessError as exc: + # Do not forward git stderr: remote servers and helpers can echo secrets. + detail = " (disk full)" if "No space left on device" in (exc.stderr or "") else "" + raise TargetResolutionError(f"git clone failed{detail}; check repository access") from None + except OSError: + raise TargetResolutionError("Unable to run git clone; check that git is installed") from None + # Remove askpass material before any agent can inspect the checkout. + for helper_path in (root / "askpass.py",): + helper_path.unlink(missing_ok=True) + yield checkout diff --git a/src/secfoo/targets/resolver.py b/src/secfoo/targets/resolver.py index 857dfe1..0c675a7 100644 --- a/src/secfoo/targets/resolver.py +++ b/src/secfoo/targets/resolver.py @@ -1,9 +1,11 @@ from __future__ import annotations +import os import re from dataclasses import dataclass from enum import Enum from pathlib import Path +from urllib.parse import urlsplit _GITHUB_URL_RE = re.compile(r"^(https?://github\.com/|git@github\.com:)", re.IGNORECASE) @@ -15,12 +17,14 @@ class TargetResolutionError(RuntimeError): class TargetKind(str, Enum): LOCAL = "local" GITHUB = "github" + GITLAB = "gitlab" + BITBUCKET = "bitbucket" @dataclass class ResolvedTarget: kind: TargetKind - path: Path | None # None for GITHUB until clone_shallow() fills it in + path: Path | None # None for remote repositories until cloning display_name: str # cwd path OR the github URL -- used as the storage project identifier is_temporary: bool @@ -29,6 +33,66 @@ def is_github_url(value: str) -> bool: return bool(_GITHUB_URL_RE.match(value.strip())) +def forge_kind(value: str) -> TargetKind | None: + """Recognize supported GitLab/Bitbucket URLs without accepting credentials.""" + if not (looks_like_url(value) or value.startswith("git@")): + return None + try: + initial = urlsplit(value) + if initial.username is not None or initial.password is not None: + raise TargetResolutionError("Repository URLs must not contain credentials; use environment tokens") + except ValueError: + raise TargetResolutionError("Invalid repository URL") from None + if is_github_url(value): + return None + hosts = {"gitlab.com": TargetKind.GITLAB, "bitbucket.org": TargetKind.BITBUCKET} + configured = os.environ.get("SECFOO_GITLAB_HOST", "").strip().lower() + if configured: + try: + parsed_host = urlsplit("https://" + configured) + except ValueError: + raise TargetResolutionError("Invalid SECFOO_GITLAB_HOST") from None + if (parsed_host.netloc != configured or parsed_host.username or parsed_host.password + or parsed_host.path or parsed_host.query or parsed_host.fragment + or not parsed_host.hostname): + raise TargetResolutionError("SECFOO_GITLAB_HOST must be a hostname with an optional port") + try: + _ = parsed_host.port + except ValueError: + raise TargetResolutionError("Invalid SECFOO_GITLAB_HOST port") from None + if parsed_host.hostname in {"github.com", "bitbucket.org"}: + raise TargetResolutionError("SECFOO_GITLAB_HOST conflicts with another forge") + hosts[configured] = TargetKind.GITLAB + if value.startswith("git@"): + match = re.fullmatch(r"git@([^/:]+):(.+)", value) + if not match: + return None + authority, path = match.groups() + kind = hosts.get(authority.lower()) + else: + try: + parsed = urlsplit(value) + if parsed.scheme.lower() not in {"https", "http"}: + return None + if parsed.username is not None or parsed.password is not None: + raise TargetResolutionError("Repository URLs must not contain credentials; use environment tokens") + authority, path = parsed.netloc.lower(), parsed.path.lstrip("/") + kind = hosts.get(authority) + if not kind: + return None + if parsed.scheme.lower() != "https" or parsed.query or parsed.fragment: + raise TargetResolutionError("GitLab and Bitbucket targets require HTTPS without query or fragment") + except ValueError: + raise TargetResolutionError("Invalid repository URL") from None + if kind: + parts = path.removesuffix(".git").split("/") + if (len(parts) < 2 or (kind is TargetKind.BITBUCKET and len(parts) != 2) + or any(not re.fullmatch(r"[A-Za-z0-9_.-]+", part) or part in {".", ".."} + for part in parts)): + raise TargetResolutionError("Repository URL must identify a namespace and repository") + return kind + + def resolve_target(target: str | None, *, cwd: Path | None = None) -> ResolvedTarget: cwd = cwd or Path.cwd() @@ -37,9 +101,15 @@ def resolve_target(target: str | None, *, cwd: Path | None = None) -> ResolvedTa return ResolvedTarget(TargetKind.LOCAL, resolved, str(resolved), False) target = target.strip() + kind = forge_kind(target) + if kind: + return ResolvedTarget(kind, None, target, True) if is_github_url(target): return ResolvedTarget(TargetKind.GITHUB, None, target, True) + if looks_like_url(target) or target.startswith(("git@", "ssh://")): + raise TargetResolutionError("Unsupported repository URL") + path = Path(target).expanduser().resolve() if not path.is_dir(): raise TargetResolutionError(f"Not a directory: {path}") diff --git a/tests/test_targets/test_forge.py b/tests/test_targets/test_forge.py new file mode 100644 index 0000000..7e5bb0b --- /dev/null +++ b/tests/test_targets/test_forge.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import shlex +import subprocess +from pathlib import Path + +import pytest + +from secfoo.runner import _project_display_name, _target_workdir +from secfoo.targets.forge import clone_forge +from secfoo.targets.resolver import TargetKind, TargetResolutionError, resolve_target + + +@pytest.mark.parametrize('url,kind', [ + ('https://gitlab.com/group/nested/repo.git', TargetKind.GITLAB), + ('https://GITLAB.com/group/repo', TargetKind.GITLAB), + ('git@gitlab.com:group/nested/repo.git', TargetKind.GITLAB), + ('https://bitbucket.org/team/repo.git', TargetKind.BITBUCKET), + ('git@bitbucket.org:team/repo', TargetKind.BITBUCKET), +]) +def test_forge_targets(url, kind): + target = resolve_target(url) + assert target.kind is kind + assert target.path is None + assert target.is_temporary + assert target.display_name == url + assert _project_display_name(target) in {'group/nested/repo', 'group/repo', 'team/repo'} + + +def test_self_hosted(monkeypatch): + monkeypatch.setenv('SECFOO_GITLAB_HOST', 'git.example.org:8443') + target = resolve_target('https://git.example.org:8443/group/sub/repo') + assert target.kind is TargetKind.GITLAB + with pytest.raises(TargetResolutionError): + resolve_target('https://git.example.org/group/repo') + + +@pytest.mark.parametrize('host', ['https://git.example.org', 'user:secret@git.example.org', + 'git.example.org/path', 'git.example.org:invalid', 'bitbucket.org']) +def test_bad_host_config(monkeypatch, host): + monkeypatch.setenv('SECFOO_GITLAB_HOST', host) + with pytest.raises(TargetResolutionError): + resolve_target('https://gitlab.com/group/repo') + + +@pytest.mark.parametrize('url', [ + 'https://gitlab.com/group', 'https://bitbucket.org/a/b/c', + 'https://gitlab.com/a/../b', 'https://gitlab.com/a//b', + 'https://gitlab.com/a/b?token=synthetic-secret', + 'https://user:synthetic-secret@gitlab.com/a/b', + 'https://user:synthetic-secret@github.com/a/b', + 'https://gitlab.com.evil.org/a/b', 'http://gitlab.com/a/b', + 'ssh://git@gitlab.com/a/b', 'https://gitlab.com/a/b#synthetic-secret', +]) +def test_reject_invalid_url_without_leaking_input(url): + with pytest.raises(TargetResolutionError) as exc: + resolve_target(url) + assert 'synthetic-secret' not in str(exc.value) + + +@pytest.mark.parametrize('kind,url,var,username', [ + (TargetKind.GITLAB, 'https://gitlab.com/a/b', 'SECFOO_GITLAB_TOKEN', 'oauth2'), + (TargetKind.BITBUCKET, 'https://bitbucket.org/a/b', 'SECFOO_BITBUCKET_TOKEN', 'x-token-auth'), +]) +def test_authenticated_clone_and_runner_dispatch(monkeypatch, kind, url, var, username): + monkeypatch.setenv(var, 'synthetic-secret') + captured = {} + real_run = subprocess.run + + def fake_run(cmd, **kwargs): + captured['cmd'] = cmd + captured['root'] = Path(cmd[-1]).parent + env = kwargs['env'] + assert 'synthetic-secret' not in repr(cmd) + assert kwargs['timeout'] == 600 + assert env['SECFOO_CLONE_TOKEN'] == 'synthetic-secret' + assert env['SECFOO_CLONE_USER'] == username + assert var not in env + assert env['GIT_TERMINAL_PROMPT'] == '0' + helper = Path(shlex.split(env['GIT_ASKPASS'])[1]) + assert 'synthetic-secret' not in helper.read_text() + assert 'synthetic-secret' not in (helper.parent / 'askpass.py').read_text() + for prompt, expected in [('Username for https://example.org', username), + ('Password for https://example.org', 'synthetic-secret')]: + result = real_run(shlex.split(env['GIT_ASKPASS']) + [prompt], env=env, + check=True, capture_output=True, text=True) + assert result.stdout.strip() == expected + Path(cmd[-1]).mkdir() + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr('secfoo.targets.forge.subprocess.run', fake_run) + with _target_workdir(resolve_target(url)) as checkout: + assert checkout.exists() + assert not (checkout.parent / 'askpass.py').exists() + assert captured['cmd'] == ['git', '-c', 'credential.helper=', '-c', 'http.followRedirects=false', 'clone', '--depth', '1', '--', url, str(checkout)] + assert not captured['root'].exists() + + +@pytest.mark.parametrize('error', ['failure', 'timeout', 'missing', 'body']) +def test_cleanup_and_safe_errors(monkeypatch, error): + root = [] + + def fake_run(cmd, **kwargs): + root.append(Path(cmd[-1]).parent) + if error == 'failure': + raise subprocess.CalledProcessError(128, cmd, stderr='synthetic-secret') + if error == 'timeout': + raise subprocess.TimeoutExpired(cmd, 5, stderr='synthetic-secret') + if error == 'missing': + raise FileNotFoundError('synthetic-secret') + Path(cmd[-1]).mkdir() + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setenv('SECFOO_GITLAB_TOKEN', 'synthetic-secret') + monkeypatch.setattr('secfoo.targets.forge.subprocess.run', fake_run) + with ( + pytest.raises((TargetResolutionError, RuntimeError)) as exc, + clone_forge('https://gitlab.com/a/b', kind=TargetKind.GITLAB), + ): + raise RuntimeError('body failure') + assert 'synthetic-secret' not in str(exc.value) + assert not root[0].exists() + + +@pytest.mark.parametrize('url', ['https://gitlab.com/a/b', 'git@gitlab.com:a/b']) +def test_no_token_uses_ordinary_git_auth(monkeypatch, url): + monkeypatch.delenv('SECFOO_GITLAB_TOKEN', raising=False) + commands = [] + + def fake_run(cmd, **kwargs): + commands.append(cmd) + Path(cmd[-1]).mkdir() + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr('secfoo.targets.forge.subprocess.run', fake_run) + with clone_forge(url, kind=TargetKind.GITLAB) as checkout: + assert commands == [['git', 'clone', '--depth', '1', '--', url, str(checkout)]] + + +def test_invalid_gitlab_configuration_keeps_local_and_github_working(monkeypatch, tmp_path): + monkeypatch.setenv('SECFOO_GITLAB_HOST', '[invalid') + assert resolve_target(str(tmp_path)).kind is TargetKind.LOCAL + assert resolve_target('https://github.com/org/repo').kind is TargetKind.GITHUB + with pytest.raises(TargetResolutionError): + resolve_target('https://gitlab.com/group/repo')