Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ After install, the `data-transform` console script is on your `PATH`.
| `tail` | Last N items of a list |
| `group-by` | Group array of objects by top-level KEY |
| `count` | Count items, per KEY value, or per group |
| `stats` | count / sum / min / max / avg of a numeric FIELD |

### Options (most commands)

Expand Down Expand Up @@ -237,6 +238,19 @@ data-transform group-by team people.json | data-transform count --minify
data-transform count status events.ndjson
```

### Stats

`stats FIELD` summarises a numeric top-level field and prints `count`, `sum`, `min`, `max`, and `avg`. On a list root it prints one summary; on `group-by` output it prints one summary per group. Numeric strings count as numbers, so CSV input works directly. Booleans, missing keys, and non-numeric values are skipped. If nothing numeric is found the summary is `count: 0`, `sum: 0`, and `null` for the rest. Whole-number results print as integers.

```bash
echo '[{"ms":120},{"ms":80},{"ms":"100"},{"ms":"n/a"}]' \
| data-transform stats ms -f json --minify
# → {"count":3,"sum":300,"min":80,"max":120,"avg":100}

# Per-group latency summary
data-transform group-by service requests.csv | data-transform stats ms
```

### Library use

```python
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "data-transform-cli"
version = "0.5.0"
version = "0.6.0"
description = "CLI for YAML ↔ JSON ↔ CSV ↔ NDJSON transforms for pipelines and shell use"
readme = "README.md"
license = { text = "MIT" }
Expand Down
4 changes: 3 additions & 1 deletion src/data_transform/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
pick_keys,
rename_keys,
sort_rows,
stats_rows,
tail_rows,
unflatten_object,
unique_rows,
Expand Down Expand Up @@ -41,6 +42,7 @@
"tail_rows",
"group_rows",
"count_rows",
"stats_rows",
]

__version__ = "0.5.0"
__version__ = "0.6.0"
42 changes: 42 additions & 0 deletions src/data_transform/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from .convert import convert, detect_format, infer_format, parse, serialize
from .shape import (
count_rows,
stats_rows,
filter_rows,
flatten_object,
get_path,
Expand Down Expand Up @@ -826,5 +827,46 @@ def count_cmd(
sys.exit(1)


@main.command("stats")
@click.argument("tokens", nargs=-1, required=True)
@click.option("-o", "--output", default=None, help="write to file instead of stdout")
@click.option(
"-f",
"--from",
"from_format",
type=click.Choice(FORMAT_CHOICES, case_sensitive=False),
default=None,
help="input format: json | yaml | csv | ndjson",
)
@click.option("--minify", is_flag=True, default=False, help="emit compact JSON")
def stats_cmd(
tokens: tuple[str, ...],
output: Optional[str],
from_format: Optional[str],
minify: bool,
) -> None:
"""Summarise a numeric FIELD: count, sum, min, max, avg.

Works on a list root (one summary) or group-by output (one summary per
group). Numeric strings count; non-numeric and missing values are skipped.

Usage: data-transform stats FIELD [file]
"""
try:
parts, input_file = _split_tokens_and_file(tokens)
if len(parts) != 1:
raise ValueError("Usage: stats FIELD [file]")
text, name = load_input(input_file)
src = resolve_from(text, name, from_format)
data = parse(text, src) # type: ignore[arg-type]
result = stats_rows(data, parts[0])
write_output(
serialize(result, "json", pretty=not minify, minify=minify), output
)
except Exception as err: # noqa: BLE001
click.echo(f"error: {err}", err=True)
sys.exit(1)


if __name__ == "__main__":
main()
43 changes: 43 additions & 0 deletions src/data_transform/shape.py
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,49 @@ def count_rows(data: Any, key: str | None = None) -> int | dict[str, int]:
return {label: len(items) for label, items in _group(rows, key).items()}


def _stats_one(rows: list[Any], field: str) -> dict[str, Any]:
values = [
_as_number(item[field])
for item in rows
if isinstance(item, Mapping) and field in item and _looks_number(item[field])
]
if not values:
return {"count": 0, "sum": 0, "min": None, "max": None, "avg": None}

def tidy(x: float) -> int | float:
return int(x) if x.is_integer() else x

total = sum(values)
return {
"count": len(values),
"sum": tidy(total),
"min": tidy(min(values)),
"max": tidy(max(values)),
"avg": tidy(total / len(values)),
}


def stats_rows(data: Any, field: str) -> dict[str, Any]:
"""
Summarise numeric top-level FIELD: count, sum, min, max, avg.

A list root returns one summary. An object whose values are all lists
(the output of :func:`group_rows`) returns one summary per group label.
Numeric strings count as numbers (handy for CSV); booleans, missing
keys, and non-numeric values are skipped. With no numeric values the
summary has count 0, sum 0, and null min/max/avg. Whole-number results
are emitted as integers.
"""
if isinstance(data, list):
return _stats_one(data, field)
if _is_grouped(data):
return {str(label): _stats_one(items, field) for label, items in data.items()}
raise ValueError(
"stats: root must be a list (array) or an object of lists "
f"(group-by output), got {type(data).__name__}"
)


def _require_count(n: Any, command: str) -> int:
"""Raise ValueError unless n is a non-negative integer (not bool)."""
if isinstance(n, bool) or not isinstance(n, int) or n < 0:
Expand Down
3 changes: 2 additions & 1 deletion tests/test_shape.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,15 @@ def test_unique_requires_list():


def test_public_exports_include_reshape_helpers():
assert data_transform.__version__ == "0.5.0"
assert data_transform.__version__ == "0.6.0"
for name in (
"rename_keys",
"flatten_object",
"head_rows",
"tail_rows",
"group_rows",
"count_rows",
"stats_rows",
"unflatten_object",
):
assert getattr(data_transform, name) is not None
Expand Down
48 changes: 48 additions & 0 deletions tests/test_stats.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import json

import pytest

from data_transform import group_rows, stats_rows
from test_cli import run_cli


def test_stats_list_mixed_values():
rows = [{"v": 1}, {"v": "2.5"}, {"v": True}, {"v": "x"}, {}, 3, {"v": 4}]
assert stats_rows(rows, "v") == {
"count": 3, "sum": 7.5, "min": 1, "max": 4, "avg": 2.5,
}


def test_stats_empty_and_no_numbers():
empty = {"count": 0, "sum": 0, "min": None, "max": None, "avg": None}
assert stats_rows([], "v") == empty
assert stats_rows([{"v": "a"}], "v") == empty


def test_stats_grouped():
rows = [{"t": "a", "v": 2}, {"t": "b", "v": 5}, {"t": "a", "v": 4}]
out = stats_rows(group_rows(rows, "t"), "v")
assert out["a"] == {"count": 2, "sum": 6, "min": 2, "max": 4, "avg": 3}
assert out["b"]["avg"] == 5


def test_stats_bad_root():
with pytest.raises(ValueError, match="stats: root"):
stats_rows({"a": 1}, "v")


def test_stats_cli_json_and_csv():
code, stdout, _ = run_cli(
["stats", "ms", "-f", "json", "--minify"], '[{"ms":120},{"ms":80}]'
)
assert code == 0
assert json.loads(stdout) == {"count": 2, "sum": 200, "min": 80, "max": 120, "avg": 100}
code, stdout, _ = run_cli(["stats", "ms", "-f", "csv", "--minify"], "ms\n1\n2\n")
assert code == 0 and json.loads(stdout)["avg"] == 1.5


def test_stats_cli_errors():
code, _, stderr = run_cli(["stats", "-f", "json"], "[]")
assert code != 0
code, _, stderr = run_cli(["stats", "a", "b", "-f", "json"], "[]")
assert code != 0 and "Usage" in stderr
Loading