From d15db3e3e8cb0e40677bc594a292f60f5aac9683 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 22 Oct 2026 20:40:41 -0700 Subject: [PATCH 1/4] Add Taiwan's political donation reports as a local corpus (tw-donors) The distinct names of every individual donor in every campaign finance report Taiwan's Control Yuan publishes on ardata.cy.gov.tw: 454 reports of 64 elections, 308,653 names on the data dated 2026-10-05, nearly all unspaced Han names written family first. The fetch reads which reports exist from the platform's listing, since the download answers an absent report and a fault alike; caches each report's zip under its listed name, so a refresh downloads only what was published or rebuilt; reads each incomes.csv by its own header's column names; keeps only the donor column of individual-donation rows; and refuses an unknown column, an unknown category, a listing that changes during the fetch, or more than a sliver of short or uncategorized rows. The CTGov JSON retry loop is generalized into _retried, shared with the zip downloads. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_local_corpora.py | 349 ++++++++++++++++++++++++++++ tools/differential/README.md | 3 +- tools/differential/local_corpora.py | 305 ++++++++++++++++++++++-- 3 files changed, 643 insertions(+), 14 deletions(-) diff --git a/tests/v2/test_local_corpora.py b/tests/v2/test_local_corpora.py index b28e35a69..34bd10b90 100644 --- a/tests/v2/test_local_corpora.py +++ b/tests/v2/test_local_corpora.py @@ -8,9 +8,11 @@ failure that matters is a SMALLER corpus passing for the whole one: its counts read as real zeros. The network is a stub answering by URL. """ +import csv import email.message import hashlib import http.client +import io import json import re import time @@ -519,6 +521,353 @@ def test_a_fec_corpus_of_another_cycle_is_refused_on_load( local.load(_FEC) +# -- Taiwan political donations ------------------------------------------------ + +_TW = "tw-donors" +_TW_ELECTION, _TW_AREA = "109年立法委員選舉", "臺北市" +#: A masked ID, an address and a phone that must never reach the corpus. +_TW_PRIVATE = ("A12****789", "臺北市中正區某某路1號", "02-2****678") + + +def _tw_row(category: str, donor: str, columns: int = 25) -> list[str]: + row = ["1", "候選人甲", _TW_ELECTION, "1", "1081201", category, donor, + _TW_PRIVATE[0], "1000", "0", "現金", "1081202", "", "", "", _TW_PRIVATE[1], + _TW_PRIVATE[2], "", "", "", "", "", "", "", ""] + return row[:columns] + + +def _tw_csv(rows: list[list[str]], header: tuple[str, ...] | None = None) -> bytes: + out = io.StringIO() + writer = csv.writer(out, lineterminator="\r\n") + writer.writerow(header or local.TW_HEADER) + writer.writerows(rows) + return out.getvalue().encode("utf-8-sig") + + +def _tw_zip(incomes: bytes | None, + date: tuple[int, int, int, int, int, int] = (2023, 9, 12, 14, 4, 54)) -> bytes: + """A report zip holding `incomes` as incomes.csv (none when it is + None) beside an expenditures.csv, as the platform's zips do.""" + out = io.BytesIO() + with zipfile.ZipFile(out, "w") as archive: + if incomes is not None: + archive.writestr(zipfile.ZipInfo("incomes.csv", date), incomes) + archive.writestr(zipfile.ZipInfo("expenditures.csv", date), b"") + return out.getvalue() + + +def _tw_report(serial: int = 1, election: str = _TW_ELECTION, area: str = _TW_AREA, + version: str = "112091204") -> dict[str, Any]: + return {"type": "Election", "yearOrSerial": float(serial), "electionName": election, + "electionArea": area, "zipFileName": f"election-{version}-{serial}.zip"} + + +#: Two reports of one election: a donor in both, and one padded with +#: spaces, a business, an anonymous gift and a donation with no name. +_TW_ZIPS = { + (_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv([ + _tw_row("個人捐贈收入", "王小明"), _tw_row("個人捐贈收入", " 陳大文 "), + _tw_row("營利事業捐贈收入", "某某股份有限公司"), _tw_row("匿名捐贈", "匿名"), + _tw_row("個人捐贈收入", " ")])), + (_TW_ELECTION, _TW_AREA, 2): _tw_zip(_tw_csv([ + _tw_row("個人捐贈收入", "王小明"), _tw_row("個人捐贈收入", "林一二")]), + date=(2024, 3, 1, 9, 0, 0)), +} +_TW_LISTING = [_tw_report(1), _tw_report(2)] + +TwAnswer = Exception | bytes + + +def _serve_tw(monkeypatch: pytest.MonkeyPatch, *, + listings: list[list[dict[str, Any]]] | None = None, + zips: dict[tuple[str, str, int], bytes] | None = None, + counted: int | None = None, + fail: Callable[[str], TwAnswer | None] = lambda url: None) -> list[str]: + """Answer the listing from `listings` in turn (the last one again + once they run out), paged by page and pageSize and counted as + `counted` (their length when None), and each download from `zips` by + its election, area and serial. `fail` may answer a URL instead. The + URLs asked.""" + monkeypatch.setattr(time, "sleep", lambda s: None) + listings = listings or [_TW_LISTING] + zips = _TW_ZIPS if zips is None else zips + asked: list[str] = [] + turn = iter(listings) + current: list[dict[str, Any]] = [] + + def urlopen(request: urllib.request.Request, timeout: int) -> _Response: + url = request.full_url + asked.append(url) + if (answer := fail(url)) is not None: + if isinstance(answer, Exception): + raise answer + return _Response(answer) + query = {k: v[0] for k, v in urllib.parse.parse_qs( + urllib.parse.urlsplit(url).query, keep_blank_values=True).items()} + if "/search/elections" in url: + if query["page"] == "1": + current[:] = next(turn, current) + size, page = int(query["pageSize"]), int(query["page"]) + body: Any = {"paging": {"recordCount": len(current) if counted is None + else counted}, + "data": current[(page - 1) * size:page * size]} + return _Response(json.dumps(body).encode()) + return _Response(zips[(query["ElectionName"], query["ElectionArea"], + int(query["YearOrSerial"]))]) + monkeypatch.setattr(urllib.request, "urlopen", urlopen) + return asked + + +def test_a_tw_fetch_keeps_each_donor_s_name_once_and_counts_the_rest( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(local, "TW_MAX_BAD", 1) + _serve_tw(monkeypatch) + corpus = local.fetch(_TW) + assert corpus.records == [{"name": "林一二"}, {"name": "王小明"}, {"name": "陳大文"}] + assert corpus.meta["data_date"] == "2024-03-01T09:00:00" # the newer report's + assert corpus.meta["detail"] == { + "reports": 2, "first_reports": 1, "elections": 1, "rows": 7, + "individual_donations": 5, "rows_from_other_categories": 2, + "donations_without_a_name": 1, "rows_of_a_report_with_no_income": 0, + "rows_without_a_category": 0, "rows_not_as_long_as_the_header": 0} + assert corpus.header().startswith("Control Yuan political donation reports") + assert "Open Government Data License, version 1.0" in corpus.header() + assert corpus.header().endswith("3 names (local corpus tw-donors)") + + +def test_a_tw_fetch_keeps_only_the_donor_column( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + _serve_tw(monkeypatch) + local.fetch(_TW) + for name in ("raw.jsonl", "corpus.jsonl", "meta.json"): + text = (tmp_path / "local" / _TW / name).read_text(encoding="utf-8") + assert not [field for field in _TW_PRIVATE if field in text], name + + +def test_a_tw_fetch_asks_for_each_listed_report_s_zip( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + asked = _serve_tw(monkeypatch) + local.fetch(_TW) + downloads = [urllib.parse.parse_qs(urllib.parse.urlsplit(u).query, + keep_blank_values=True) + for u in asked if "/Search/download" in u] + assert [(d["ElectionName"], d["ElectionArea"], d["YearOrSerial"], d["DownloadType"]) + for d in downloads] == [([_TW_ELECTION], [_TW_AREA], [s], ["3"]) + for s in ("1", "2")] + + +def test_a_tw_listing_of_several_pages_is_read_whole( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(local, "TW_PAGE", 1) + _serve_tw(monkeypatch) + assert local.fetch(_TW).meta["detail"]["reports"] == 2 + + +@pytest.mark.parametrize(("listing", "counted", "match"), [ + (_TW_LISTING[:1], 2, "listed 1 reports of the 2"), + ([_TW_LISTING[0], _TW_LISTING[0]], None, "listed twice"), + ([{**_TW_LISTING[0], "type": "Party"}], None, "type 'Party'"), + ([{**_TW_LISTING[0], "yearOrSerial": 7.0}], None, "serial 7.0"), +], ids=["cut short", "a report twice", "not an election's", "an unknown serial"]) +def test_a_tw_listing_that_cannot_be_trusted_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + listing: list[dict[str, Any]], counted: int | None, match: str) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + _serve_tw(monkeypatch, listings=[listing], counted=counted) + with pytest.raises(ValueError, match=match): + local.fetch(_TW) + assert local.load(_TW) is None + + +@pytest.mark.parametrize("changed", [ + [*_TW_LISTING, _tw_report(1, area="新北市")], + [_TW_LISTING[0], _tw_report(2, version="113010100")], +], ids=["a report published", "a report replaced"]) +def test_a_tw_fetch_whose_listing_moves_under_it_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + changed: list[dict[str, Any]]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + _serve_tw(monkeypatch, listings=[_TW_LISTING, changed]) + with pytest.raises(ValueError, match="published or replaced a report"): + local.fetch(_TW) + assert local.load(_TW) is None + + +_TW_RENAMED = tuple("捐贈者" if c == "捐贈者／支出對象" else c for c in local.TW_HEADER) +#: A column inserted before the donor's, the last dropped: 25 columns, +#: and the donor column now holding the category. +_TW_SHIFTED = (*local.TW_HEADER[:5], "新欄位", *local.TW_HEADER[5:24]) +#: The category column named twice, in place of a column not read. +_TW_TWICE = tuple("收支科目" if c == "地址" else c for c in local.TW_HEADER) + + +@pytest.mark.parametrize(("incomes", "match"), [ + (_tw_csv([_tw_row("個人捐贈收入", "王小明")], header=_TW_RENAMED), "has the header"), + (_tw_csv([_tw_row("個人捐贈收入", "王小明")], header=_TW_SHIFTED), "has the header"), + (_tw_csv([_tw_row("個人捐贈收入", "王小明")], header=_TW_TWICE), "has the header"), + (_tw_csv([_tw_row("個人捐贈收入", "王小明")], header=local.TW_HEADER[:5]), + "has the header"), + (_tw_csv([_tw_row("政治團體捐贈收入", "王小明")]), "the category '政治團體捐贈收入'"), + (_tw_csv([_tw_row("個人捐贈收入", "王小明")]).decode("utf-8-sig").encode("big5"), + "is not UTF-8"), +], ids=["a column renamed", "a column inserted", "a column twice", + "a needed column missing", "an unknown category", "Big5"]) +def test_a_tw_report_whose_layout_has_changed_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, incomes: bytes, match: str) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(incomes)}) + with pytest.raises(ValueError, match=match): + local.fetch(_TW) + assert local.load(_TW) is None + + +def test_a_tw_report_missing_a_column_it_does_not_read_is_read_by_name( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # A 2020 by-election's report had no 更正註記, as a test of the + # header against all 25 columns found on 2026-10-22. The columns + # moved, too, so a reader indexing TW_HEADER reads the wrong one. + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + header = tuple(reversed([c for c in local.TW_HEADER if c != "更正註記"])) + row = dict(zip(local.TW_HEADER, _tw_row("個人捐贈收入", "王小明"))) + zips = {(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv([[row[c] for c in header]], + header=header))} + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips=zips) + assert local.fetch(_TW).records == [{"name": "王小明"}] + + +def test_a_tw_report_with_no_income_is_counted_and_read_past( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Such a report writes one row, every column a dash but the + # candidate's, as a by-election's report did on 2026-10-22. + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + empty = ["-"] * 25 + empty[1] = "候選人乙" + zips = {**_TW_ZIPS, (_TW_ELECTION, _TW_AREA, 2): _tw_zip(_tw_csv([empty]))} + _serve_tw(monkeypatch, zips=zips) + corpus = local.fetch(_TW) + assert corpus.records == [{"name": "王小明"}, {"name": "陳大文"}] + assert corpus.meta["detail"]["rows_of_a_report_with_no_income"] == 1 + assert corpus.meta["detail"]["rows"] == 6 + + +def test_a_tw_dash_category_in_a_row_that_is_not_empty_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + zips = {(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv([_tw_row("-", "王小明")]))} + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips=zips) + with pytest.raises(ValueError, match="the category '-'"): + local.fetch(_TW) + + +@pytest.mark.parametrize(("bad", "key", "match"), [ + (_tw_row("個人捐贈收入", "王小明", columns=24), "rows_not_as_long_as_the_header", + "1 of 2 rows are not as long as"), + (_tw_row("", "王小明"), "rows_without_a_category", "1 of 2 rows are without a category"), +], ids=["short rows", "rows without a category"]) +def test_a_tw_report_with_too_many_bad_rows_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + bad: list[str], key: str, match: str) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + zips = {(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv([bad, _tw_row("個人捐贈收入", "陳大文")]))} + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips=zips) + with pytest.raises(ValueError, match=match): + local.fetch(_TW) + monkeypatch.setattr(local, "TW_MAX_BAD", 1) # under the share: counted + corpus = local.fetch(_TW) + assert corpus.records == [{"name": "陳大文"}] + assert corpus.meta["detail"][key] == 1 + + +def test_a_tw_zip_with_no_incomes_file_is_refused_without_retrying( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + asked = _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(None)}) + with pytest.raises(ValueError, match="holds no incomes.csv"): + local.fetch(_TW) + assert sum("/Search/download" in u for u in asked) == 1 + + +def test_a_tw_zip_cut_short_is_fetched_again( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + cut = [_TW_ZIPS[(_TW_ELECTION, _TW_AREA, 1)][:-30]] + asked = _serve_tw(monkeypatch, fail=lambda url: cut.pop() + if "/Search/download" in url and cut else None) + assert len(local.fetch(_TW).records) == 3 + assert sum("/Search/download" in u for u in asked) == 3 + + +def _tw_downloads(monkeypatch: pytest.MonkeyPatch, + listings: list[list[dict[str, Any]]] | None = None, + zips: dict[tuple[str, str, int], bytes] | None = None) -> list[str]: + """Fetch over _serve_tw; the serial of each report downloaded.""" + asked = _serve_tw(monkeypatch, listings=listings, zips=zips) + local.fetch(_TW) + return [urllib.parse.parse_qs(urllib.parse.urlsplit(u).query)["YearOrSerial"][0] + for u in asked if "/Search/download" in u] + + +def test_a_tw_refetch_downloads_only_the_reports_rebuilt_since( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # A fetch that stops partway, or a refresh, would otherwise ask the + # platform for all 454 reports again. + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + reports = tmp_path / "local" / _TW / "reports" + assert _tw_downloads(monkeypatch) == ["1", "2"] + assert _tw_downloads(monkeypatch) == [] + rebuilt = [_TW_LISTING[0], _tw_report(2, version="113010100")] + zips = {**_TW_ZIPS, (_TW_ELECTION, _TW_AREA, 2): _tw_zip(_tw_csv([ + _tw_row("個人捐贈收入", "趙六")]))} + assert _tw_downloads(monkeypatch, listings=[rebuilt], zips=zips) == ["2"] + assert sorted(p.name for p in reports.iterdir()) == [ + "election-112091204-1.zip", "election-113010100-2.zip"] + corpus = local.load(_TW) + assert corpus is not None and {"name": "趙六"} in corpus.records + + +def test_a_cached_tw_zip_cut_short_is_downloaded_again( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + _tw_downloads(monkeypatch) + cached = tmp_path / "local" / _TW / "reports" / "election-112091204-1.zip" + cached.write_bytes(cached.read_bytes()[:-30]) + assert _tw_downloads(monkeypatch) == ["1"] + assert cached.read_bytes() == _TW_ZIPS[(_TW_ELECTION, _TW_AREA, 1)] + assert "downloading again a cached zip cut short" in capsys.readouterr().err + + +@pytest.mark.parametrize("name", ["../meta.json", "a/b.zip", "", None]) +def test_a_tw_zip_name_that_is_no_file_name_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, name: str | None) -> None: + # The name is the platform's, and the cache writes a file under it. + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + _serve_tw(monkeypatch, listings=[[{**_TW_LISTING[0], "zipFileName": name}]]) + with pytest.raises(ValueError, match="no file name this tool caches"): + local.fetch(_TW) + assert not (tmp_path / "local" / "meta.json").exists() + + +def test_a_listed_tw_report_that_keeps_failing_is_raised( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # The platform answers a fault with the 500 it gives a report that + # does not exist; the listing says this one does, so it is no absence. + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + asked = _serve_tw(monkeypatch, fail=lambda url: _http_error(500) + if "/Search/download" in url else None) + with pytest.raises(urllib.error.HTTPError) as raised: + local.fetch(_TW) + raised.value.close() + assert sum("/Search/download" in u for u in asked) == local.CTGOV_RETRIES + assert local.load(_TW) is None + + # -- load ---------------------------------------------------------------------- def _fetched(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tools/differential/README.md b/tools/differential/README.md index 924aa48ec..5c74a02fc 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -254,8 +254,9 @@ uv run python tools/differential/snapshot.py --local ctgov-officials |---|---|---| | `ctgov-officials` | every overall official of every study on ClinicalTrials.gov, whatever the role (principal investigators, study directors and chairs, and a few sub-investigators and officials with no role), the name as the sponsor's staff typed it into a free-text field: credentials run together (`MBBS,FCPS`), job titles in the name, all-caps and all-lowercase names, typos. Fetched from API v2 (NLM) | names outside clinical research, which is mostly medicine and heavily credentialed; names nobody typed by hand | | `fec-individuals` | the distinct names of every individual contributor in the FEC's bulk file for the 2026 cycle (`indiv26.zip`, downloaded once into the cache and read without unpacking; `fetch --refresh` downloads it again). Mostly a filer's software writing a form's fields in a fixed order: of its 1,893,964 names on 2026-10-18 every one was in capitals and 1,893,961 held a comma, with hand-typed text inside the fields -- 3,306 held a second comma, for a credential run or a suffix in a part of its own (`DOE, JR., JOHN`). The population for the one-case family-comma shape. The FEC's terms (52 U.S.C. 30111(a)(4)) bar using the names to solicit contributions or for any commercial purpose | case, since every name is in capitals; names written in their spoken order, or without a comma; small donors, since a committee need itemize only a donor whose gifts to it pass $200; names outside the US; names in any script but Latin | +| `tw-donors` | the distinct names of every individual donor in every campaign finance report Taiwan's Control Yuan publishes on its political donation platform (ardata.cy.gov.tw, whose zips data.gov.tw also lists): 454 reports of 64 elections on 2026-10-22, presidential, legislative and local: every report from 2018's elections on, and the residual reports of 2014's and 2016's. Each report's zip is cached under the name the platform lists it by, which carries the date it was built, so a refresh downloads only the reports published or rebuilt since. Names as a candidate's staff typed them into a filing, published in full where the ID, address and phone beside them are masked. Of the 308,653 names dated 2026-10-05, 296,803 were three Han characters, 5,449 four and 5,260 two; the population for unspaced Han names written family first, and for the married double surname (冠夫姓) a four-character name often is. Under the Open Government Data License, version 1.0, with the platform's own bar on using the data for profit, credit investigation or another improper purpose | names in spoken order; Latin script (80 names held a Latin letter); the comma (3 held a fullwidth one, none an ASCII one); case; names outside Taiwan; donors whose names the report does not publish, such as anonymous gifts; Japanese kanji names, which the Han script cannot tell from Chinese | -A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. `tools/name_usage.py` looks a word up in each corpus's word index, cached as `wordindex.json` beside it on first use and keyed to the records' sha256 and to the code that cuts and folds a word, so a refetch or a change to the fold rebuilds it; a single-word lookup it answers parses neither the records nor the names. Measured 2026-10-22 with both corpora fetched, a single-word `name_usage.py --no-wikidata` run took 2.5s with their indexes cached (12.5s before the cache, 14.8s on the run that builds it). A spaced word (`Ph. D.`) still scans every name, which took 8.6s, nearly all of it the FEC corpus's. `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. The fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. +A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. `tools/name_usage.py` looks a word up in each corpus's word index, cached as `wordindex.json` beside it on first use and keyed to the records' sha256 and to the code that cuts and folds a word, so a refetch or a change to the fold rebuilds it; a single-word lookup it answers parses neither the records nor the names. Measured 2026-10-22 with both corpora fetched, a single-word `name_usage.py --no-wikidata` run took 2.5s with their indexes cached (12.5s before the cache, 14.8s on the run that builds it). A spaced word (`Ph. D.`) still scans every name, which took 8.6s, nearly all of it the FEC corpus's. `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. `tw-donors` keeps one zip per report under `reports/`, whole, its masked ID, address and phone columns included, as the FEC's zip is kept: the platform's download answers a report that does not exist and a fault alike, with a 500 and an incident number, so the fetch reads which reports exist from the platform's listing and refuses a listed report it cannot download. It reads each file's columns by name, since a 2020 report lacks a column the newest ones write, and refuses a column it does not know, a category it does not know, or a listing that changes during the fetch. A layout check written from one surveyed file meets every other file only at fetch time: `tw-donors`' first three fetches were each refused partway, by a report with no income writing one row of dashes, by a 2020 report with 24 columns where the surveyed one had 25, and by a row with no category. Each refusal cost every download again until the source cached its zips, so a new source that downloads many files caches each one before its checks run. The fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. ## Corpus provenance diff --git a/tools/differential/local_corpora.py b/tools/differential/local_corpora.py index 58d24a5cd..6031fb3a0 100644 --- a/tools/differential/local_corpora.py +++ b/tools/differential/local_corpora.py @@ -20,8 +20,8 @@ - raw.jsonl -- one record per name as the source gave it, with where it came from where the source says: a ClinicalTrials.gov official, with - the study and role; one distinct FEC name, alone. The counts are - counted from here. + the study and role; one distinct FEC or Taiwan donor name, alone. + The counts are counted from here. - corpus.jsonl -- the distinct names, in the corpus line format snapshot.py and compare.py --corpus read. - meta.json -- the source's own date for the data, what was fetched, and @@ -69,15 +69,35 @@ 30111(a)(4)) bar using the names to solicit contributions or for any commercial purpose, which the credit line every report opens with repeats; only the NAME column is kept. + +tw-donors -- the distinct names of every individual donor in every +campaign finance report Taiwan's Control Yuan publishes on its political +donation platform (ardata.cy.gov.tw, the zips data.gov.tw lists): 454 +reports of 64 elections on 2026-10-22, presidential, legislative and +local: every report from 2018's elections on, and the residual reports +of 2014's and 2016's. Each report's zip is cached whole beside the +corpus (40 MB), under the name the platform's listing gives it, so a +refresh downloads only the reports published or rebuilt since. Names as a candidate's staff typed them into a filing, published +in full where the ID, address and phone beside them are masked: of the +308,653 on 2026-10-05, 296,803 were three Han characters, 5,449 four +and 5,260 two, so it is the population for unspaced Han names written +family first, and for the married double surname (冠夫姓) a four- +character name often is. Under the Open Government Data License, +version 1.0, which the credit line every report opens with names, with +the platform's own bar on using the data for profit, credit +investigation or another improper purpose; only the donor column is +kept. """ from __future__ import annotations import argparse +import csv import functools import dataclasses import hashlib import http.client import inspect +import io import json import os import re @@ -93,7 +113,7 @@ from collections.abc import Callable, Iterator from dataclasses import dataclass, field from pathlib import Path -from typing import Any +from typing import Any, TypeVar # The shared fetch and cache helpers (corpus_build.py), found by putting # this directory on sys.path: see corpus_build's docstring for why. @@ -204,24 +224,37 @@ def index(self) -> WordIndex: _TRANSIENT = (OSError, http.client.HTTPException) +_T = TypeVar("_T") + + class _Unparsed(Exception): """A response body that is not JSON: cut short in transit.""" def _get_json(url: str) -> dict[str, Any]: - """`url`'s JSON object, retried with a backoff and a line on stderr - for each retry: one page per thousand studies (608 on 2026-10-21), - and starting over would cost the whole run. Any other 4xx is the - request's fault and is raised, as is the last attempt's error.""" + """`url`'s JSON object, fetched by _retried: one page per thousand + studies (608 on 2026-10-21), and starting over would cost the whole + run.""" + data = _retried(url, lambda body: corpus_build.parse_json(body, url), 0.1) + if not isinstance(data, dict): + raise ValueError(f"{url} answered {type(data).__name__}, not an object") + return data + + +def _retried(url: str, read: Callable[[bytes], _T], delay: float) -> _T: + """`url`'s body as `read` reads it, retried with a backoff and a line + on stderr for each retry, `read` raising ValueError for a body cut + short in transit; `delay` seconds of sleep after each request. Any + other 4xx is the request's fault and is raised, as is the last + attempt's error.""" for attempt in range(CTGOV_RETRIES): try: body = corpus_build.fetch_bytes(url, user_agent=USER_AGENT, - timeout=60, delay=0.1) + timeout=60, delay=delay) try: - data = corpus_build.parse_json(body, url) + return read(body) except ValueError as exc: raise _Unparsed(str(exc)) from None - break except (*_TRANSIENT, _Unparsed) as exc: if (isinstance(exc, urllib.error.HTTPError) and exc.code < 500 and exc.code != 429 or attempt == CTGOV_RETRIES - 1): @@ -235,9 +268,7 @@ def _get_json(url: str) -> dict[str, Any]: print(f" retrying in {wait}s after {type(exc).__name__}: {url}", file=sys.stderr) time.sleep(wait) - if not isinstance(data, dict): - raise ValueError(f"{url} answered {type(data).__name__}, not an object") - return data + raise AssertionError("unreachable: the last attempt raises") def _ctgov_version() -> tuple[str, str]: @@ -441,6 +472,248 @@ def check_fec(meta: dict[str, Any]) -> str: f"holds the {cycle} cycle, and this tool reads the {FEC_CYCLE} cycle") +# -- Taiwan political donations ------------------------------------------------ + +TW_API = "https://ardata.cy.gov.tw/api/v1" +TW_PAGE = 1000 # one page held all 454 reports on 2026-10-22 +#: The seconds of sleep after each report's download. +TW_DELAY = 1.0 +TW_INCOMES = "incomes.csv" +#: The columns incomes.csv may hold, in the order the newest reports +#: write them; an older report omits some (a 2020 by-election's had no +#: 更正註記), so a file is read by its own header's names. +TW_HEADER = ( + "序號", "擬參選人／政黨", "選舉名稱", "申報序號／年度", "交易日期", "收支科目", + "捐贈者／支出對象", "身分證／統一編號", "收入金額", "支出金額", "捐贈方式", + "存入專戶日期", "返還/繳庫", "支出用途", "金錢類", "地址", "聯絡電話", + "應揭露之支出對象", "支出對象之內部人員姓名", "支出對象之內部人員職稱", + "政黨之內部人員姓名", "政黨之內部人員職稱", "關係", "更正註記", "資料更正日期") +TW_CATEGORY, TW_DONOR = "收支科目", "捐贈者／支出對象" +TW_INDIVIDUAL = "個人捐贈收入" +#: The income categories a candidate's report holds (the platform's +#: classTypes with ctrlTp 0): individual, business, party and civic +#: group donations, anonymous gifts, other income. +TW_CATEGORIES = frozenset({ + TW_INDIVIDUAL, "營利事業捐贈收入", "政黨捐贈收入", "人民團體捐贈收入", + "匿名捐贈", "其他收入"}) +#: The candidate's column, the one a report with no income fills in +#: the single row it writes, every other column holding TW_EMPTY. +TW_CANDIDATE = "擬參選人／政黨" +#: The columns every file must hold, the ones read. +TW_NEEDED = frozenset({TW_CATEGORY, TW_DONOR, TW_CANDIDATE}) +TW_EMPTY = "-" +#: A report's serial: 1 the first report, 2 to 6 the five reports on +#: the money left over. +TW_SERIALS = range(1, 7) +#: More than this share of rows not as long as their file's header, or +#: of rows with an empty category, refuses the fetch, as FEC_MAX_BAD +#: does. A row with no category is read past: one 2021 residual report +#: held one, an income of NT$53 with no donor ID (2026-10-22). +TW_MAX_BAD = 1 / 10_000 + + +def _tw_reports() -> list[dict[str, Any]]: + """Every election report the platform lists, paged: the listing is + what says a report exists, since the download answers a report that + does not and a fault alike, with a 500 and an incident number.""" + reports: list[dict[str, Any]] = [] + seen: set[tuple[str, str, int]] = set() + page, count = 1, None + while count is None or len(reports) < count: + query = urllib.parse.urlencode({"page": page, "pageSize": TW_PAGE}) + data = _get_json(f"{TW_API}/search/elections?{query}") + count = int(data["paging"]["recordCount"]) + if not data["data"]: + break + for row in data["data"]: + serial = row["yearOrSerial"] + if row["type"] != "Election" or serial not in TW_SERIALS: + raise ValueError(f"a listed report is not an election's first or " + f"residual report: type {row['type']!r}, serial " + f"{serial!r}; the listing has changed") + key = (row["electionName"], row["electionArea"], int(serial)) + if key in seen: + raise ValueError(f"{key} was listed twice: the listing shifted " + f"under the fetch; fetch again") + seen.add(key) + reports.append(row) + page += 1 + if len(reports) != count: + raise ValueError(f"listed {len(reports):,} reports of the {count:,} the " + f"platform counted; fetch again") + return reports + + +def _tw_versions(reports: list[dict[str, Any]]) -> set[tuple[str, str, int, str]]: + """Each report with the zip its listing names, which carries the + date the platform built it: a replaced report lists another.""" + return {(r["electionName"], r["electionArea"], int(r["yearOrSerial"]), + r["zipFileName"]) for r in reports} + + +def _tw_url(report: dict[str, Any]) -> str: + """A report's zip, asked for by the fields its listing row gives.""" + return f"{TW_API}/Search/download?" + urllib.parse.urlencode({ + "ElectionName": report["electionName"], + "ElectionArea": report["electionArea"], "AccountNumber": "", + "YearOrSerial": int(report["yearOrSerial"]), "Version": "", + "SearchType": 2, "DownloadType": 3}) + + +def _tw_incomes(body: bytes) -> tuple[bytes, str] | None: + """A report zip's incomes.csv and its timestamp, None where the zip + holds none, or ValueError where the body is no whole zip: a download + cut short, which _retried retries.""" + try: + with zipfile.ZipFile(io.BytesIO(body)) as archive: + if TW_INCOMES not in archive.namelist(): + return None + member = archive.getinfo(TW_INCOMES) + stamp = "{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}".format(*member.date_time) + return archive.read(member), stamp + except zipfile.BadZipFile as exc: + raise ValueError(f"not a whole zip ({exc})") from None + + +#: The cached reports' folder, beside the corpus. +TW_REPORTS = "reports" +#: A listed zip's name, the only kind cached under it: the name is the +#: platform's and becomes a path. +_TW_ZIP_NAME = re.compile(r"[\w-]+\.zip", re.ASCII) + + +def _tw_whole(body: bytes) -> bytes: + _tw_incomes(body) # raises ValueError where the zip is cut short + return body + + +def _tw_report_zip(report: dict[str, Any], label: str) -> bytes: + """A listed report's zip, read from the reports folder where an + earlier fetch cached it under the name the listing gives, else + downloaded and cached there. The name carries the date the platform + built the zip, so a rebuilt report is downloaded anew, and a cached + zip that is no whole zip is too.""" + name = report["zipFileName"] + if not isinstance(name, str) or not _TW_ZIP_NAME.fullmatch(name): + raise ValueError(f"{label} is listed with the zip name {name!r}, which is " + f"no file name this tool caches; the listing has changed") + path = directory("tw-donors") / TW_REPORTS / name + if path.exists(): + body = path.read_bytes() + try: + return _tw_whole(body) + except ValueError: + print(f" downloading again a cached zip cut short: {path}", file=sys.stderr) + body = _retried(_tw_url(report), _tw_whole, TW_DELAY) + path.parent.mkdir(exist_ok=True) + corpus_build.write_atomic(path, body) + return body + + +def _tw_read(incomes: bytes, label: str, names: set[str], counts: Counter[str]) -> None: + """Add a report's donor names to `names`, counting into `counts` + what is read past. A header or a category this tool was not written + for is refused, not read: a moved column would put another field's + values in the corpus, so a header holding a column TW_HEADER does + not name, a column twice, or not every column TW_NEEDED names is + refused; one missing a column the tool does not read is read. The + one row a report with no income writes, every column a dash but the + candidate's, is counted apart.""" + try: + text = incomes.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise ValueError(f"{label}'s {TW_INCOMES} is not UTF-8 ({exc})") from None + rows = csv.reader(io.StringIO(text, newline="")) + header = next(rows, []) + if (not set(header) <= set(TW_HEADER) or len(set(header)) != len(header) + or not TW_NEEDED <= set(header)): + raise ValueError(f"{label}'s {TW_INCOMES} has the header {tuple(header)}, " + f"which is not columns of TW_HEADER, each once, holding " + f"those TW_NEEDED names") + category_at, donor_at, candidate_at = ( + header.index(column) for column in (TW_CATEGORY, TW_DONOR, TW_CANDIDATE)) + for row in rows: + counts["rows"] += 1 + if len(row) != len(header): + counts["rows_not_as_long_as_the_header"] += 1 + continue + category = row[category_at] + if {f for i, f in enumerate(row) if i != candidate_at} == {TW_EMPTY}: + counts["rows_of_a_report_with_no_income"] += 1 + continue + if not category: + counts["rows_without_a_category"] += 1 + continue + if category not in TW_CATEGORIES: + raise ValueError(f"{label}'s {TW_INCOMES} holds the category " + f"{category!r}, which TW_CATEGORIES does not list") + if category != TW_INDIVIDUAL: + counts["rows_from_other_categories"] += 1 + continue + counts["individual_donations"] += 1 + if name := row[donor_at].strip(): + names.add(name) + else: + counts["donations_without_a_name"] += 1 + + +def fetch_tw() -> Fetched: + """The distinct names of every individual donor in every election + report the platform lists, one record per name. + + Every report is read, the first and the residual ones alike, and a + name is kept once however many reports, candidates or elections it + gave to, as the FEC source keeps one per name. Only the donor column + of the rows the category marks as individual donations is kept, its + spaces stripped; the masked ID, address and phone columns and the + amounts are read past. Each report's zip is cached (_tw_report_zip), + so a fetch cut short or refused goes on where it stopped, and a + refresh downloads only the reports published or rebuilt since; a + cached zip the listing no longer names is deleted once the fetch has + read the rest. The listing is read again at the end, and a + report published or replaced during the fetch refuses it. The data + date is the newest incomes.csv's own timestamp, which carries no time + zone.""" + reports = _tw_reports() + names: set[str] = set() + counts: Counter[str] = Counter() + stamps: list[str] = [] + for number, report in enumerate(reports, 1): + if number % 50 == 0: + print(f" {number:,} of {len(reports):,} reports", file=sys.stderr) + label = f"{report['electionName']} {report['electionArea']} " \ + f"report {int(report['yearOrSerial'])}" + got = _tw_incomes(_tw_report_zip(report, label)) + if got is None: + raise ValueError(f"{label}'s zip holds no {TW_INCOMES}: the reports' " + f"layout has changed") + stamps.append(got[1]) + _tw_read(got[0], label, names, counts) + for key, what in (("rows_not_as_long_as_the_header", + "not as long as their file's header"), + ("rows_without_a_category", "without a category")): + if counts[key] > counts["rows"] * TW_MAX_BAD: + raise ValueError(f"{counts[key]:,} of {counts['rows']:,} rows are " + f"{what}: the files' layout has changed") + if _tw_versions(_tw_reports()) != _tw_versions(reports): + raise ValueError("the platform published or replaced a report during " + "the fetch; fetch again") + listed = {r["zipFileName"] for r in reports} + for stale in (directory("tw-donors") / TW_REPORTS).glob("*.zip"): + if stale.name not in listed: + stale.unlink() + return Fetched([{"name": name} for name in sorted(names)], max(stamps, default=""), { + "reports": len(reports), + "first_reports": sum(int(r["yearOrSerial"]) == 1 for r in reports), + "elections": len({r["electionName"] for r in reports}), + "rows": counts["rows"], "individual_donations": counts["individual_donations"], + "rows_from_other_categories": counts["rows_from_other_categories"], + "donations_without_a_name": counts["donations_without_a_name"], + "rows_of_a_report_with_no_income": counts["rows_of_a_report_with_no_income"], + "rows_without_a_category": counts["rows_without_a_category"], + "rows_not_as_long_as_the_header": counts["rows_not_as_long_as_the_header"]}) + + SOURCES: dict[str, Source] = { "ctgov-officials": Source( credit="ClinicalTrials.gov (NLM) study officials", unit="officials", @@ -449,6 +722,12 @@ def check_fec(meta: dict[str, Any]) -> str: credit=f"FEC individual contributions, {FEC_CYCLE} cycle " f"(52 U.S.C. 30111(a)(4): not for soliciting or commercial use)", unit="names", fetch=fetch_fec, renew=renew_fec, check=check_fec), + "tw-donors": Source( + credit="Control Yuan political donation reports (ardata.cy.gov.tw), " + "Open Government Data License, version 1.0 (Taiwan) " + "(the platform's terms: not for profit, credit investigation " + "or other improper use)", + unit="names", fetch=fetch_tw), } From 0b3b34946643e3a4374dd1552fdd11e647096a09 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 22 Oct 2026 21:14:06 -0700 Subject: [PATCH 2/4] tw-donors: read back rows a stray quote swallowed, and check each zip is the report asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #844, measured on the cached zips: - A stray quote opening an address or phone field took 157 whole rows into four fields in two 2020 reports, losing their donors with no count. A record whose field spans lines is now cut back into its lines where any of them reads as a whole row; a quoted field holding two numbers on two lines (8 records) stays one row. - Each row's serial column must name the report asked for (首次申報, 第N次賸餘 in either numerals), so a zip served for the wrong report is refused; the election-name column cannot do this, since it names the office where the listing names the election. - A report more than half of whose rows are read past is refused: the fetch-wide 1/10,000 share let a small report's rows all go. - A downloaded zip is cached only once its rows are read, so a refused zip (no incomes.csv, another report's) is asked for again rather than refused from the cache on every later fetch. - Donor cells holding 匿名 or a dash (19) are counted as naming no one, and masked names (3 with an asterisk) are counted and refused past the share. - Two listed reports under one zip name are refused; a member that does not inflate (zlib.error, EOFError) is retried like a cut zip; a body that does not parse names its URL. - Docstring and README wording: deletion only after a fetch passes its checks, a withdrawn report refuses the fetch too, counts dated, the FEC sentence moved back beside the FEC. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_local_corpora.py | 368 ++++++++++++++++++++-------- tools/differential/README.md | 4 +- tools/differential/local_corpora.py | 290 +++++++++++++++------- 3 files changed, 470 insertions(+), 192 deletions(-) diff --git a/tests/v2/test_local_corpora.py b/tests/v2/test_local_corpora.py index 34bd10b90..1e87d2e07 100644 --- a/tests/v2/test_local_corpora.py +++ b/tests/v2/test_local_corpora.py @@ -527,10 +527,14 @@ def test_a_fec_corpus_of_another_cycle_is_refused_on_load( _TW_ELECTION, _TW_AREA = "109年立法委員選舉", "臺北市" #: A masked ID, an address and a phone that must never reach the corpus. _TW_PRIVATE = ("A12****789", "臺北市中正區某某路1號", "02-2****678") +#: An area's code in its zip's name, as the platform's names carry one. +_TW_AREA_CODES = {"臺北市": "63000", "新北市": "65000"} +_FIRST, _RESIDUAL = "首次申報", "第1次賸餘" -def _tw_row(category: str, donor: str, columns: int = 25) -> list[str]: - row = ["1", "候選人甲", _TW_ELECTION, "1", "1081201", category, donor, +def _tw_row(category: str, donor: str, serial: str = _FIRST, + columns: int = 25) -> list[str]: + row = ["1", "候選人甲", _TW_ELECTION, serial, "1081201", category, donor, _TW_PRIVATE[0], "1000", "0", "現金", "1081202", "", "", "", _TW_PRIVATE[1], _TW_PRIVATE[2], "", "", "", "", "", "", "", ""] return row[:columns] @@ -547,11 +551,13 @@ def _tw_csv(rows: list[list[str]], header: tuple[str, ...] | None = None) -> byt def _tw_zip(incomes: bytes | None, date: tuple[int, int, int, int, int, int] = (2023, 9, 12, 14, 4, 54)) -> bytes: """A report zip holding `incomes` as incomes.csv (none when it is - None) beside an expenditures.csv, as the platform's zips do.""" + None) beside an expenditures.csv, deflated as the platform's zips + are.""" out = io.BytesIO() - with zipfile.ZipFile(out, "w") as archive: + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as archive: if incomes is not None: - archive.writestr(zipfile.ZipInfo("incomes.csv", date), incomes) + archive.writestr(zipfile.ZipInfo("incomes.csv", date, ), incomes, + zipfile.ZIP_DEFLATED) archive.writestr(zipfile.ZipInfo("expenditures.csv", date), b"") return out.getvalue() @@ -559,19 +565,27 @@ def _tw_zip(incomes: bytes | None, def _tw_report(serial: int = 1, election: str = _TW_ELECTION, area: str = _TW_AREA, version: str = "112091204") -> dict[str, Any]: return {"type": "Election", "yearOrSerial": float(serial), "electionName": election, - "electionArea": area, "zipFileName": f"election-{version}-{serial}.zip"} + "electionArea": area, + "zipFileName": f"election-{version}-{serial}-{_TW_AREA_CODES[area]}.zip"} -#: Two reports of one election: a donor in both, and one padded with -#: spaces, a business, an anonymous gift and a donation with no name. +def _tw_first(*rows: list[str]) -> dict[tuple[str, str, int], bytes]: + """The zips of a listing of the first report alone, holding `rows`.""" + return {(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv(list(rows)))} + + +#: Two reports of one election: a donor in both, one padded with +#: spaces, a business, an anonymous gift, and donations naming no one. +#: The first report is the newer file, so the data date is not simply +#: the last report's. _TW_ZIPS = { (_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv([ _tw_row("個人捐贈收入", "王小明"), _tw_row("個人捐贈收入", " 陳大文 "), _tw_row("營利事業捐贈收入", "某某股份有限公司"), _tw_row("匿名捐贈", "匿名"), - _tw_row("個人捐贈收入", " ")])), + _tw_row("個人捐贈收入", " ")]), date=(2024, 3, 1, 9, 0, 0)), (_TW_ELECTION, _TW_AREA, 2): _tw_zip(_tw_csv([ - _tw_row("個人捐贈收入", "王小明"), _tw_row("個人捐贈收入", "林一二")]), - date=(2024, 3, 1, 9, 0, 0)), + _tw_row("個人捐贈收入", "王小明", _RESIDUAL), _tw_row("個人捐贈收入", "林一二", _RESIDUAL), + _tw_row("個人捐贈收入", "匿名", _RESIDUAL), _tw_row("個人捐贈收入", "-", _RESIDUAL)])), } _TW_LISTING = [_tw_report(1), _tw_report(2)] @@ -618,37 +632,59 @@ def urlopen(request: urllib.request.Request, timeout: int) -> _Response: return asked -def test_a_tw_fetch_keeps_each_donor_s_name_once_and_counts_the_rest( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def _tw_downloads(monkeypatch: pytest.MonkeyPatch, + listings: list[list[dict[str, Any]]] | None = None, + zips: dict[tuple[str, str, int], bytes] | None = None) -> list[str]: + """Fetch over _serve_tw; the serial of each report downloaded.""" + asked = _serve_tw(monkeypatch, listings=listings, zips=zips) + local.fetch(_TW) + return _tw_serials(asked) + + +def _tw_serials(asked: list[str]) -> list[str]: + return [urllib.parse.parse_qs(urllib.parse.urlsplit(u).query)["YearOrSerial"][0] + for u in asked if "/Search/download" in u] + + +def _tw_cache(tmp_path: Path) -> Path: + return tmp_path / "local" / _TW / "reports" + + +@pytest.fixture +def _tw_home(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) - monkeypatch.setattr(local, "TW_MAX_BAD", 1) + return tmp_path + + +def test_a_tw_fetch_keeps_each_donor_s_name_once_and_counts_the_rest( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: _serve_tw(monkeypatch) corpus = local.fetch(_TW) assert corpus.records == [{"name": "林一二"}, {"name": "王小明"}, {"name": "陳大文"}] - assert corpus.meta["data_date"] == "2024-03-01T09:00:00" # the newer report's + assert corpus.meta["data_date"] == "2024-03-01T09:00:00" # the newer file's assert corpus.meta["detail"] == { - "reports": 2, "first_reports": 1, "elections": 1, "rows": 7, - "individual_donations": 5, "rows_from_other_categories": 2, - "donations_without_a_name": 1, "rows_of_a_report_with_no_income": 0, - "rows_without_a_category": 0, "rows_not_as_long_as_the_header": 0} + "reports": 2, "first_reports": 1, "elections": 1, "rows": 9, + "individual_donations": 7, "rows_from_other_categories": 2, + "donations_without_a_name": 3, "donations_naming_a_masked_donor": 0, + "rows_of_a_report_with_no_income": 0, "records_cut_back_into_their_lines": 0, + "rows_not_as_long_as_the_header": 0, "rows_without_a_category": 0, + "rows_cut_by_a_stray_quote": 0} assert corpus.header().startswith("Control Yuan political donation reports") assert "Open Government Data License, version 1.0" in corpus.header() assert corpus.header().endswith("3 names (local corpus tw-donors)") def test_a_tw_fetch_keeps_only_the_donor_column( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: _serve_tw(monkeypatch) local.fetch(_TW) for name in ("raw.jsonl", "corpus.jsonl", "meta.json"): - text = (tmp_path / "local" / _TW / name).read_text(encoding="utf-8") + text = (_tw_home / "local" / _TW / name).read_text(encoding="utf-8") assert not [field for field in _TW_PRIVATE if field in text], name def test_a_tw_fetch_asks_for_each_listed_report_s_zip( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: asked = _serve_tw(monkeypatch) local.fetch(_TW) downloads = [urllib.parse.parse_qs(urllib.parse.urlsplit(u).query, @@ -659,24 +695,35 @@ def test_a_tw_fetch_asks_for_each_listed_report_s_zip( for s in ("1", "2")] +def test_a_tw_fetch_pauses_after_each_download( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + sleeps: list[float] = [] + _serve_tw(monkeypatch) + monkeypatch.setattr(time, "sleep", sleeps.append) + local.fetch(_TW) + assert sleeps.count(local.TW_DELAY) == 2 and local.TW_DELAY >= 1 + + def test_a_tw_listing_of_several_pages_is_read_whole( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: monkeypatch.setattr(local, "TW_PAGE", 1) _serve_tw(monkeypatch) assert local.fetch(_TW).meta["detail"]["reports"] == 2 +@pytest.mark.timeout(30) # a listing loop that never stops hangs, not fails @pytest.mark.parametrize(("listing", "counted", "match"), [ (_TW_LISTING[:1], 2, "listed 1 reports of the 2"), ([_TW_LISTING[0], _TW_LISTING[0]], None, "listed twice"), + ([_TW_LISTING[0], {**_TW_LISTING[1], "zipFileName": _TW_LISTING[0]["zipFileName"]}], + None, "zip name .* another report has"), ([{**_TW_LISTING[0], "type": "Party"}], None, "type 'Party'"), ([{**_TW_LISTING[0], "yearOrSerial": 7.0}], None, "serial 7.0"), -], ids=["cut short", "a report twice", "not an election's", "an unknown serial"]) +], ids=["cut short", "a report twice", "two reports one zip", "not an election's", + "an unknown serial"]) def test_a_tw_listing_that_cannot_be_trusted_is_refused( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, listing: list[dict[str, Any]], counted: int | None, match: str) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) _serve_tw(monkeypatch, listings=[listing], counted=counted) with pytest.raises(ValueError, match=match): local.fetch(_TW) @@ -686,20 +733,20 @@ def test_a_tw_listing_that_cannot_be_trusted_is_refused( @pytest.mark.parametrize("changed", [ [*_TW_LISTING, _tw_report(1, area="新北市")], [_TW_LISTING[0], _tw_report(2, version="113010100")], -], ids=["a report published", "a report replaced"]) + _TW_LISTING[:1], +], ids=["a report published", "a report replaced", "a report withdrawn"]) def test_a_tw_fetch_whose_listing_moves_under_it_is_refused( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, changed: list[dict[str, Any]]) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) _serve_tw(monkeypatch, listings=[_TW_LISTING, changed]) - with pytest.raises(ValueError, match="published or replaced a report"): + with pytest.raises(ValueError, match="published, replaced or withdrew a report"): local.fetch(_TW) assert local.load(_TW) is None _TW_RENAMED = tuple("捐贈者" if c == "捐贈者／支出對象" else c for c in local.TW_HEADER) -#: A column inserted before the donor's, the last dropped: 25 columns, -#: and the donor column now holding the category. +#: A column inserted before the category's, the last dropped: 25 +#: columns, and the donor column now holding the category. _TW_SHIFTED = (*local.TW_HEADER[:5], "新欄位", *local.TW_HEADER[5:24]) #: The category column named twice, in place of a column not read. _TW_TWICE = tuple("收支科目" if c == "地址" else c for c in local.TW_HEADER) @@ -717,8 +764,7 @@ def test_a_tw_fetch_whose_listing_moves_under_it_is_refused( ], ids=["a column renamed", "a column inserted", "a column twice", "a needed column missing", "an unknown category", "Big5"]) def test_a_tw_report_whose_layout_has_changed_is_refused( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, incomes: bytes, match: str) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, incomes: bytes, match: str) -> None: _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(incomes)}) with pytest.raises(ValueError, match=match): @@ -726,12 +772,32 @@ def test_a_tw_report_whose_layout_has_changed_is_refused( assert local.load(_TW) is None +@pytest.mark.parametrize("serial", [_FIRST, "第一次賸餘", _RESIDUAL]) +def test_a_tw_zip_of_another_report_is_refused( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, serial: str) -> None: + # A server answering a residual report with the first report's zip, + # or the reverse, would merge one report's donors into another's. + listing = [_tw_report(3)] + zips = {(_TW_ELECTION, _TW_AREA, 3): _tw_zip(_tw_csv([ + _tw_row("個人捐贈收入", "王小明", serial)]))} + _serve_tw(monkeypatch, listings=[listing], zips=dict(zips)) + with pytest.raises(ValueError, match="the zip is another report's"): + local.fetch(_TW) + assert not _tw_cache(_tw_home).exists() # so a later fetch asks again + # Its own serial, in either numerals, is read. + for version, own in (("113010101", "第2次賸餘"), ("113010102", "第二次賸餘")): + zips[(_TW_ELECTION, _TW_AREA, 3)] = _tw_zip(_tw_csv([ + _tw_row("個人捐贈收入", "王小明", own)])) + listing[0] = _tw_report(3, version=version) + assert _tw_downloads(monkeypatch, listings=[listing], zips=zips) == ["3"], own + + def test_a_tw_report_missing_a_column_it_does_not_read_is_read_by_name( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - # A 2020 by-election's report had no 更正註記, as a test of the - # header against all 25 columns found on 2026-10-22. The columns - # moved, too, so a reader indexing TW_HEADER reads the wrong one. - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + # A 2020 by-election's report had no 更正註記, which a check of the + # header against all 25 columns found on 2026-10-22; the column + # after it moved up one, so a reader indexing TW_HEADER reads the + # wrong one. The test reverses the header to move every column. header = tuple(reversed([c for c in local.TW_HEADER if c != "更正註記"])) row = dict(zip(local.TW_HEADER, _tw_row("個人捐贈收入", "王小明"))) zips = {(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv([[row[c] for c in header]], @@ -741,10 +807,9 @@ def test_a_tw_report_missing_a_column_it_does_not_read_is_read_by_name( def test_a_tw_report_with_no_income_is_counted_and_read_past( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: # Such a report writes one row, every column a dash but the # candidate's, as a by-election's report did on 2026-10-22. - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) empty = ["-"] * 25 empty[1] = "候選人乙" zips = {**_TW_ZIPS, (_TW_ELECTION, _TW_AREA, 2): _tw_zip(_tw_csv([empty]))} @@ -755,11 +820,20 @@ def test_a_tw_report_with_no_income_is_counted_and_read_past( assert corpus.meta["detail"]["rows"] == 6 +def _dashed(**filled: str) -> list[str]: + """A row of dashes but the candidate's column and `filled`.""" + row: dict[str, str] = {**dict.fromkeys(local.TW_HEADER, "-"), + "擬參選人／政黨": "候選人乙", **filled} + return [row[c] for c in local.TW_HEADER] + + +@pytest.mark.parametrize("row", [ + _tw_row("-", "王小明"), + _dashed(收入金額="1000"), +], ids=["a donor named", "an amount filled"]) def test_a_tw_dash_category_in_a_row_that_is_not_empty_is_refused( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) - zips = {(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv([_tw_row("-", "王小明")]))} - _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips=zips) + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, row: list[str]) -> None: + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips=_tw_first(row)) with pytest.raises(ValueError, match="the category '-'"): local.fetch(_TW) @@ -769,12 +843,13 @@ def test_a_tw_dash_category_in_a_row_that_is_not_empty_is_refused( "1 of 2 rows are not as long as"), (_tw_row("", "王小明"), "rows_without_a_category", "1 of 2 rows are without a category"), ], ids=["short rows", "rows without a category"]) -def test_a_tw_report_with_too_many_bad_rows_is_refused( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +def test_a_tw_fetch_with_too_many_bad_rows_is_refused( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, bad: list[str], key: str, match: str) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) - zips = {(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv([bad, _tw_row("個人捐贈收入", "陳大文")]))} - _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips=zips) + # The share is the whole fetch's: one bad row of two is half, which + # the share for one report lets through. + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips=_tw_first(bad, _tw_row("個人捐贈收入", "陳大文"))) with pytest.raises(ValueError, match=match): local.fetch(_TW) monkeypatch.setattr(local, "TW_MAX_BAD", 1) # under the share: counted @@ -783,89 +858,180 @@ def test_a_tw_report_with_too_many_bad_rows_is_refused( assert corpus.meta["detail"][key] == 1 -def test_a_tw_zip_with_no_incomes_file_is_refused_without_retrying( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) +def test_a_small_tw_report_whose_rows_are_all_bad_is_refused( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + # Over the whole fetch its rows are a sliver; alone they are all of + # it, and a report whose donors all go must not pass for read. + big = [_tw_row("個人捐贈收入", f"王小{n}") for n in range(30)] + short = [_tw_row("個人捐贈收入", f"陳大{n}", _RESIDUAL, columns=24) for n in range(3)] + zips = {(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_csv(big)), + (_TW_ELECTION, _TW_AREA, 2): _tw_zip(_tw_csv(short))} + monkeypatch.setattr(local, "TW_MAX_BAD", 1) + _serve_tw(monkeypatch, zips=zips) + with pytest.raises(ValueError, match="3 of .* report 2's rows are read past"): + local.fetch(_TW) + + +def _tw_incomes_text(lines: list[str]) -> bytes: + return ("\r\n".join([",".join(local.TW_HEADER), *lines]) + "\r\n").encode("utf-8-sig") + + +def _line(donor: str, address: str = "臺北市", phone: str = "02") -> str: + row = _tw_row("個人捐贈收入", donor) + row[15], row[16] = address, phone + return ",".join(row) + + +def test_rows_a_stray_quote_takes_in_are_read_back( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + # An address opening with a quote nothing closes until a later row: + # the rows between, two 2020 reports' 157 on 2026-10-22, would go + # into one field and their donors with them. + lines = [_line("王小明", address='"臺北市'), _line("陳大文"), _line("林一二"), + _line("趙六", phone='02"')] + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_incomes_text(lines))}) + monkeypatch.setattr(local, "TW_MAX_BAD", 1) + corpus = local.fetch(_TW) + assert {"name": "陳大文"} in corpus.records and {"name": "林一二"} in corpus.records + detail = corpus.meta["detail"] + assert detail["records_cut_back_into_their_lines"] == 1 + assert detail["rows_cut_by_a_stray_quote"] >= 1 + monkeypatch.setattr(local, "TW_MAX_BAD", 1 / 10_000) + with pytest.raises(ValueError, match="rows are cut by a stray quote"): + local.fetch(_TW) + + +def test_a_quoted_field_spanning_two_lines_stays_one_row( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + # Two phone numbers on two lines, as 8 records held on 2026-10-22. + lines = [_line("王小明", phone='"02\r\n09"'), _line("陳大文")] + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_incomes_text(lines))}) + corpus = local.fetch(_TW) + assert corpus.records == [{"name": "王小明"}, {"name": "陳大文"}] + assert corpus.meta["detail"]["records_cut_back_into_their_lines"] == 0 + + +def test_a_masked_donor_name_is_counted_and_refused_past_the_share( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + # A platform that began masking names as it masks IDs would fill the + # corpus with masked ones. + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips=_tw_first( + _tw_row("個人捐贈收入", "王*明"), _tw_row("個人捐贈收入", "陳大文"))) + with pytest.raises(ValueError, match="1 of 2 individual_donations are naming " + "a masked donor"): + local.fetch(_TW) + monkeypatch.setattr(local, "TW_MAX_BAD", 1) + corpus = local.fetch(_TW) + assert corpus.records == [{"name": "陳大文"}] + assert corpus.meta["detail"]["donations_naming_a_masked_donor"] == 1 + + +def test_a_tw_zip_with_no_incomes_file_is_refused_and_not_cached( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: asked = _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(None)}) with pytest.raises(ValueError, match="holds no incomes.csv"): local.fetch(_TW) - assert sum("/Search/download" in u for u in asked) == 1 + assert _tw_serials(asked) == ["1"] # not retried + assert not list(_tw_cache(_tw_home).glob("*.zip")) # nor kept to refuse again -def test_a_tw_zip_cut_short_is_fetched_again( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) - cut = [_TW_ZIPS[(_TW_ELECTION, _TW_AREA, 1)][:-30]] - asked = _serve_tw(monkeypatch, fail=lambda url: cut.pop() - if "/Search/download" in url and cut else None) +def _corrupt(body: bytes) -> bytes: + """`body` with a byte of its deflated member flipped.""" + at = body.index(b"incomes.csv") + 40 + return body[:at] + bytes([body[at] ^ 0xFF]) + body[at + 1:] + + +@pytest.mark.parametrize("damage", [lambda b: b[:-30], _corrupt], + ids=["cut short", "a member that does not inflate"]) +def test_a_damaged_tw_zip_is_fetched_again( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, + damage: Callable[[bytes], bytes]) -> None: + bad = [damage(_TW_ZIPS[(_TW_ELECTION, _TW_AREA, 1)])] + asked = _serve_tw(monkeypatch, fail=lambda url: bad.pop() + if "/Search/download" in url and bad else None) assert len(local.fetch(_TW).records) == 3 - assert sum("/Search/download" in u for u in asked) == 3 + assert _tw_serials(asked) == ["1", "1", "2"] -def _tw_downloads(monkeypatch: pytest.MonkeyPatch, - listings: list[list[dict[str, Any]]] | None = None, - zips: dict[tuple[str, str, int], bytes] | None = None) -> list[str]: - """Fetch over _serve_tw; the serial of each report downloaded.""" - asked = _serve_tw(monkeypatch, listings=listings, zips=zips) - local.fetch(_TW) - return [urllib.parse.parse_qs(urllib.parse.urlsplit(u).query)["YearOrSerial"][0] - for u in asked if "/Search/download" in u] +def test_a_listed_tw_report_that_keeps_failing_is_raised( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + # The platform answers a fault with the 500 it gives a report that + # does not exist; the listing says this one does, so it is no absence. + asked = _serve_tw(monkeypatch, fail=lambda url: _http_error(500) + if "/Search/download" in url else None) + with pytest.raises(urllib.error.HTTPError) as raised: + local.fetch(_TW) + raised.value.close() + assert len(_tw_serials(asked)) == local.CTGOV_RETRIES + assert local.load(_TW) is None def test_a_tw_refetch_downloads_only_the_reports_rebuilt_since( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - # A fetch that stops partway, or a refresh, would otherwise ask the - # platform for all 454 reports again. - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) - reports = tmp_path / "local" / _TW / "reports" + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + # A refresh would otherwise download every listed report again. assert _tw_downloads(monkeypatch) == ["1", "2"] assert _tw_downloads(monkeypatch) == [] rebuilt = [_TW_LISTING[0], _tw_report(2, version="113010100")] zips = {**_TW_ZIPS, (_TW_ELECTION, _TW_AREA, 2): _tw_zip(_tw_csv([ - _tw_row("個人捐贈收入", "趙六")]))} + _tw_row("個人捐贈收入", "趙六", _RESIDUAL)]))} assert _tw_downloads(monkeypatch, listings=[rebuilt], zips=zips) == ["2"] - assert sorted(p.name for p in reports.iterdir()) == [ - "election-112091204-1.zip", "election-113010100-2.zip"] + assert sorted(p.name for p in _tw_cache(_tw_home).iterdir()) == [ + "election-112091204-1-63000.zip", "election-113010100-2-63000.zip"] corpus = local.load(_TW) assert corpus is not None and {"name": "趙六"} in corpus.records +def test_a_refused_tw_fetch_keeps_its_zips_and_goes_on_where_it_stopped( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + stale = _tw_cache(_tw_home) / "election-100000000-1-63000.zip" + stale.parent.mkdir(parents=True) + stale.write_bytes(b"a report the listing no longer names") + asked = _serve_tw(monkeypatch, fail=lambda url: _http_error(500) + if "YearOrSerial=2" in url else None) + with pytest.raises(urllib.error.HTTPError) as raised: + local.fetch(_TW) + raised.value.close() + assert stale.exists() # deleted only by a fetch that passes its checks + assert _tw_serials(asked)[0] == "1" + assert _tw_downloads(monkeypatch) == ["2"] + assert not stale.exists() + + +def test_a_tw_refresh_rebuilds_the_corpus_from_the_cached_zips( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + asked = _serve_tw(monkeypatch) + assert local.main(["fetch", _TW]) == 0 + first = (_tw_home / "local" / _TW / "meta.json").read_bytes() + assert local.main(["fetch", _TW, "--refresh"]) == 0 + assert _tw_serials(asked) == ["1", "2"] # each downloaded once + assert sum("/search/elections" in u for u in asked) == 4 # listed again + corpus = local.load(_TW) + assert corpus is not None and len(corpus.records) == 3 + assert json.loads(first)["sha256"] == corpus.meta["sha256"] + + def test_a_cached_tw_zip_cut_short_is_downloaded_again( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, capsys: pytest.CaptureFixture[str]) -> None: - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) _tw_downloads(monkeypatch) - cached = tmp_path / "local" / _TW / "reports" / "election-112091204-1.zip" + cached = _tw_cache(_tw_home) / "election-112091204-1-63000.zip" cached.write_bytes(cached.read_bytes()[:-30]) assert _tw_downloads(monkeypatch) == ["1"] assert cached.read_bytes() == _TW_ZIPS[(_TW_ELECTION, _TW_AREA, 1)] - assert "downloading again a cached zip cut short" in capsys.readouterr().err + assert "downloading again a cached zip that is not whole" in capsys.readouterr().err @pytest.mark.parametrize("name", ["../meta.json", "a/b.zip", "", None]) def test_a_tw_zip_name_that_is_no_file_name_is_refused( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, name: str | None) -> None: + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, name: str | None) -> None: # The name is the platform's, and the cache writes a file under it. - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) _serve_tw(monkeypatch, listings=[[{**_TW_LISTING[0], "zipFileName": name}]]) with pytest.raises(ValueError, match="no file name this tool caches"): local.fetch(_TW) - assert not (tmp_path / "local" / "meta.json").exists() - - -def test_a_listed_tw_report_that_keeps_failing_is_raised( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - # The platform answers a fault with the 500 it gives a report that - # does not exist; the listing says this one does, so it is no absence. - monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) - asked = _serve_tw(monkeypatch, fail=lambda url: _http_error(500) - if "/Search/download" in url else None) - with pytest.raises(urllib.error.HTTPError) as raised: - local.fetch(_TW) - raised.value.close() - assert sum("/Search/download" in u for u in asked) == local.CTGOV_RETRIES - assert local.load(_TW) is None + assert not _tw_cache(_tw_home).exists() # -- load ---------------------------------------------------------------------- diff --git a/tools/differential/README.md b/tools/differential/README.md index 5c74a02fc..e209ab405 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -254,9 +254,9 @@ uv run python tools/differential/snapshot.py --local ctgov-officials |---|---|---| | `ctgov-officials` | every overall official of every study on ClinicalTrials.gov, whatever the role (principal investigators, study directors and chairs, and a few sub-investigators and officials with no role), the name as the sponsor's staff typed it into a free-text field: credentials run together (`MBBS,FCPS`), job titles in the name, all-caps and all-lowercase names, typos. Fetched from API v2 (NLM) | names outside clinical research, which is mostly medicine and heavily credentialed; names nobody typed by hand | | `fec-individuals` | the distinct names of every individual contributor in the FEC's bulk file for the 2026 cycle (`indiv26.zip`, downloaded once into the cache and read without unpacking; `fetch --refresh` downloads it again). Mostly a filer's software writing a form's fields in a fixed order: of its 1,893,964 names on 2026-10-18 every one was in capitals and 1,893,961 held a comma, with hand-typed text inside the fields -- 3,306 held a second comma, for a credential run or a suffix in a part of its own (`DOE, JR., JOHN`). The population for the one-case family-comma shape. The FEC's terms (52 U.S.C. 30111(a)(4)) bar using the names to solicit contributions or for any commercial purpose | case, since every name is in capitals; names written in their spoken order, or without a comma; small donors, since a committee need itemize only a donor whose gifts to it pass $200; names outside the US; names in any script but Latin | -| `tw-donors` | the distinct names of every individual donor in every campaign finance report Taiwan's Control Yuan publishes on its political donation platform (ardata.cy.gov.tw, whose zips data.gov.tw also lists): 454 reports of 64 elections on 2026-10-22, presidential, legislative and local: every report from 2018's elections on, and the residual reports of 2014's and 2016's. Each report's zip is cached under the name the platform lists it by, which carries the date it was built, so a refresh downloads only the reports published or rebuilt since. Names as a candidate's staff typed them into a filing, published in full where the ID, address and phone beside them are masked. Of the 308,653 names dated 2026-10-05, 296,803 were three Han characters, 5,449 four and 5,260 two; the population for unspaced Han names written family first, and for the married double surname (冠夫姓) a four-character name often is. Under the Open Government Data License, version 1.0, with the platform's own bar on using the data for profit, credit investigation or another improper purpose | names in spoken order; Latin script (80 names held a Latin letter); the comma (3 held a fullwidth one, none an ASCII one); case; names outside Taiwan; donors whose names the report does not publish, such as anonymous gifts; Japanese kanji names, which the Han script cannot tell from Chinese | +| `tw-donors` | the distinct names of every individual donor in every campaign finance report Taiwan's Control Yuan publishes on its political donation platform (ardata.cy.gov.tw, whose zips data.gov.tw also lists): 454 reports of 64 elections on 2026-10-22, presidential, legislative and local: every report from 2018's elections on, and the residual reports of 2014's and 2016's. Each report's zip is cached under the name the platform lists it by, which carries the date it was built, so a refresh downloads only the reports published or rebuilt since. Names as a candidate's staff typed them into a filing, published in full where the ID, address and phone beside them are masked. Of the 308,663 names dated 2026-10-05, 296,816 were three Han characters, 5,449 four and 5,260 two; the population for unspaced Han names written family first, and for the married double surname (冠夫姓) a four-character name often is. Under the Open Government Data License, version 1.0, with the platform's own bar on using the data for profit, credit investigation or another improper purpose | names in spoken order; Latin script (80 of the names dated 2026-10-05 held a Latin letter); the comma (3 of them held a fullwidth one, none an ASCII one); case; names outside Taiwan; donors whose names the report does not publish, such as anonymous gifts; Japanese kanji names, which the Han script cannot tell from Chinese | -A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. `tools/name_usage.py` looks a word up in each corpus's word index, cached as `wordindex.json` beside it on first use and keyed to the records' sha256 and to the code that cuts and folds a word, so a refetch or a change to the fold rebuilds it; a single-word lookup it answers parses neither the records nor the names. Measured 2026-10-22 with both corpora fetched, a single-word `name_usage.py --no-wikidata` run took 2.5s with their indexes cached (12.5s before the cache, 14.8s on the run that builds it). A spaced word (`Ph. D.`) still scans every name, which took 8.6s, nearly all of it the FEC corpus's. `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. `tw-donors` keeps one zip per report under `reports/`, whole, its masked ID, address and phone columns included, as the FEC's zip is kept: the platform's download answers a report that does not exist and a fault alike, with a 500 and an incident number, so the fetch reads which reports exist from the platform's listing and refuses a listed report it cannot download. It reads each file's columns by name, since a 2020 report lacks a column the newest ones write, and refuses a column it does not know, a category it does not know, or a listing that changes during the fetch. A layout check written from one surveyed file meets every other file only at fetch time: `tw-donors`' first three fetches were each refused partway, by a report with no income writing one row of dashes, by a 2020 report with 24 columns where the surveyed one had 25, and by a row with no category. Each refusal cost every download again until the source cached its zips, so a new source that downloads many files caches each one before its checks run. The fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. +A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. `tools/name_usage.py` looks a word up in each corpus's word index, cached as `wordindex.json` beside it on first use and keyed to the records' sha256 and to the code that cuts and folds a word, so a refetch or a change to the fold rebuilds it; a single-word lookup it answers parses neither the records nor the names. Measured 2026-10-22 with both corpora fetched, a single-word `name_usage.py --no-wikidata` run took 2.5s with their indexes cached (12.5s before the cache, 14.8s on the run that builds it). A spaced word (`Ph. D.`) still scans every name, which took 8.6s, nearly all of it the FEC corpus's. `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. The `fec-individuals` fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. `tw-donors` keeps one zip per report under `reports/`, whole, its masked ID, address and phone columns included, as the FEC's zip is kept: the platform's download answers a report that does not exist and a fault alike, with a 500 and an incident number, so the fetch reads which reports exist from the platform's listing and refuses a listed report it cannot download. It reads each file's columns by name, since a 2020 report lacks a column the newest ones write, and refuses a column it does not know, a category it does not know, a row whose serial column names another report (the zip is not the one asked for), a listing that changes during the fetch, and a report more than half of whose rows it reads past. It reads back the rows a stray quote took into one field (157 in two 2020 reports on 2026-10-22), counts donor cells that hold no name (匿名, a dash) or a masked one, and caches a downloaded zip only once its rows are read, so a refused zip is asked for again. A zip the platform rebuilt under the name it had is not seen: delete `reports/` to download every report again. A layout check written from one surveyed file meets every other file only at fetch time: `tw-donors`' first three fetches were each refused partway, by a report with no income writing one row of dashes, by a 2020 report with 24 columns where the surveyed one had 25, and by a row with no category. Each refusal cost every download again until the source cached its zips, so a new source that downloads many files caches each one before its checks run. ## Corpus provenance diff --git a/tools/differential/local_corpora.py b/tools/differential/local_corpora.py index 6031fb3a0..c9af105a7 100644 --- a/tools/differential/local_corpora.py +++ b/tools/differential/local_corpora.py @@ -41,7 +41,9 @@ a later fetch may differ. A new source is a SOURCES row: a fetch function returning the records and the source's data date, and for a source that caches its own input, the hooks that renew it on a -refresh and refuse a corpus read from another version of it. +refresh and refuse a corpus read from another version of it (tw-donors +needs neither: each zip it caches is named by the version the listing +gives). ctgov-officials -- every overall official of every study on ClinicalTrials.gov, whatever the role (principal investigators, study @@ -75,11 +77,13 @@ donation platform (ardata.cy.gov.tw, the zips data.gov.tw lists): 454 reports of 64 elections on 2026-10-22, presidential, legislative and local: every report from 2018's elections on, and the residual reports -of 2014's and 2016's. Each report's zip is cached whole beside the -corpus (40 MB), under the name the platform's listing gives it, so a -refresh downloads only the reports published or rebuilt since. Names as a candidate's staff typed them into a filing, published -in full where the ID, address and phone beside them are masked: of the -308,653 on 2026-10-05, 296,803 were three Han characters, 5,449 four +of 2014's and 2016's. Each report's zip is cached whole in a reports/ +folder beside the corpus (40 MB on 2026-10-22), under the name the +platform's listing gives it, so a refresh downloads only the reports +published or rebuilt since. Names as a candidate's staff typed them +into a filing, published in full where the ID, address and phone +beside them are masked: of the +308,663 on 2026-10-05, 296,816 were three Han characters, 5,449 four and 5,260 two, so it is the population for unspaced Han names written family first, and for the married double surname (冠夫姓) a four- character name often is. Under the Open Government Data License, @@ -109,6 +113,7 @@ import urllib.parse import urllib.request import zipfile +import zlib from collections import Counter from collections.abc import Callable, Iterator from dataclasses import dataclass, field @@ -232,9 +237,9 @@ class _Unparsed(Exception): def _get_json(url: str) -> dict[str, Any]: - """`url`'s JSON object, fetched by _retried: one page per thousand - studies (608 on 2026-10-21), and starting over would cost the whole - run.""" + """`url`'s JSON object, fetched by _retried: a long paged run + (ClinicalTrials.gov's 608 pages on 2026-10-21) should not start over + for one failed page.""" data = _retried(url, lambda body: corpus_build.parse_json(body, url), 0.1) if not isinstance(data, dict): raise ValueError(f"{url} answered {type(data).__name__}, not an object") @@ -243,10 +248,10 @@ def _get_json(url: str) -> dict[str, Any]: def _retried(url: str, read: Callable[[bytes], _T], delay: float) -> _T: """`url`'s body as `read` reads it, retried with a backoff and a line - on stderr for each retry, `read` raising ValueError for a body cut - short in transit; `delay` seconds of sleep after each request. Any - other 4xx is the request's fault and is raised, as is the last - attempt's error.""" + on stderr for each retry; any ValueError from `read` is retried as a + body cut short in transit, and `delay` seconds of sleep follow each + request. A 4xx other than 429 is the request's fault and is raised, + as is the last attempt's error.""" for attempt in range(CTGOV_RETRIES): try: body = corpus_build.fetch_bytes(url, user_agent=USER_AGENT, @@ -254,7 +259,7 @@ def _retried(url: str, read: Callable[[bytes], _T], delay: float) -> _T: try: return read(body) except ValueError as exc: - raise _Unparsed(str(exc)) from None + raise _Unparsed(f"{url}: {exc}") from None except (*_TRANSIENT, _Unparsed) as exc: if (isinstance(exc, urllib.error.HTTPError) and exc.code < 500 and exc.code != 429 or attempt == CTGOV_RETRIES - 1): @@ -488,7 +493,13 @@ def check_fec(meta: dict[str, Any]) -> str: "存入專戶日期", "返還/繳庫", "支出用途", "金錢類", "地址", "聯絡電話", "應揭露之支出對象", "支出對象之內部人員姓名", "支出對象之內部人員職稱", "政黨之內部人員姓名", "政黨之內部人員職稱", "關係", "更正註記", "資料更正日期") -TW_CATEGORY, TW_DONOR = "收支科目", "捐贈者／支出對象" +TW_CATEGORY, TW_DONOR, TW_SERIAL = "收支科目", "捐贈者／支出對象", "申報序號／年度" +#: The candidate's column, the one a report with no income fills in +#: the single row it writes, every other column holding TW_EMPTY. +TW_CANDIDATE = "擬參選人／政黨" +#: The columns every file must hold, the ones read. +TW_NEEDED = frozenset({TW_CATEGORY, TW_DONOR, TW_SERIAL, TW_CANDIDATE}) +TW_EMPTY = "-" TW_INDIVIDUAL = "個人捐贈收入" #: The income categories a candidate's report holds (the platform's #: classTypes with ctrlTp 0): individual, business, party and civic @@ -496,28 +507,40 @@ def check_fec(meta: dict[str, Any]) -> str: TW_CATEGORIES = frozenset({ TW_INDIVIDUAL, "營利事業捐贈收入", "政黨捐贈收入", "人民團體捐贈收入", "匿名捐贈", "其他收入"}) -#: The candidate's column, the one a report with no income fills in -#: the single row it writes, every other column holding TW_EMPTY. -TW_CANDIDATE = "擬參選人／政黨" -#: The columns every file must hold, the ones read. -TW_NEEDED = frozenset({TW_CATEGORY, TW_DONOR, TW_CANDIDATE}) -TW_EMPTY = "-" #: A report's serial: 1 the first report, 2 to 6 the five reports on #: the money left over. TW_SERIALS = range(1, 7) -#: More than this share of rows not as long as their file's header, or -#: of rows with an empty category, refuses the fetch, as FEC_MAX_BAD -#: does. A row with no category is read past: one 2021 residual report -#: held one, an income of NT$53 with no donor ID (2026-10-22). +_TW_NUMERALS = "一二三四五" +#: A donor cell that holds no name: anonymous (19 individual donations +#: wrote it on 2026-10-22) or a dash. +TW_NO_NAME = frozenset({"匿名", TW_EMPTY}) +#: A character that masks part of a name (3 donor names held an +#: asterisk on 2026-10-22): such a name is counted, not kept. +_TW_MASK = re.compile(r"[*＊○]") +#: More than this share of the fetch's rows not as long as their file's +#: header, without a category, or cut off by a stray quote, or of its +#: individual donations naming a masked donor, refuses the fetch, as +#: FEC_MAX_BAD does. One 2021 residual report held a row with no +#: category, an income of NT$53 with no donor ID (2026-10-22). TW_MAX_BAD = 1 / 10_000 +#: More than this share of ONE report's rows bad refuses the fetch too: +#: the share over the whole fetch would let a small report's rows all +#: go (the worst report on 2026-10-22 had 1 bad row of 6). +TW_MAX_REPORT_BAD = 1 / 2 +#: What a row is counted as when it is read past without a name. +_TW_BAD = ("rows_not_as_long_as_the_header", "rows_without_a_category", + "rows_cut_by_a_stray_quote") def _tw_reports() -> list[dict[str, Any]]: """Every election report the platform lists, paged: the listing is what says a report exists, since the download answers a report that - does not and a fault alike, with a 500 and an incident number.""" + does not and a fault alike, with a 500 and an incident number. Two + reports listed under one zip name are refused, since the cache keeps + a zip by its name.""" reports: list[dict[str, Any]] = [] seen: set[tuple[str, str, int]] = set() + zips: set[object] = set() page, count = 1, None while count is None or len(reports) < count: query = urllib.parse.urlencode({"page": page, "pageSize": TW_PAGE}) @@ -535,7 +558,11 @@ def _tw_reports() -> list[dict[str, Any]]: if key in seen: raise ValueError(f"{key} was listed twice: the listing shifted " f"under the fetch; fetch again") + if row["zipFileName"] in zips: + raise ValueError(f"{key} is listed under the zip name " + f"{row['zipFileName']!r} another report has") seen.add(key) + zips.add(row["zipFileName"]) reports.append(row) page += 1 if len(reports) != count: @@ -560,10 +587,19 @@ def _tw_url(report: dict[str, Any]) -> str: "SearchType": 2, "DownloadType": 3}) +def _tw_serial_names(serial: int) -> frozenset[str]: + """How a report's rows write its serial: 首次申報 for the first, 第N次賸餘 + for a residual one, N in either numerals (both are written).""" + if serial == 1: + return frozenset({"首次申報"}) + return frozenset({f"第{serial - 1}次賸餘", f"第{_TW_NUMERALS[serial - 2]}次賸餘"}) + + def _tw_incomes(body: bytes) -> tuple[bytes, str] | None: """A report zip's incomes.csv and its timestamp, None where the zip - holds none, or ValueError where the body is no whole zip: a download - cut short, which _retried retries.""" + holds none, or ValueError where the body is no whole zip or its + member does not inflate: a download cut short or damaged, which + _retried retries.""" try: with zipfile.ZipFile(io.BytesIO(body)) as archive: if TW_INCOMES not in archive.namelist(): @@ -571,8 +607,10 @@ def _tw_incomes(body: bytes) -> tuple[bytes, str] | None: member = archive.getinfo(TW_INCOMES) stamp = "{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}".format(*member.date_time) return archive.read(member), stamp - except zipfile.BadZipFile as exc: - raise ValueError(f"not a whole zip ({exc})") from None + except (zipfile.BadZipFile, zlib.error, EOFError) as exc: + start = body[:2] + raise ValueError(f"not a whole zip ({type(exc).__name__}: {exc}; it starts " + f"{start!r})") from None #: The cached reports' folder, beside the corpus. @@ -587,74 +625,143 @@ def _tw_whole(body: bytes) -> bytes: return body -def _tw_report_zip(report: dict[str, Any], label: str) -> bytes: - """A listed report's zip, read from the reports folder where an - earlier fetch cached it under the name the listing gives, else - downloaded and cached there. The name carries the date the platform - built the zip, so a rebuilt report is downloaded anew, and a cached - zip that is no whole zip is too.""" +def _tw_cached(report: dict[str, Any]) -> Path: + return directory("tw-donors") / TW_REPORTS / report["zipFileName"] + + +def _tw_report_zip(report: dict[str, Any], + label: str) -> tuple[bytes, str, bytes | None]: + """A listed report's incomes.csv, its timestamp, and the zip when it + was downloaded rather than read from the reports folder, where an + earlier fetch cached it under the name the listing gives. The name + carries the date the platform built the zip, so a rebuilt report is + downloaded anew, and a cached zip that is no whole zip is too. A zip + holding no incomes.csv is refused without being retried: a layout + the tool was not written for, or one bad answer. The caller caches a + downloaded zip only once its rows are read (_tw_cache_zip), since a + cached zip that is refused would refuse every later fetch.""" name = report["zipFileName"] if not isinstance(name, str) or not _TW_ZIP_NAME.fullmatch(name): raise ValueError(f"{label} is listed with the zip name {name!r}, which is " f"no file name this tool caches; the listing has changed") - path = directory("tw-donors") / TW_REPORTS / name + path = _tw_cached(report) if path.exists(): - body = path.read_bytes() try: - return _tw_whole(body) + if got := _tw_incomes(path.read_bytes()): + return (*got, None) except ValueError: - print(f" downloading again a cached zip cut short: {path}", file=sys.stderr) + print(f" downloading again a cached zip that is not whole: {path}", + file=sys.stderr) body = _retried(_tw_url(report), _tw_whole, TW_DELAY) + if (got := _tw_incomes(body)) is None: + raise ValueError(f"{label}'s zip holds no {TW_INCOMES}: the reports' " + f"layout has changed") + return (*got, body) + + +def _tw_cache_zip(report: dict[str, Any], body: bytes) -> None: + path = _tw_cached(report) path.parent.mkdir(exist_ok=True) corpus_build.write_atomic(path, body) - return body -def _tw_read(incomes: bytes, label: str, names: set[str], counts: Counter[str]) -> None: +def _tw_lines(text: str) -> list[str]: + """`text` cut into its lines at "\\n" alone, each keeping its ending: + str.splitlines() would cut at U+2028 and the other separators, which + a name may hold (corpus_build.jsonl_lines).""" + return re.findall(r"[^\n]*\n|[^\n]+$", text) + + +def _tw_rows(text: str, width: int, counts: Counter[str]) -> Iterator[list[str]]: + """incomes.csv's records after its header. A record whose field + holds line breaks is a quoted field spanning lines, or a stray quote + opening a field and taking the rows after it in until another + closes it: two 2020 reports held four, which took 157 rows in + (2026-10-22). Where one of the lines after its first reads as a + whole row alone, the record is cut back into its lines, each read + alone: the lines as wide as the header are rows, and the rest are + counted as cut by the quote. Otherwise it stays one record (a phone + field holding two numbers on two lines, 8 on 2026-10-22).""" + lines = _tw_lines(text) + reader = csv.reader(lines) + next(reader, None) + start = reader.line_num + for record in reader: + end = reader.line_num + if end - start > 1: + alone = [next(csv.reader([line]), []) for line in lines[start:end]] + if any(len(row) == width for row in alone[1:]): + counts["records_cut_back_into_their_lines"] += 1 + for row in alone: + if len(row) == width: + yield row + else: + counts["rows_cut_by_a_stray_quote"] += 1 + start = end + continue + start = end + yield record + + +def _tw_read(incomes: bytes, report: dict[str, Any], label: str, names: set[str], + counts: Counter[str]) -> None: """Add a report's donor names to `names`, counting into `counts` what is read past. A header or a category this tool was not written for is refused, not read: a moved column would put another field's values in the corpus, so a header holding a column TW_HEADER does not name, a column twice, or not every column TW_NEEDED names is - refused; one missing a column the tool does not read is read. The - one row a report with no income writes, every column a dash but the - candidate's, is counted apart.""" + refused; one missing a column the tool does not read is read. A row + whose serial is not the report's is refused: the zip is another + report's. The one row a report with no income writes, every column a + dash but the candidate's, is counted apart.""" try: text = incomes.decode("utf-8-sig") except UnicodeDecodeError as exc: raise ValueError(f"{label}'s {TW_INCOMES} is not UTF-8 ({exc})") from None - rows = csv.reader(io.StringIO(text, newline="")) - header = next(rows, []) + header = next(csv.reader(_tw_lines(text)), []) if (not set(header) <= set(TW_HEADER) or len(set(header)) != len(header) or not TW_NEEDED <= set(header)): raise ValueError(f"{label}'s {TW_INCOMES} has the header {tuple(header)}, " f"which is not columns of TW_HEADER, each once, holding " f"those TW_NEEDED names") - category_at, donor_at, candidate_at = ( - header.index(column) for column in (TW_CATEGORY, TW_DONOR, TW_CANDIDATE)) - for row in rows: - counts["rows"] += 1 + category_at, donor_at, serial_at, candidate_at = ( + header.index(column) for column in (TW_CATEGORY, TW_DONOR, TW_SERIAL, TW_CANDIDATE)) + serials = _tw_serial_names(int(report["yearOrSerial"])) + mine = Counter[str]() + for row in _tw_rows(text, len(header), mine): + mine["rows"] += 1 if len(row) != len(header): - counts["rows_not_as_long_as_the_header"] += 1 + mine["rows_not_as_long_as_the_header"] += 1 continue - category = row[category_at] if {f for i, f in enumerate(row) if i != candidate_at} == {TW_EMPTY}: - counts["rows_of_a_report_with_no_income"] += 1 + mine["rows_of_a_report_with_no_income"] += 1 continue + category = row[category_at] if not category: - counts["rows_without_a_category"] += 1 + mine["rows_without_a_category"] += 1 continue if category not in TW_CATEGORIES: raise ValueError(f"{label}'s {TW_INCOMES} holds the category " f"{category!r}, which TW_CATEGORIES does not list") + if row[serial_at] not in serials: + raise ValueError(f"{label}'s {TW_INCOMES} holds a row of the report " + f"{row[serial_at]!r}: the zip is another report's") if category != TW_INDIVIDUAL: - counts["rows_from_other_categories"] += 1 + mine["rows_from_other_categories"] += 1 continue - counts["individual_donations"] += 1 - if name := row[donor_at].strip(): - names.add(name) + mine["individual_donations"] += 1 + name = row[donor_at].strip() + if not name or name in TW_NO_NAME: + mine["donations_without_a_name"] += 1 + elif _TW_MASK.search(name): + mine["donations_naming_a_masked_donor"] += 1 else: - counts["donations_without_a_name"] += 1 + names.add(name) + bad = sum(mine[key] for key in _TW_BAD) + if bad > (mine["rows"] + mine["rows_cut_by_a_stray_quote"]) * TW_MAX_REPORT_BAD: + raise ValueError(f"{bad:,} of {label}'s rows are read past as malformed: " + f"the file's layout has changed") + counts.update(mine) def fetch_tw() -> Fetched: @@ -666,14 +773,19 @@ def fetch_tw() -> Fetched: gave to, as the FEC source keeps one per name. Only the donor column of the rows the category marks as individual donations is kept, its spaces stripped; the masked ID, address and phone columns and the - amounts are read past. Each report's zip is cached (_tw_report_zip), - so a fetch cut short or refused goes on where it stopped, and a - refresh downloads only the reports published or rebuilt since; a - cached zip the listing no longer names is deleted once the fetch has - read the rest. The listing is read again at the end, and a - report published or replaced during the fetch refuses it. The data - date is the newest incomes.csv's own timestamp, which carries no time - zone.""" + amounts are read past, and a donor cell that holds no name or a + masked one is counted. Each report's zip is cached + (_tw_report_zip), so a fetch cut short or refused goes on where it + stopped, and a refresh downloads only the reports published or + rebuilt since: a zip rebuilt under the name it had is not seen, and + deleting the reports folder downloads every one again. A cached zip + the listing no longer names is deleted at the end of a fetch that + passes its checks, and a downloaded zip is cached only once its rows + are read, so a zip refused is downloaded again rather than refused + from the cache for good. The listing is read again at the end, and a + report published, replaced or withdrawn during the fetch refuses it. + The data date is the newest incomes.csv's own timestamp, which + carries no time zone.""" reports = _tw_reports() names: set[str] = set() counts: Counter[str] = Counter() @@ -683,35 +795,35 @@ def fetch_tw() -> Fetched: print(f" {number:,} of {len(reports):,} reports", file=sys.stderr) label = f"{report['electionName']} {report['electionArea']} " \ f"report {int(report['yearOrSerial'])}" - got = _tw_incomes(_tw_report_zip(report, label)) - if got is None: - raise ValueError(f"{label}'s zip holds no {TW_INCOMES}: the reports' " + incomes, stamp, downloaded = _tw_report_zip(report, label) + stamps.append(stamp) + _tw_read(incomes, report, label, names, counts) + if downloaded is not None: + _tw_cache_zip(report, downloaded) + for key, total in [(key, "rows") for key in _TW_BAD] + [ + ("donations_naming_a_masked_donor", "individual_donations")]: + if counts[key] > counts[total] * TW_MAX_BAD: + raise ValueError(f"{counts[key]:,} of {counts[total]:,} {total} are " + f"{key.split('_', 1)[1].replace('_', ' ')}: the files' " f"layout has changed") - stamps.append(got[1]) - _tw_read(got[0], label, names, counts) - for key, what in (("rows_not_as_long_as_the_header", - "not as long as their file's header"), - ("rows_without_a_category", "without a category")): - if counts[key] > counts["rows"] * TW_MAX_BAD: - raise ValueError(f"{counts[key]:,} of {counts['rows']:,} rows are " - f"{what}: the files' layout has changed") if _tw_versions(_tw_reports()) != _tw_versions(reports): - raise ValueError("the platform published or replaced a report during " - "the fetch; fetch again") + raise ValueError("the platform published, replaced or withdrew a report " + "during the fetch; fetch again") listed = {r["zipFileName"] for r in reports} for stale in (directory("tw-donors") / TW_REPORTS).glob("*.zip"): if stale.name not in listed: stale.unlink() - return Fetched([{"name": name} for name in sorted(names)], max(stamps, default=""), { + detail: dict[str, Any] = { "reports": len(reports), "first_reports": sum(int(r["yearOrSerial"]) == 1 for r in reports), - "elections": len({r["electionName"] for r in reports}), - "rows": counts["rows"], "individual_donations": counts["individual_donations"], - "rows_from_other_categories": counts["rows_from_other_categories"], - "donations_without_a_name": counts["donations_without_a_name"], - "rows_of_a_report_with_no_income": counts["rows_of_a_report_with_no_income"], - "rows_without_a_category": counts["rows_without_a_category"], - "rows_not_as_long_as_the_header": counts["rows_not_as_long_as_the_header"]}) + "elections": len({r["electionName"] for r in reports})} + for key in ("rows", "individual_donations", "rows_from_other_categories", + "donations_without_a_name", "donations_naming_a_masked_donor", + "rows_of_a_report_with_no_income", "records_cut_back_into_their_lines", + *_TW_BAD): + detail[key] = counts[key] + return Fetched([{"name": name} for name in sorted(names)], max(stamps, default=""), + detail) SOURCES: dict[str, Source] = { From f70825bbf886bc3e84261ccbc2af75206d076182 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 22 Oct 2026 21:18:54 -0700 Subject: [PATCH 3/4] tw-donors: split lines as csv does, recover paired stray quotes, name an unreadable CSV's report Second review of #844, on the first round's fixes: - Cutting lines at LF alone made a bare CR, which csv over a file opened with newline="" reads as a line end, raise csv.Error naming no report where it had been a counted short row. Lines are cut at CRLF, CR or LF again. - Two stray quotes in rows one after the other close each other and make one record exactly as wide as the header out of two rows, so a donor went uncounted. A record spanning lines whose every line reads as a whole row with quotes taken literally is cut back into them. - csv.Error is refused as a ValueError naming the report. - The fetch-wide share messages read as sentences. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_local_corpora.py | 58 ++++++++++++++++++++++++++--- tools/differential/local_corpora.py | 43 +++++++++++++++------ 2 files changed, 85 insertions(+), 16 deletions(-) diff --git a/tests/v2/test_local_corpora.py b/tests/v2/test_local_corpora.py index 1e87d2e07..5a38d6186 100644 --- a/tests/v2/test_local_corpora.py +++ b/tests/v2/test_local_corpora.py @@ -891,17 +891,65 @@ def test_rows_a_stray_quote_takes_in_are_read_back( _line("趙六", phone='02"')] _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_incomes_text(lines))}) + corpus = local.fetch(_TW) # every line whole with quotes taken literally + assert len(corpus.records) == 4 + assert corpus.meta["detail"]["records_cut_back_into_their_lines"] == 1 + assert corpus.meta["detail"]["rows_cut_by_a_stray_quote"] == 0 + # A line between holding a quoted comma is no whole row read + # literally: the lines are read alone as quoted, the opening line cut. + lines[2] = _line("林一二", phone='"02,03"') + for cached in _tw_cache(_tw_home).iterdir(): + cached.unlink() + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_incomes_text(lines))}) monkeypatch.setattr(local, "TW_MAX_BAD", 1) corpus = local.fetch(_TW) assert {"name": "陳大文"} in corpus.records and {"name": "林一二"} in corpus.records - detail = corpus.meta["detail"] - assert detail["records_cut_back_into_their_lines"] == 1 - assert detail["rows_cut_by_a_stray_quote"] >= 1 + assert corpus.meta["detail"]["rows_cut_by_a_stray_quote"] >= 1 monkeypatch.setattr(local, "TW_MAX_BAD", 1 / 10_000) with pytest.raises(ValueError, match="rows are cut by a stray quote"): local.fetch(_TW) +def test_two_stray_quotes_one_row_after_another_lose_neither_row( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + # The second quote closes the first: one record as wide as the + # header out of two rows, which no line read alone as quoted shows. + lines = [_line("王小明", address='"臺北市'), _line("陳大文", address='"臺南市'), + _line("林一二")] + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_incomes_text(lines))}) + corpus = local.fetch(_TW) + assert corpus.records == [{"name": "林一二"}, {"name": "王小明"}, {"name": "陳大文"}] + assert corpus.meta["detail"]["records_cut_back_into_their_lines"] == 1 + + +@pytest.mark.parametrize("ending", ["\r", "\r\n", "\n"], ids=["CR", "CRLF", "LF"]) +def test_a_bare_cr_ends_a_line_as_csv_reads_one( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path, ending: str) -> None: + # A stray CR in a cell cuts its row in two short rows, counted, as + # csv.reader over a file opened with newline="" reads it, and a file + # whose lines end in CR alone reads whole. + text = ending.join([",".join(local.TW_HEADER), _line("王小明"), + _line("陳大文", address="臺北\r市"), _line("林一二")]) + ending + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(text.encode("utf-8-sig"))}) + monkeypatch.setattr(local, "TW_MAX_BAD", 1) + corpus = local.fetch(_TW) + assert corpus.records == [{"name": "林一二"}, {"name": "王小明"}] + assert corpus.meta["detail"]["rows_not_as_long_as_the_header"] == 2 + + +def test_a_tw_file_csv_cannot_read_is_refused_naming_the_report( + monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: + huge = _line("王小明", address='"' + "x" * 200_000) # past csv's field limit + text = f"{','.join(local.TW_HEADER)}\r\n{huge}\r\n" + _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], + zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(text.encode("utf-8-sig"))}) + with pytest.raises(ValueError, match="report 1's incomes.csv is not a CSV file"): + local.fetch(_TW) + + def test_a_quoted_field_spanning_two_lines_stays_one_row( monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: # Two phone numbers on two lines, as 8 records held on 2026-10-22. @@ -919,8 +967,8 @@ def test_a_masked_donor_name_is_counted_and_refused_past_the_share( # corpus with masked ones. _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips=_tw_first( _tw_row("個人捐贈收入", "王*明"), _tw_row("個人捐贈收入", "陳大文"))) - with pytest.raises(ValueError, match="1 of 2 individual_donations are naming " - "a masked donor"): + with pytest.raises(ValueError, match="1 of 2 individual donations name a " + "masked donor"): local.fetch(_TW) monkeypatch.setattr(local, "TW_MAX_BAD", 1) corpus = local.fetch(_TW) diff --git a/tools/differential/local_corpora.py b/tools/differential/local_corpora.py index c9af105a7..545a39964 100644 --- a/tools/differential/local_corpora.py +++ b/tools/differential/local_corpora.py @@ -666,10 +666,11 @@ def _tw_cache_zip(report: dict[str, Any], body: bytes) -> None: def _tw_lines(text: str) -> list[str]: - """`text` cut into its lines at "\\n" alone, each keeping its ending: - str.splitlines() would cut at U+2028 and the other separators, which - a name may hold (corpus_build.jsonl_lines).""" - return re.findall(r"[^\n]*\n|[^\n]+$", text) + """`text` cut into its lines at CRLF, CR or LF, each keeping its + ending, as csv reads a file opened with newline="" and no further: + str.splitlines() would cut at U+2028 and the other separators too, + which a name may hold (corpus_build.jsonl_lines).""" + return re.findall(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+$", text) def _tw_rows(text: str, width: int, counts: Counter[str]) -> Iterator[list[str]]: @@ -681,7 +682,11 @@ def _tw_rows(text: str, width: int, counts: Counter[str]) -> Iterator[list[str]] whole row alone, the record is cut back into its lines, each read alone: the lines as wide as the header are rows, and the rest are counted as cut by the quote. Otherwise it stays one record (a phone - field holding two numbers on two lines, 8 on 2026-10-22).""" + field holding two numbers on two lines, 8 on 2026-10-22). Two stray + quotes in rows one after the other close each other and make one + record as wide as the header out of two rows, which no line read + alone as quoted shows: where every line of a record reads as a whole + row with quotes taken literally, those are its rows.""" lines = _tw_lines(text) reader = csv.reader(lines) next(reader, None) @@ -689,6 +694,13 @@ def _tw_rows(text: str, width: int, counts: Counter[str]) -> Iterator[list[str]] for record in reader: end = reader.line_num if end - start > 1: + literal = [next(csv.reader([line], quoting=csv.QUOTE_NONE), []) + for line in lines[start:end]] + if all(len(row) == width for row in literal): + counts["records_cut_back_into_their_lines"] += 1 + yield from literal + start = end + continue alone = [next(csv.reader([line]), []) for line in lines[start:end]] if any(len(row) == width for row in alone[1:]): counts["records_cut_back_into_their_lines"] += 1 @@ -728,7 +740,12 @@ def _tw_read(incomes: bytes, report: dict[str, Any], label: str, names: set[str] header.index(column) for column in (TW_CATEGORY, TW_DONOR, TW_SERIAL, TW_CANDIDATE)) serials = _tw_serial_names(int(report["yearOrSerial"])) mine = Counter[str]() - for row in _tw_rows(text, len(header), mine): + try: + rows = list(_tw_rows(text, len(header), mine)) + except csv.Error as exc: + raise ValueError(f"{label}'s {TW_INCOMES} is not a CSV file csv can " + f"read ({exc})") from None + for row in rows: mine["rows"] += 1 if len(row) != len(header): mine["rows_not_as_long_as_the_header"] += 1 @@ -800,12 +817,16 @@ def fetch_tw() -> Fetched: _tw_read(incomes, report, label, names, counts) if downloaded is not None: _tw_cache_zip(report, downloaded) - for key, total in [(key, "rows") for key in _TW_BAD] + [ - ("donations_naming_a_masked_donor", "individual_donations")]: + for key, total, what in ( + ("rows_not_as_long_as_the_header", "rows", + "rows are not as long as their file's header"), + ("rows_without_a_category", "rows", "rows are without a category"), + ("rows_cut_by_a_stray_quote", "rows", "rows are cut by a stray quote"), + ("donations_naming_a_masked_donor", "individual_donations", + "individual donations name a masked donor")): if counts[key] > counts[total] * TW_MAX_BAD: - raise ValueError(f"{counts[key]:,} of {counts[total]:,} {total} are " - f"{key.split('_', 1)[1].replace('_', ' ')}: the files' " - f"layout has changed") + raise ValueError(f"{counts[key]:,} of {counts[total]:,} {what}: the " + f"files' layout has changed") if _tw_versions(_tw_reports()) != _tw_versions(reports): raise ValueError("the platform published, replaced or withdrew a report " "during the fetch; fetch again") From 88546442797483690ca067b471a01a9e64509e51 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 22 Oct 2026 21:48:40 -0700 Subject: [PATCH 4/4] tw-donors: the 8 one-break records were rows two stray quotes joined; recount Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_local_corpora.py | 6 ++++-- tools/differential/README.md | 4 ++-- tools/differential/local_corpora.py | 24 ++++++++++++------------ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/v2/test_local_corpora.py b/tests/v2/test_local_corpora.py index 2cfde8c52..2ac4c2713 100644 --- a/tests/v2/test_local_corpora.py +++ b/tests/v2/test_local_corpora.py @@ -885,7 +885,7 @@ def _line(donor: str, address: str = "臺北市", phone: str = "02") -> str: def test_rows_a_stray_quote_takes_in_are_read_back( monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: # An address opening with a quote nothing closes until a later row: - # the rows between, two 2020 reports' 157 on 2026-10-22, would go + # the rows between, 169 in three reports on 2026-10-22, would go # into one field and their donors with them. lines = [_line("王小明", address='"臺北市'), _line("陳大文"), _line("林一二"), _line("趙六", phone='02"')] @@ -952,7 +952,9 @@ def test_a_tw_file_csv_cannot_read_is_refused_naming_the_report( def test_a_quoted_field_spanning_two_lines_stays_one_row( monkeypatch: pytest.MonkeyPatch, _tw_home: Path) -> None: - # Two phone numbers on two lines, as 8 records held on 2026-10-22. + # A quoted field that truly spans lines, its second line no row + # (none was found on 2026-10-22: all 12 multi-line records were + # rows a stray quote joined). lines = [_line("王小明", phone='"02\r\n09"'), _line("陳大文")] _serve_tw(monkeypatch, listings=[_TW_LISTING[:1]], zips={(_TW_ELECTION, _TW_AREA, 1): _tw_zip(_tw_incomes_text(lines))}) diff --git a/tools/differential/README.md b/tools/differential/README.md index c1c33aa02..7c7325533 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -254,10 +254,10 @@ uv run python tools/differential/snapshot.py --local ctgov-officials |---|---|---| | `ctgov-officials` | every overall official of every study on ClinicalTrials.gov, whatever the role (principal investigators, study directors and chairs, and a few sub-investigators and officials with no role), the name as the sponsor's staff typed it into a free-text field: credentials run together (`MBBS,FCPS`), job titles in the name, all-caps and all-lowercase names, typos. Fetched from API v2 (NLM) | names outside clinical research, which is mostly medicine and heavily credentialed; names nobody typed by hand | | `fec-individuals` | the distinct names of every individual contributor in the FEC's bulk file for the 2026 cycle (`indiv26.zip`, downloaded once into the cache and read without unpacking; `fetch --refresh` downloads it again). Mostly a filer's software writing a form's fields in a fixed order: of its 1,893,964 names on 2026-10-18 every one was in capitals and 1,893,961 held a comma, with hand-typed text inside the fields -- 3,306 held a second comma, for a credential run or a suffix in a part of its own (`DOE, JR., JOHN`). The population for the one-case family-comma shape. The FEC's terms (52 U.S.C. 30111(a)(4)) bar using the names to solicit contributions or for any commercial purpose | case, since every name is in capitals; names written in their spoken order, or without a comma; small donors, since a committee need itemize only a donor whose gifts to it pass $200; names outside the US; names in any script but Latin | -| `tw-donors` | the distinct names of every individual donor in every campaign finance report Taiwan's Control Yuan publishes on its political donation platform (ardata.cy.gov.tw, whose zips data.gov.tw also lists): 454 reports of 64 elections on 2026-10-22, presidential, legislative and local: every report from 2018's elections on, and the residual reports of 2014's and 2016's. Each report's zip is cached under the name the platform lists it by, which carries the date it was built, so a refresh downloads only the reports published or rebuilt since. Names as a candidate's staff typed them into a filing, published in full where the ID, address and phone beside them are masked. Of the 308,663 names dated 2026-10-05, 296,816 were three Han characters, 5,449 four and 5,260 two; the population for unspaced Han names written family first, and for the married double surname (冠夫姓) a four-character name often is. Under the Open Government Data License, version 1.0, with the platform's own bar on using the data for profit, credit investigation or another improper purpose | names in spoken order; Latin script (80 of the names dated 2026-10-05 held a Latin letter); the comma (3 of them held a fullwidth one, none an ASCII one); case; names outside Taiwan; donors whose names the report does not publish, such as anonymous gifts; Japanese kanji names, which the Han script cannot tell from Chinese | +| `tw-donors` | the distinct names of every individual donor in every campaign finance report Taiwan's Control Yuan publishes on its political donation platform (ardata.cy.gov.tw, whose zips data.gov.tw also lists): 454 reports of 64 elections on 2026-10-22, presidential, legislative and local: every report from 2018's elections on, and the residual reports of 2014's and 2016's. Each report's zip is cached under the name the platform lists it by, which carries the date it was built, so a refresh downloads only the reports published or rebuilt since. Names as a candidate's staff typed them into a filing, published in full where the ID, address and phone beside them are masked. Of the 308,667 names dated 2026-10-05, 296,820 were three Han characters, 5,449 four and 5,260 two; the population for unspaced Han names written family first, and for the married double surname (冠夫姓) a four-character name often is. Under the Open Government Data License, version 1.0, with the platform's own bar on using the data for profit, credit investigation or another improper purpose | names in spoken order; Latin script (80 of the names dated 2026-10-05 held a Latin letter); the comma (3 of them held a fullwidth one, none an ASCII one); case; names outside Taiwan; donors whose names the report does not publish, such as anonymous gifts; Japanese kanji names, which the Han script cannot tell from Chinese | | `ukec-individuals` | the distinct names of the individual donors in the UK Electoral Commission's register of donations (to parties, their members, third parties, permitted participants and regulated donees), from the search's CSV export under the site's own default filter, fetched whole on every fetch (9.5 MB on 2026-10-22, accepted dates from 2001 to 2026). The default leaves out donations reported only before a poll (`prePoll=false`: 374 rows and 21 names on 2026-10-22). A name as the donee's return gave it, mostly in its spoken order and mixed case, with the donor's title: of its 15,214 names on 2026-10-22, `parse()` read a title in 8,219 (its `title` field non-empty) (`Mr`, `Ms`, `Mrs`, `Lord`, `Dr`, `Sir`; 32 a run of two or more, `The Rt Hon`) and a suffix in 320 (`MP`, `OBE`, `CBE`, `QC`). The population for titled British names. The Commission publishes its documents under the Open Government Licence v3.0, but the register states no reuse terms of its own, which the credit line says | the family-comma shape (2 names hold a comma) and case (9 are in capitals); small donors, since a donation is reported only above a threshold that depends on the donee; names outside the UK; names in any script but Latin | -A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. `tools/name_usage.py` looks a word up in each corpus's word index, cached as `wordindex.json` beside it on first use and keyed to the records' sha256 and to the code that cuts and folds a word, so a refetch or a change to the fold rebuilds it; a single-word lookup it answers parses neither the records nor the names. Measured 2026-10-22 with both corpora fetched, a single-word `name_usage.py --no-wikidata` run took 2.5s with their indexes cached (12.5s before the cache, 14.8s on the run that builds it). A spaced word (`Ph. D.`) still scans every name, which took 8.6s, nearly all of it the FEC corpus's. `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. The `fec-individuals` fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. A `ukec-individuals` record is a distinct name too, not a donation or a `DonorId`: one name often holds several IDs, which seem to come with reports rather than with people (15,214 names held 24,473 on 2026-10-22, the IDs counted in `meta.json`'s detail; 2,663 names held several at one `RegulatedEntityId` alone, counted from the export and not kept, though some of those may be two people of one name), so a count of IDs would weight a name by how often it was reported. Its fetch reads the register's total from the JSON search under the same filter before and after the export, and refuses an export holding another number of rows, or a register that changed in between: the export ignores `rows=`, so the total is what tells a cut or capped export from the whole register. It refuses an export whose last row is cut short (Python's `csv` returns a body cut inside a row as a short last row, which the row count would otherwise take for a whole one), and a header other than the 29 columns `UKEC_COLUMNS` names, and counts into `meta.json`'s detail the rows it skips: rows not 29 fields, rows from a donor that is not an individual (the filter asks the server to leave those out), and individuals with no name. Past `UKEC_MAX_BAD` (1 in 10,000) of either of the first two, or `UKEC_MAX_NAMELESS` (1 in 1,000; 4 of 42,355 rows were nameless on 2026-10-22) of the third, it refuses the export. The data date is when the first search answered, in UTC: the register states no date of its own, so the counts above, dated by the day they were fetched (2026-10-22), carry the data date 2026-10-23T03:13:19Z in their header. `tw-donors` keeps one zip per report under `reports/`, whole, its masked ID, address and phone columns included, as the FEC's zip is kept: the platform's download answers a report that does not exist and a fault alike, with a 500 and an incident number, so the fetch reads which reports exist from the platform's listing and refuses a listed report it cannot download. It reads each file's columns by name, since a 2020 report lacks a column the newest ones write, and refuses a column it does not know, a category it does not know, a row whose serial column names another report (the zip is not the one asked for), a listing that changes during the fetch, and a report more than half of whose rows it reads past. It reads back the rows a stray quote took into one field (157 in two 2020 reports on 2026-10-22), counts donor cells that hold no name (匿名, a dash) or a masked one, and caches a downloaded zip only once its rows are read, so a refused zip is asked for again. A zip the platform rebuilt under the name it had is not seen: delete `reports/` to download every report again. A layout check written from one surveyed file meets every other file only at fetch time: `tw-donors`' first three fetches were each refused partway, by a report with no income writing one row of dashes, by a 2020 report with 24 columns where the surveyed one had 25, and by a row with no category. Each refusal cost every download again until the source cached its zips, so a new source that downloads many files caches each one before its checks run. +A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. `tools/name_usage.py` looks a word up in each corpus's word index, cached as `wordindex.json` beside it on first use and keyed to the records' sha256 and to the code that cuts and folds a word, so a refetch or a change to the fold rebuilds it; a single-word lookup it answers parses neither the records nor the names. Measured 2026-10-22 with both corpora fetched, a single-word `name_usage.py --no-wikidata` run took 2.5s with their indexes cached (12.5s before the cache, 14.8s on the run that builds it). A spaced word (`Ph. D.`) still scans every name, which took 8.6s, nearly all of it the FEC corpus's. `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. The `fec-individuals` fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. A `ukec-individuals` record is a distinct name too, not a donation or a `DonorId`: one name often holds several IDs, which seem to come with reports rather than with people (15,214 names held 24,473 on 2026-10-22, the IDs counted in `meta.json`'s detail; 2,663 names held several at one `RegulatedEntityId` alone, counted from the export and not kept, though some of those may be two people of one name), so a count of IDs would weight a name by how often it was reported. Its fetch reads the register's total from the JSON search under the same filter before and after the export, and refuses an export holding another number of rows, or a register that changed in between: the export ignores `rows=`, so the total is what tells a cut or capped export from the whole register. It refuses an export whose last row is cut short (Python's `csv` returns a body cut inside a row as a short last row, which the row count would otherwise take for a whole one), and a header other than the 29 columns `UKEC_COLUMNS` names, and counts into `meta.json`'s detail the rows it skips: rows not 29 fields, rows from a donor that is not an individual (the filter asks the server to leave those out), and individuals with no name. Past `UKEC_MAX_BAD` (1 in 10,000) of either of the first two, or `UKEC_MAX_NAMELESS` (1 in 1,000; 4 of 42,355 rows were nameless on 2026-10-22) of the third, it refuses the export. The data date is when the first search answered, in UTC: the register states no date of its own, so the counts above, dated by the day they were fetched (2026-10-22), carry the data date 2026-10-23T03:13:19Z in their header. `tw-donors` keeps one zip per report under `reports/`, whole, its masked ID, address and phone columns included, as the FEC's zip is kept: the platform's download answers a report that does not exist and a fault alike, with a 500 and an incident number, so the fetch reads which reports exist from the platform's listing and refuses a listed report it cannot download. It reads each file's columns by name, since a 2020 report lacks a column the newest ones write, and refuses a column it does not know, a category it does not know, a row whose serial column names another report (the zip is not the one asked for), a listing that changes during the fetch, and a report more than half of whose rows it reads past. It reads back the rows a stray quote took into one field (169 rows in 12 records of three reports on 2026-10-22, 8 of them two rows joined by two stray quotes), counts donor cells that hold no name (匿名, a dash) or a masked one, and caches a downloaded zip only once its rows are read, so a refused zip is asked for again. A zip the platform rebuilt under the name it had is not seen: delete `reports/` to download every report again. A layout check written from one surveyed file meets every other file only at fetch time: `tw-donors`' first three fetches were each refused partway, by a report with no income writing one row of dashes, by a 2020 report with 24 columns where the surveyed one had 25, and by a row with no category. Each refusal cost every download again until the source cached its zips, so a new source that downloads many files caches each one before its checks run. ## Corpus provenance diff --git a/tools/differential/local_corpora.py b/tools/differential/local_corpora.py index da2cc4844..fe053d998 100644 --- a/tools/differential/local_corpora.py +++ b/tools/differential/local_corpora.py @@ -83,7 +83,7 @@ published or rebuilt since. Names as a candidate's staff typed them into a filing, published in full where the ID, address and phone beside them are masked: of the -308,663 on 2026-10-05, 296,816 were three Han characters, 5,449 four +308,667 on 2026-10-05, 296,820 were three Han characters, 5,449 four and 5,260 two, so it is the population for unspaced Han names written family first, and for the married double surname (冠夫姓) a four- character name often is. Under the Open Government Data License, @@ -694,17 +694,17 @@ def _tw_lines(text: str) -> list[str]: def _tw_rows(text: str, width: int, counts: Counter[str]) -> Iterator[list[str]]: """incomes.csv's records after its header. A record whose field holds line breaks is a quoted field spanning lines, or a stray quote - opening a field and taking the rows after it in until another - closes it: two 2020 reports held four, which took 157 rows in - (2026-10-22). Where one of the lines after its first reads as a - whole row alone, the record is cut back into its lines, each read - alone: the lines as wide as the header are rows, and the rest are - counted as cut by the quote. Otherwise it stays one record (a phone - field holding two numbers on two lines, 8 on 2026-10-22). Two stray - quotes in rows one after the other close each other and make one - record as wide as the header out of two rows, which no line read - alone as quoted shows: where every line of a record reads as a whole - row with quotes taken literally, those are its rows.""" + opening a field and taking the rows after it in until another quote + closes it: on 2026-10-22, 12 records in three reports took in 169 + rows that way, and every line of all 12 read as a whole row with + quotes taken literally. So where every line of a record does, those + are its rows; this also parts two rows that two stray quotes, one + after the other, joined into one record exactly as wide as the + header (8 of the 12). Else, where one of the lines after its first + reads as a whole row alone as quoted, the record is cut back into + its lines, each read alone: the lines as wide as the header are + rows, and the rest are counted as cut by the quote. Otherwise it + stays one record, a quoted field that truly spans lines.""" lines = _tw_lines(text) reader = csv.reader(lines) next(reader, None)