diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 976248071..402ee2e33 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -414,7 +414,7 @@ print(len(MAIDEN_MARKERS), 'members ->', len(cls), 'structural classes')" **How it works.** WATCH asks whether a change moved a name: a committed corpus, `snapshot.jsonl` and a test that fails, or a local corpus read with `snapshot.py --local`, which no CI run sees. SCORE asks whether the parser agrees with someone who knows the fields: a committed fields file and agreement record, read by `cldr_agreement.py` from the repository alone. COUNT asks how one word is used: a cached source behind a row of `tools/name_usage.py`, with only the counts cited. DERIVE asks what a shipped list should hold: a cached source and a script writing the vocabulary (`tools/zh_surnames.py`). SURVEY asks how people write whole names, and how OFTEN they write each shape: a cached local corpus (`tools/differential/local_corpora.py`). -A second axis cuts across the jobs: whether the names are real or invented, labelled with fields or bare text, chosen or unselected. Every committed corpus is CHOSEN: invented to fill fields (CLDR, the conventions table, the rules examples), cataloged (GND), filtered to names whose every word aligns with a stated part (both Wikidata corpora, a filter that drops credentials and typos), or picked by someone testing or reporting a case (v1's test names in `corpus.jsonl`, the case-table rows in `corpus_cjk.jsonl` and `corpus_shapes.jsonl`, and the strings reporters typed into `corpus_issues.jsonl`). The last group does hold hand-typed shapes — a job title in the name (`Bob Jones, author`), a credential before the comma (`Washington Jr. MD, Franklin`) — but one instance of each, as chosen. What SURVEY adds is the unselected population and its frequencies: ClinicalTrials.gov's study officials (local corpus `ctgov-officials`, measured 2026-10-21) are typed into a free-text field by sponsors' staff, so they show how often each shape occurs, and hold some no committed corpus does, such as `MBBS,FCPS` with no space. One source may do several jobs, each through its own code: Wikidata does COUNT live and WATCH and SCORE through its builders. A live source has no fixed file to pin, so its cached copy is the pin, carrying the source's own data date and a sha256 checked on every load (a truncated file otherwise reads as a run of false zeros, #775). +A second axis cuts across the jobs: whether the names are real or invented, labelled with fields or bare text, chosen or unselected. Every committed corpus is CHOSEN: invented to fill fields (CLDR, the conventions table, the rules examples), cataloged (GND), filtered to names whose every word aligns with a stated part (both Wikidata corpora, a filter that drops credentials and typos), or picked by someone testing or reporting a case (v1's test names in `corpus.jsonl`, the case-table rows in `corpus_cjk.jsonl` and `corpus_shapes.jsonl`, and the strings reporters typed into `corpus_issues.jsonl`). The last group does hold hand-typed shapes — a job title in the name (`Bob Jones, author`), a credential before the comma (`Washington Jr. MD, Franklin`) — but one instance of each, as chosen. What SURVEY adds is the unselected population and its frequencies: ClinicalTrials.gov's study officials (local corpus `ctgov-officials`, measured 2026-10-21) are typed into a free-text field by sponsors' staff, so they show how often each shape occurs, and hold some no committed corpus does, such as `MBBS,FCPS` with no space. The FEC's individual contributors (local corpus `fec-individuals`, measured 2026-10-22) are unselected too, but mostly a filer's software writing a form's fields in a fixed order, so nearly all of its 1.89 million names share one shape (capitals, family, comma, given) and the hand-typed part is what goes inside the fields (a credential run or a suffix after a second comma): the population for that shape's variants, and blind to case. One source may do several jobs, each through its own code: Wikidata does COUNT live and WATCH and SCORE through its builders. A live source has no fixed file to pin, so its cached copy is the pin, carrying the source's own data date and a sha256 checked on every load (a truncated file otherwise reads as a run of false zeros, #775). **Reach for it when.** A source is proposed, before choosing between a builder and a tool column; and when a corpus would hold real people's names nobody vetted for publication, which is the case for a local corpus rather than a committed one. diff --git a/tests/v2/test_local_corpora.py b/tests/v2/test_local_corpora.py index 92ef1f281..ce167b904 100644 --- a/tests/v2/test_local_corpora.py +++ b/tests/v2/test_local_corpora.py @@ -16,6 +16,7 @@ import urllib.error import urllib.parse import urllib.request +import zipfile from collections.abc import Callable from pathlib import Path from typing import Any @@ -273,6 +274,238 @@ def test_an_unknown_source_names_the_known_ones() -> None: local.fetch("nppes") +# -- FEC individual contributions ------------------------------------------------ + +_FEC = "fec-individuals" +_FEC_DATE = (2026, 10, 18, 16, 17, 0) + + +def _fec_row(entity: str, name: str, columns: int = 21) -> str: + cols = ["C00000001", "N", "Q3", "P2026", "1", "15", entity, name, + "BRAINTREE", "MA", "02184", "SELF", "HOMEMAKER", "10132026", + "100", "", "SA1", "1", "", "", "1"] + return "|".join(cols[:columns]) + + +#: Two people, one giving twice and once with the name padded, a +#: committee, a person with no name, a line cut short and a line in +#: Latin-1: 7 lines. +_FEC_LINES = [ + _fec_row("IND", "LEE, ANN"), _fec_row("IND", "ROE JR, BOB MR"), + _fec_row("IND", " LEE, ANN "), _fec_row("PAC", "ACME PAC"), + _fec_row("IND", " "), _fec_row("IND", "WU, BO", columns=20), +] +_FEC_LATIN1 = _fec_row("IND", "PE\xd1A, JOS\xc9").encode("latin-1") + + +def _fec_body(lines: list[str], *extra: bytes) -> bytes: + return b"".join([*(line.encode() + b"\n" for line in lines), + *(line + b"\n" for line in extra)]) + + +def _fec_zip(path: Path, body: bytes | None = None, + date: tuple[int, int, int, int, int, int] = _FEC_DATE) -> bytes: + """A cycle's zip at `path`, holding the whole file and a by_date + slice holding only its first line, stamped alike: a fetch reading + the slice instead would lose names; its bytes.""" + if body is None: + body = _fec_body(_FEC_LINES, _FEC_LATIN1) + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w") as archive: + archive.writestr(zipfile.ZipInfo("itcont.txt", date), body) + archive.writestr(zipfile.ZipInfo("by_date/itcont_2026_slice.txt", date), + body.split(b"\n", 1)[0] + b"\n") + return path.read_bytes() + + +class _Download: + """A streamed download: read(n) hands out `body` in pieces, the + headers stating `length` (none when it is None), and raises `fail` + once the first piece is out.""" + + def __init__(self, body: bytes, length: int | None, + fail: Exception | None = None) -> None: + self.body, self.fail = body, fail + self.headers = {} if length is None else {"Content-Length": str(length)} + self.sent = False + + def __enter__(self) -> "_Download": + return self + + def __exit__(self, *exc: object) -> None: + return None + + def read(self, n: int) -> bytes: + if self.sent and self.fail: + raise self.fail + chunk, self.body = self.body[:n], self.body[n:] + self.sent = True + return chunk + + +def _fec_cache(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(local, "FEC_CHUNK", 64) # several reads, not one + return tmp_path / "local" / _FEC / local.FEC_ZIP + + +def _serve_fec(monkeypatch: pytest.MonkeyPatch, *responses: _Download) -> list[str]: + """Answer each download with the next of `responses`; the URLs asked.""" + queue, asked = list(responses), [] + + def urlopen(request: urllib.request.Request, timeout: int) -> _Download: + asked.append(request.full_url) + if not queue: + pytest.fail(f"an unexpected download of {request.full_url}") + return queue.pop(0) + monkeypatch.setattr(urllib.request, "urlopen", urlopen) + return asked + + +def test_a_fec_fetch_keeps_each_person_s_name_once_and_counts_the_rest( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _fec_zip(_fec_cache(monkeypatch, tmp_path)) + _serve_fec(monkeypatch) # a cached zip is not downloaded again + monkeypatch.setattr(local, "FEC_MAX_BAD", 1) # a fixture of 7 lines + corpus = local.fetch(_FEC) + assert corpus.records == [{"name": "LEE, ANN"}, {"name": "PEÑA, JOSÉ"}, + {"name": "ROE JR, BOB MR"}] + assert corpus.meta["data_date"] == "2026-10-18T16:17:00" + assert corpus.meta["detail"] == { + "cycle": local.FEC_CYCLE, "file": f"{local.FEC_ZIP}/itcont.txt", + "rows": 7, "individual_contributions": 5, "rows_from_other_entities": 1, + "individuals_without_a_name": 1, "lines_not_21_columns": 1, + "lines_not_utf8": 1} + assert corpus.header().endswith("3 names (local corpus fec-individuals)") + + +@pytest.mark.parametrize(("lines", "match"), [ + ([_fec_row("IND", "LEE, ANN") + "|X"] * 2 + [_fec_row("IND", "WU, BO")], + "2 of 3 lines .* are not 21 columns"), + # The 22nd column dropped and one inserted before NAME: 21 columns, + # entity in place, and NAME now holding the city. + (["|".join(_fec_row("IND", "x").split("|")[:7] + ["BRAINTREE", "LEE, ANN"] + + _fec_row("IND", "x").split("|")[8:20])], + "0 of 1 names hold a comma"), +], ids=["lines not 21 columns", "a column moved under NAME"]) +def test_a_fec_file_whose_layout_has_changed_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + lines: list[str], match: str) -> None: + _fec_zip(_fec_cache(monkeypatch, tmp_path), _fec_body(lines)) + with pytest.raises(ValueError, match=match): + local.fetch(_FEC) + assert local.load(_FEC) is None + + +def test_a_fec_file_mostly_not_utf8_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _fec_zip(_fec_cache(monkeypatch, tmp_path), + _fec_body([_fec_row("IND", "LEE, ANN")], _FEC_LATIN1)) + with pytest.raises(ValueError, match="1 of 2 lines .* are not UTF-8"): + local.fetch(_FEC) + + +@pytest.mark.parametrize("damage", ["not a zip", "no itcont.txt"]) +def test_a_fec_zip_that_cannot_be_read_names_the_way_out( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, damage: str) -> None: + path = _fec_cache(monkeypatch, tmp_path) + path.parent.mkdir(parents=True) + if damage == "not a zip": + path.write_bytes(_fec_zip(tmp_path / "whole.zip")[:-40]) + match = "fetch fec-individuals --refresh" + else: + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("itcont_renamed.txt", b"") + match = "holds no itcont.txt" + with pytest.raises(ValueError, match=match): + local.fetch(_FEC) + + +def test_a_fec_fetch_downloads_the_zip_when_it_is_missing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + path = _fec_cache(monkeypatch, tmp_path) + monkeypatch.setattr(local, "FEC_MAX_BAD", 1) + body = _fec_zip(tmp_path / "elsewhere.zip") + stale = path.with_name("indiv24.zip") + stale.parent.mkdir(parents=True) + stale.write_bytes(b"the last cycle's zip") + assert _serve_fec(monkeypatch, _Download(body, len(body))) == [] + assert len(local.fetch(_FEC).records) == 3 + assert path.read_bytes() == body + assert not stale.exists() + + +@pytest.mark.parametrize("stated", [10**6, -1, None], + ids=["cut short", "longer than stated", "no length"]) +def test_a_fec_download_that_does_not_match_its_length_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, stated: int | None) -> None: + path = _fec_cache(monkeypatch, tmp_path) + body = _fec_zip(tmp_path / "elsewhere.zip") + length = None if stated is None else len(body) - 1 if stated < 0 else stated + _serve_fec(monkeypatch, _Download(body, length)) + with pytest.raises(ValueError, match="fetch again"): + local.fetch(_FEC) + assert not path.exists() and not path.with_name(path.name + ".part").exists() + assert local.load(_FEC) is None + + +def test_a_fec_download_cut_off_mid_stream_leaves_no_part_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + path = _fec_cache(monkeypatch, tmp_path) + body = _fec_zip(tmp_path / "elsewhere.zip") + _serve_fec(monkeypatch, _Download(body, len(body), + http.client.IncompleteRead(b""))) + with pytest.raises(http.client.IncompleteRead): + local.fetch(_FEC) + assert not path.exists() and not path.with_name(path.name + ".part").exists() + + +def test_a_fec_refresh_rebuilds_the_corpus_from_a_new_download( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Re-reading the cached zip would reproduce the same file under a + # refresh, with the same names and data date. + path = _fec_cache(monkeypatch, tmp_path) + monkeypatch.setattr(local, "FEC_MAX_BAD", 1) + _fec_zip(path) + newer = _fec_zip(tmp_path / "newer.zip", _fec_body([_fec_row("IND", "WU, BO")]), + date=(2026, 10, 25, 9, 0, 0)) + asked = _serve_fec(monkeypatch, _Download(newer, len(newer))) + assert local.main(["fetch", _FEC]) == 0 + assert asked == [] + assert local.main(["fetch", _FEC, "--refresh"]) == 0 + assert asked == [local.FEC_URL] + corpus = local.load(_FEC) + assert corpus is not None and corpus.records == [{"name": "WU, BO"}] + assert corpus.meta["data_date"] == "2026-10-25T09:00:00" + + +def test_a_failed_fec_refresh_keeps_the_zip_and_the_corpus_it_had( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + path = _fec_cache(monkeypatch, tmp_path) + monkeypatch.setattr(local, "FEC_MAX_BAD", 1) + cached = _fec_zip(path) + _serve_fec(monkeypatch, _Download(cached[:100], len(cached))) + before = local.fetch(_FEC) + with pytest.raises(ValueError, match="fetch again"): + local.main(["fetch", _FEC, "--refresh"]) + assert path.read_bytes() == cached + assert not path.with_name(path.name + ".part").exists() + assert local.load(_FEC) == before + + +def test_a_fec_corpus_of_another_cycle_is_refused_on_load( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Bumping FEC_CYCLE would otherwise label the cached cycle's names + # with the new cycle's credit. + _fec_zip(_fec_cache(monkeypatch, tmp_path)) + monkeypatch.setattr(local, "FEC_MAX_BAD", 1) + local.fetch(_FEC) + monkeypatch.setattr(local, "FEC_CYCLE", "2028") + with pytest.raises(ValueError, match=f"holds the 2026 cycle.*reads the 2028 cycle" + f".*fetch {_FEC} --refresh"): + local.load(_FEC) + + # -- load ---------------------------------------------------------------------- def _fetched(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index c6e2883fc..e4eba05e5 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -2185,11 +2185,15 @@ def _local(*names: str) -> object: def test_a_local_corpus_not_fetched_names_the_fetch( monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) - [(source, corpus, error)] = _LOAD_LOCALS() - assert (source, corpus, error) == ("ctgov-officials", None, "") - row = tool.local_usage("MD", source, corpus) - assert row.status == "not downloaded" - assert "local_corpora.py fetch ctgov-officials" in row.note + # One row per source, every one not fetched naming its own fetch. + loaded = _LOAD_LOCALS() + assert loaded == [(source, None, "") + for source in sorted(tool._local_corpora().SOURCES)] + assert {"ctgov-officials", "fec-individuals"} <= {s for s, _, _ in loaded} + for source, corpus, _ in loaded: + row = tool.local_usage("MD", source, corpus) + assert row.status == "not downloaded" + assert f"local_corpora.py fetch {source}" in row.note def test_a_local_corpus_row_counts_officials_and_the_comma_side() -> None: diff --git a/tools/differential/README.md b/tools/differential/README.md index 89fd37ca3..2c681e6d2 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -243,8 +243,9 @@ uv run python tools/differential/snapshot.py --local ctgov-officials | Source | What it is | Blind to | |---|---|---| | `ctgov-officials` | every overall official of every study on ClinicalTrials.gov, whatever the role (principal investigators, study directors and chairs, and a few sub-investigators and officials with no role), the name as the sponsor's staff typed it into a free-text field: credentials run together (`MBBS,FCPS`), job titles in the name, all-caps and all-lowercase names, typos. Fetched from API v2 (NLM) | names outside clinical research, which is mostly medicine and heavily credentialed; names nobody typed by hand | +| `fec-individuals` | the distinct names of every individual contributor in the FEC's bulk file for the 2026 cycle (`indiv26.zip`, downloaded once into the cache and read without unpacking; `fetch --refresh` downloads it again). Mostly a filer's software writing a form's fields in a fixed order: of its 1,893,964 names on 2026-10-18 every one was in capitals and 1,893,961 held a comma, with hand-typed text inside the fields -- 3,306 held a second comma, for a credential run or a suffix in a part of its own (`DOE, JR., JOHN`). The population for the one-case family-comma shape. The FEC's terms (52 U.S.C. 30111(a)(4)) bar using the names to solicit contributions or for any commercial purpose | case, since every name is in capitals; names written in their spoken order, or without a comma; small donors, since a committee need itemize only a donor whose gifts to it pass $200; names outside the US; names in any script but Latin | -A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. +A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. Once fetched, loading it adds about 8 seconds to a `tools/name_usage.py` run (measured 2026-10-22: 4.4s without it, 12.5s with it). `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. The fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. ## Corpus provenance diff --git a/tools/differential/local_corpora.py b/tools/differential/local_corpora.py index eba37f433..33e101b73 100644 --- a/tools/differential/local_corpora.py +++ b/tools/differential/local_corpora.py @@ -19,7 +19,9 @@ `snapshot.py --local --record` keeps its snapshot beside them: - raw.jsonl -- one record per name as the source gave it, with where it - came from. The counts are counted from here. + came from where the source says: a ClinicalTrials.gov official, with + the study and role; one distinct FEC name, alone. The counts are + counted from here. - corpus.jsonl -- the distinct names, in the corpus line format snapshot.py and compare.py --corpus read. - meta.json -- the source's own date for the data, what was fetched, and @@ -32,7 +34,9 @@ A live source has no fixed snapshot to pin, so the cached copy IS the pin: a count quoted from it names the data date the header prints, and a later fetch may differ. A new source is a SOURCES row: a fetch -function returning the records and the source's data date. +function returning the records and the source's data date, and for a +source that caches its own input, the hooks that renew it on a +refresh and refuse a corpus read from another version of it. ctgov-officials -- every overall official of every study on ClinicalTrials.gov, whatever the role (principal investigators, study @@ -43,6 +47,23 @@ credit it, show NLM's processing date and say what was changed: every report here and the tools/name_usage.py row carry the credit and the date, and the names are kept unchanged. + +fec-individuals -- the distinct names of every individual contributor +in the FEC's bulk file of itemized contributions by individuals for the +cycle (indiv26.zip, 2.3 GB on 2026-10-18, downloaded into the cache once +and read without unpacking; `fetch --refresh` downloads it again). +Itemized contributions only: a committee need itemize only a donor +whose gifts to it pass $200, so a smaller donor is there only where a +committee itemized the gift anyway. Mostly a filer's software writing a form's fields in a fixed +order, family, comma, given, in capitals -- every one of the 1,893,964 +names on 2026-10-18 was in capitals and all but three held a comma -- +with hand-typed text inside the fields: 3,306 held a second comma, for +a credential run or a suffix in a part of its own. So it is the +population for the one-case family-comma shape, and blind to case and +to names written in their spoken order. The FEC's terms (52 U.S.C. +30111(a)(4)) bar using the names to solicit contributions or for any +commercial purpose, which the credit line every report opens with +repeats; only the NAME column is kept. """ from __future__ import annotations @@ -51,12 +72,15 @@ import hashlib import http.client import json +import os import re import sys import time import unicodedata import urllib.error import urllib.parse +import urllib.request +import zipfile from collections import Counter from collections.abc import Callable, Iterator from dataclasses import dataclass @@ -94,9 +118,16 @@ class Fetched: @dataclass(frozen=True) class Source: credit: str - #: What one record is, for counts: "officials". + #: What one record is, for counts: "officials", "names". unit: str fetch: Callable[[], Fetched] + #: What `fetch --refresh` runs first, for a source whose fetch reads + #: an input cached beside the corpus: fetches that input anew, + #: replacing the cached copy only once the new one is whole. + renew: Callable[[], None] | None = None + #: Why load() should refuse a meta.json this version of the source + #: did not write (another FEC cycle), or "" to accept it. + check: Callable[[dict[str, Any]], str] | None = None @dataclass(frozen=True) @@ -109,7 +140,8 @@ class Local: @property def credit(self) -> str: """The source's credit and its data date, which ClinicalTrials.gov's - terms ask every report to carry.""" + terms ask every report to carry, as the FEC's use restriction is + carried in its credit.""" return f"{SOURCES[self.source].credit}, data as of {self.meta['data_date']}" def header(self) -> str: @@ -240,10 +272,149 @@ def fetch_ctgov() -> Fetched: "officials_without_a_name": nameless}) +# -- FEC individual contributions -------------------------------------------- + +FEC_CYCLE = "2026" +FEC_ZIP = f"indiv{FEC_CYCLE[2:]}.zip" +FEC_URL = f"https://www.fec.gov/files/bulk-downloads/{FEC_CYCLE}/{FEC_ZIP}" +#: The whole cycle in one file; the zip's by_date/ members repeat it in slices. +FEC_MEMBER = "itcont.txt" +#: The file's columns (the FEC's contributions-by-individuals file +#: description): 21, pipe-separated, ENTITY_TP the 7th and NAME the 8th. +FEC_COLUMNS, FEC_ENTITY, FEC_NAME = 21, 6, 7 +FEC_CHUNK = 1 << 20 +#: The file has no header row to check the columns against, so a fetch +#: refuses one that no longer looks like the file the constants above +#: were written for: more than this share of its lines malformed or not +#: UTF-8 (none were, of 34,264,742 lines in the file dated 2026-10-18), +#: or fewer than FEC_MIN_COMMA of its names holding a comma (1,893,961 +#: of 1,893,964 did), which is what a column moved under NAME would show. +FEC_MAX_BAD = 1 / 10_000 +FEC_MIN_COMMA = 0.9 + + +def _fec_download(path: Path) -> None: + """Download FEC_URL to `path` through a .part file, refusing a body + whose length differs from the stated Content-Length, or with no + Content-Length at all: a cut zip would otherwise fail later as a + BadZipFile that says nothing about the download. The cached zip is + replaced only once the new one is whole, a failed download leaves no + .part behind, and a zip of another cycle is removed once this one + has arrived.""" + print(f"downloading {FEC_URL} ...", file=sys.stderr) + request = urllib.request.Request(FEC_URL, headers={"User-Agent": USER_AGENT}) + partial = path.with_name(path.name + ".part") + try: + got = 0 + with urllib.request.urlopen(request, timeout=60) as response, \ + partial.open("wb") as out: + stated = response.headers.get("Content-Length", "") + while chunk := response.read(FEC_CHUNK): + out.write(chunk) + got += len(chunk) + if not stated.isdigit() or int(stated) != got: + raise ValueError(f"{FEC_URL} sent {got:,} bytes where it stated " + f"{stated or 'no length'}; fetch again") + except BaseException: + partial.unlink(missing_ok=True) + raise + os.replace(partial, path) + for stale in path.parent.glob("indiv*.zip"): + if stale != path: + stale.unlink() + + +def _fec_read(path: Path) -> Fetched: + names: set[str] = set() + entities: Counter[str] = Counter() + rows = malformed = latin1 = nameless = 0 + with zipfile.ZipFile(path) as archive: + if FEC_MEMBER not in archive.namelist(): + raise ValueError(f"{path} holds no {FEC_MEMBER}") + member = archive.getinfo(FEC_MEMBER) + stamp = "{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}".format(*member.date_time) + with archive.open(member) as lines: + for line in lines: + rows += 1 + try: + text = line.decode("utf-8") + except UnicodeDecodeError: + latin1 += 1 + text = line.decode("latin-1") + cols = text.rstrip("\r\n").split("|") + if len(cols) != FEC_COLUMNS: + malformed += 1 + continue + entities[cols[FEC_ENTITY]] += 1 + if cols[FEC_ENTITY] != "IND": + continue + if name := cols[FEC_NAME].strip(): + names.add(name) + else: + nameless += 1 + for count, what in ((malformed, f"not {FEC_COLUMNS} columns"), + (latin1, "not UTF-8")): + if count > rows * FEC_MAX_BAD: + raise ValueError(f"{count:,} of {rows:,} lines of {path} are {what}: " + f"the file's layout has changed under FEC_COLUMNS") + with_comma = sum("," in name for name in names) + if names and with_comma < len(names) * FEC_MIN_COMMA: + raise ValueError(f"{with_comma:,} of {len(names):,} names hold a comma: " + f"column {FEC_NAME + 1} of {path} no longer holds NAME") + return Fetched([{"name": name} for name in sorted(names)], stamp, { + "cycle": FEC_CYCLE, "file": f"{FEC_ZIP}/{FEC_MEMBER}", "rows": rows, + "individual_contributions": entities["IND"], + "rows_from_other_entities": sum(entities.values()) - entities["IND"], + "individuals_without_a_name": nameless, + "lines_not_21_columns": malformed, "lines_not_utf8": latin1}) + + +def fetch_fec() -> Fetched: + """The distinct names of every individual contributor in the cycle's + file, one record per name, read from the zip cached beside the + corpus (downloaded when missing, and again by `fetch --refresh`). + + A record is a NAME, not a contribution or a contributor: one donor + gives many times (34,264,742 rows held 1,893,964 names in the file + dated 2026-10-18), and a contribution count would weight a shape by + how often its bearers give. Only the name is kept, stripped of the + spaces around it; the address, employer and amount columns are read + past. The data date is the member's own timestamp, which carries no + time zone. A line that is not 21 columns is counted and skipped, a + line that is not UTF-8 is counted and read as Latin-1, and a row from + an entity that is not a person and a person with no name are + counted; past FEC_MAX_BAD, or with too few names holding a comma, the + file is refused rather than read as a smaller or a different corpus.""" + path = directory("fec-individuals") / FEC_ZIP + if not path.exists(): + _fec_download(path) + try: + return _fec_read(path) + except zipfile.BadZipFile as exc: + raise ValueError(f"{path} is not a readable zip ({exc}); download it " + f"again: {fetch_command('fec-individuals', refresh=True)}" + ) from None + + +def renew_fec() -> None: + """Download the current file over the cached zip.""" + _fec_download(directory("fec-individuals") / FEC_ZIP) + + +def check_fec(meta: dict[str, Any]) -> str: + cycle = meta["detail"].get("cycle") + return "" if cycle == FEC_CYCLE else ( + f"holds the {cycle} cycle, and this tool reads the {FEC_CYCLE} cycle") + + SOURCES: dict[str, Source] = { "ctgov-officials": Source( credit="ClinicalTrials.gov (NLM) study officials", unit="officials", fetch=fetch_ctgov), + "fec-individuals": Source( + credit=f"FEC individual contributions, {FEC_CYCLE} cycle " + f"(52 U.S.C. 30111(a)(4): not for soliciting or commercial use)", + unit="names", fetch=fetch_fec, renew=renew_fec, check=check_fec), } @@ -311,6 +482,8 @@ def load(source: str) -> Local | None: value = meta.get(key) if not isinstance(value, kind) or isinstance(value, bool): raise ValueError(f"{path / META} has no {kind.__name__} {key!r}{again}") + if (check := SOURCES[source].check) and (why := check(meta)): + raise ValueError(f"{path / META} {why}{again}") def verified(name: str, key: str) -> bytes: body = (path / name).read_bytes() if (path / name).exists() else b"" @@ -519,6 +692,8 @@ def main(argv: list[str] | None = None) -> int: print(f"the cached copy is unusable, fetching again: {exc}", file=sys.stderr) if local is None: + if args.refresh and (renew := SOURCES[args.source].renew): + renew() local = fetch(args.source) where = f"fetched into {directory(args.source)}" else: diff --git a/tools/name_usage.py b/tools/name_usage.py index d69f0f72d..4b9922a06 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -97,7 +97,7 @@ it; the zip itself is never unpacked. - Local corpora (tools/differential/local_corpora.py): whole names as - people typed them, fetched into the cache by that script and never + a source holds them, fetched into the cache by that script and never committed, so this tool only reads them. One row per local corpus, one not yet fetched naming the command that fetches it: the source's credit and data date, how many records write the word anywhere in the @@ -106,8 +106,11 @@ (local_corpora.word_key); a spaced word ('Ph. D.') is matched as its words in a row, with the joined spelling's count beside it. The first is ClinicalTrials.gov's study officials, where `MD` after a comma is the - usual case and a credential before it is the noise worth seeing. - Not a census of anyone: it counts what the source's typists wrote. + usual case and a credential before it is the noise worth seeing; the + second the FEC's individual contributors, nearly every name written + family, comma, given in capitals by a filer's form, so a count there is + a word typed into a field rather than into a whole name. + Not a census of anyone: it counts what each source holds. Both US sources spell names in ASCII letters only, without spaces or punctuation (O'Brien is OBRIEN), so a word is folded the same way