def enrich_frequencies(
spec_dir: Path,
*,
mode: str = "best_effort",
offline: bool = False,
populations: list[str] | None = None,
dataset: str = FREQUENCY_DATASET_LABEL,
write: bool = True,
client: GnomadClient | None = None,
) -> FrequencyResult:
"""Fill `frequencies.csv` from the coordinates already in `resolution.csv`.
`populations` restricts the emitted ancestry groups (e.g. `["global"]` keeps the table to one row
per allele — the human-legibility escape hatch for a module that wants the number, not the
breakdown). `None` keeps every group the source reports.
Existing rows are authoritative and are merged, never clobbered — the same rule `enrich()` applies
to `resolution.csv`, and it is what makes a hand-corrected number survive a re-run. Note the same
consequence, too: to regenerate after a machinery change you must delete the file first.
"""
spec_dir = Path(spec_dir)
# Both through the shared resolver: the module may keep either table under `derived/` (RM49), and
# a pass that read one layout and wrote the other would leave two copies behind.
resolution_path = sidecar_path(spec_dir, "resolution.csv", error=FrequencyEnrichmentError)
frequencies_path = sidecar_path(spec_dir, "frequencies.csv", error=FrequencyEnrichmentError)
if write:
# Fail on a placeholder or half-edited licence table now, before the fetch (S98, RM231).
require_sources_file(spec_dir, error=FrequencyEnrichmentError)
if not resolution_path.exists():
raise FrequencyEnrichmentError(
f"no resolution.csv in {spec_dir} — the frequency pass reads resolved coordinates, so run "
f"`just-dna-enricher enrich` first."
)
resolution_rows, errors, _ = load_csv_rows(resolution_path, ResolutionRow, resolution_path.name)
if errors:
raise FrequencyEnrichmentError(f"resolution.csv is invalid: {errors[0]}")
# **The overlay, before anything reads a coordinate** (RM136). This is an INPUT read: the pass
# consumes `resolution.csv` and writes a different file, so it must see what the module *asserts*
# — the same rows the compiler's checks see — rather than what the last enrichment happened to
# write. Without this an author's correction was honoured one tier over and invisible here, so the
# same finding came back on every run with no way to clear it.
resolution_rows = overlaid_input_rows(
spec_dir, "resolution.csv", resolution_rows, error=FrequencyEnrichmentError
)
# A study's LD proxy has a row only for the compiler's allele check (RM331); it is not one of the
# module's loci, so this pass does not ask about it or write a fact row the compile calls an orphan.
try:
proxies = study_proxy_rsids(spec_dir)
except EnrichmentError as exc:
raise FrequencyEnrichmentError(str(exc)) from exc
resolution_rows = [row for row in resolution_rows if row.variant_key not in proxies]
existing: dict[tuple, FrequencyRow] = {}
if frequencies_path.exists():
rows, errors, _ = load_csv_rows(frequencies_path, FrequencyRow, "frequencies.csv")
if errors:
raise FrequencyEnrichmentError(f"existing frequencies.csv is invalid: {errors[0]}")
for row in rows:
existing[merge_key(row)] = row
alleles, off_build = _alleles_from_resolution(resolution_rows)
if off_build:
# One counted line, not one per row, and it names the build rather than the count alone: an
# author on GRCh37 needs to know the pass declined rather than found nothing.
logger.warning(
"Frequency enrichment skipped %d row(s) whose genome_build is not %s: gnomAD v4 is "
"%s-only and its variant id carries no assembly, so querying a coordinate from another "
"build would return a different variant's counts under this module's key. Examples: %s",
len(off_build),
FREQUENCY_GENOME_BUILD,
FREQUENCY_GENOME_BUILD,
off_build[:3],
)
wanted_populations = {p.strip().lower() for p in populations} if populations else None
if offline:
logger.warning(
"Frequency enrichment skipped: --offline and gnomAD has no offline snapshot (the v4.1 "
"sites VCFs are 58 GB exomes / 742 GB genomes). Any existing frequencies.csv is kept as "
"the pin; compiles stay reproducible from it."
)
out = sorted(existing.values(), key=_sort_key)
if write and existing:
_write_frequencies_csv(out, frequencies_path)
return FrequencyResult(
rows=out,
sources=sorted({r.source for r in out if r.source}),
mode=mode,
skipped_offline=True,
missing=sorted({key for key, *_ in alleles} - {r.variant_key for r in out}),
)
# Only fetch what no existing row already covers — a re-run after adding two variants costs one
# request, not a full refetch of a table that is already right.
# Read off the rows, never by unpacking the merge key positionally: the key is
# `FrequencyRow._KEY_FIELDS` now, so a member added there would silently reshape this tuple (S51).
covered_keys = {row.variant_key for row in existing.values()}
wanted = [a for a in alleles if a[0] not in covered_keys]
# A locus gnomAD does not cover is not asked about at all. Two reasons, and the second is the
# important one: the request would spend a slot of a 10-per-minute budget to learn nothing, and
# gnomAD's silence about a locus it never looked at is indistinguishable from a real absence — so
# asking is how the false `not_found` got written in the first place.
to_fetch = [a for a in wanted if covers_locus(a[1], a[2]) is not False]
uncovered = [a for a in wanted if covers_locus(a[1], a[2]) is False]
fetched: dict[str, dict] = {}
unanswered_ids: set[str] = set()
if to_fetch:
owned = client is None
gnomad = client or GnomadClient()
try:
fetched = gnomad.fetch_frequencies(
[_variant_id(chrom, start, ref, alt) for _, chrom, start, ref, alt in to_fetch]
)
except GnomadError as exc:
raise FrequencyUnavailable(f"gnomAD could not be reached: {exc}") from exc
finally:
if owned:
gnomad.close()
unanswered_ids = set(getattr(gnomad, "unanswered", {})) & {
_variant_id(chrom, start, ref, alt) for _, chrom, start, ref, alt in to_fetch
}
fetched_at = now_utc_iso()
out: list[FrequencyRow] = list(existing.values())
covered: list[str] = []
missing: list[str] = []
unanswered: list[str] = []
for key, chrom, start, ref, alt in to_fetch:
payload = fetched.get(_variant_id(chrom, start, ref, alt))
if not payload and _variant_id(chrom, start, ref, alt) in unanswered_ids:
# gnomAD answered this allele's alias with an error, not with an absence (RM361): no row,
# so nothing is pinned, and the next run asks again.
unanswered.append(key)
continue
if not payload or not payload.get("populations"):
missing.append(key)
# A `not_found` row is a FACT — "gnomAD was asked and does not have this allele" — and is
# materially different both from a variant that was never queried (no row at all) and from
# one at a locus gnomAD does not cover (`not_covered`, below). Reaching this branch means
# the locus IS in the callset and the allele is absent from those samples.
out.append(
FrequencyRow(
variant_key=key,
chrom=chrom,
start=start,
ref=ref,
alt=alt,
population="global",
dataset=dataset,
source="gnomad",
status="not_found",
fetched_at=fetched_at,
)
)
continue
covered.append(key)
for entry in payload["populations"]:
if wanted_populations is not None and entry["population"] not in wanted_populations:
continue
out.append(
FrequencyRow(
variant_key=key,
rsid=payload.get("rsid"),
chrom=chrom,
start=start,
ref=ref,
alt=alt,
dataset=dataset,
vrs_id=payload.get("vrs_id"),
caid=payload.get("caid"),
source="gnomad",
status="resolved",
fetched_at=fetched_at,
**entry,
)
)
# Recorded rather than omitted: an author looking at the table should be able to see that gnomAD has
# no answer here, instead of wondering why the row is absent. One aggregated line, not one per row.
for key, chrom, start, ref, alt in uncovered:
out.append(
FrequencyRow(
variant_key=key,
chrom=chrom,
start=start,
ref=ref,
alt=alt,
population="global",
dataset=dataset,
source="gnomad",
status="not_covered",
fetched_at=fetched_at,
)
)
if uncovered:
logger.warning(
"%d allele(s) sit outside gnomAD's callset and were recorded as not_covered, not asked "
"about and not counted as absent (%s). gnomAD hard-masks the Y pseudoautosomal region — "
"those bases duplicate the X PAR — so it has no frequency to give there; the X spelling of "
"the same place does. This is an unknown, not a zero.",
len(uncovered),
", ".join(f"{chrom}:{start} {ref}>{alt}" for _, chrom, start, ref, alt in uncovered),
)
out.sort(key=_sort_key)
result = FrequencyResult(
rows=out,
covered=sorted(set(covered)),
missing=sorted(set(missing)),
unanswered=sorted(set(unanswered)),
uncovered=sorted({key for key, *_ in uncovered}),
sources=sorted({r.source for r in out if r.source}),
mode=mode,
)
# `result.uncovered` is deliberately NOT part of this gate. `strict` means "a reproducible
# artifact", and a locus outside gnomAD's callset is perfectly reproducible — it will be outside it
# on every run. Refusing would make a pseudoautosomal module uncompilable under `strict` for a
# reason no authored edit could fix, which is the opposite of what a strict failure is for.
if result.unanswered:
logger.warning(
"gnomAD answered %d allele(s) with an error rather than a record (%s): no row is written for "
"them, which is unchecked rather than absent, and the next run asks again.",
len(result.unanswered),
result.unanswered[:3],
)
if mode == "strict":
raise FrequencyUnavailable(
f"strict frequency enrichment: gnomAD answered {len(result.unanswered)} allele(s) with an "
f"error rather than a record: {result.unanswered}. Nothing was established about them."
)
if mode == "strict" and result.missing:
raise FrequencyEnrichmentError(
f"strict frequency enrichment: {len(result.missing)} resolved allele(s) have no gnomAD "
f"frequency: {result.missing}. gnomAD genuinely lacks rare/private alleles, so this is "
f"often correct data rather than a fetch failure — add the rows by hand or use "
f"mode='best_effort'."
)
if write:
# Same rule as every other pass that consults a source: record its terms, or the module cannot
# account for it. gnomAD is CC0 and asks for attribution, which is what this row carries.
# Inside the table's commit, so neither file exists without the other (S98, RM231).
_write_frequencies_csv(
out,
frequencies_path,
# Each source's licence row labelled by the release every committed row it answered
# states, or withheld when they differ (RM373, `licensing.record_table_sources`).
before_commit=lambda: record_table_sources(
table_datasets(out, release=lambda row: row.dataset),
"frequency",
spec_dir,
error=FrequencyEnrichmentError,
# A routine merge declares nothing, so it retires no licence row (RM383).
answered_every_row=False,
),
)
return result