def compile_module(
spec_dir: Path,
output_dir: Path,
compression: str = "zstd",
resolve_with_ensembl: bool = True,
ensembl_cache: Path | None = None,
compiled_by: str | None = None,
ensembl_reference: str | None = None,
log_files: list[Path] | None = None,
provenance_file: Path | None = None,
logo_file: Path | None = None,
readme_file: Path | None = None,
authority_keys: Iterable[str] | None = None,
strict: bool = False,
ba1_threshold: float = BA1_ALLELE_FREQUENCY_THRESHOLD,
) -> CompilationResult:
"""Compile a module spec directory into parquet files plus a `manifest.json`.
Args:
spec_dir: Path to the module spec directory.
output_dir: Directory for output parquet files + manifest.json.
compression: Parquet compression codec.
resolve_with_ensembl: Master switch for resolution — of **every** kind, despite the name.
With a `resolution.csv` present it drives the preferred, source-independent table path;
`ensembl_cache` is the deprecated fallback. Setting it False with a `resolution.csv`
present disables that table too, and warns, because the compile then succeeds with no
resolved position on any row.
ensembl_cache: **Deprecated (removed at 1.0).** Path to a prebuilt Ensembl DuckDB or parquet
cache dir. In-compiler DuckDB resolution has moved to `just-dna-enricher`; when given, this
emits a `DeprecationWarning` and routes to the enricher (which must be installed). Prefer
producing a `resolution.csv` (`just-dna-enricher enrich`) — the compiler then resolves with
no reference and no network.
compiled_by: Provenance tag for the manifest (the marketplace passes "marketplace-server";
a local compile leaves it None, so downloaders treat it as untrusted).
ensembl_reference: Pinned reference id recorded in the manifest for reproducibility.
log_files: Explicit run/provenance log files to record. If None, auto-discovers a top-level
`*.log` plus, under `spec_dir/logs/`, every `*.log` and each `.txt`/`.md`/`.json`/
`.jsonl`/`.png`/`.jpg`/`.jpeg` whose content matches its name (`_collect_logs`, RM338);
each other path under `logs/` draws a `log_file_not_shipped` warning with its reason
(RM337). An explicit list draws none. Logs are optional.
provenance_file: Explicit structured-provenance document. If None, auto-discovers
`spec_dir/provenance.json`. Optional; summarized into `manifest.provenance`.
logo_file: Explicit module logo image. If None, auto-discovers `spec_dir/logo.{png,jpg,jpeg}`.
Optional; hashed into `manifest.logo`, kept out of `artifact.digest`.
readme_file: Explicit module readme. If None, auto-discovers the first of
`manifest.README_CANDIDATES` (`README.md` first) beside the spec. Optional; hashed into
`manifest.readme`, kept out of `artifact.digest` and `content_signature` — prose about the
module is not part of its identity, so fixing a caveat is a patch (S25).
authority_keys: Inject-only set of consumer/registry-owned identity keys to strip from the
authored `module:` block before validation (e.g. `just_dna_format.normalize.
IDENTITY_AUTHORITY_KEYS`). None strips nothing.
strict: All-or-nothing compile. When True, fail (rather than emit a partial artifact) if any
variant still lacks a resolved genomic position (`chrom`+`start`) after resolution — an
unresolved position means the injected reference was incomplete/absent and the parquet
bytes (hence `artifact.digest`) would not be reproducible. Default False keeps the
best-effort behavior (positions left unset, surfaced as warnings).
ba1_threshold: Allele frequency above which a `pathogenic` variant draws the ACMG BA1 warning
(`_check_ba1_lint`). Defaults to ACMG's 5%. Raise it for a module curating a common
recessive carrier allele, where the default fires on correct data. Warning-only in both
modes, so this tunes noise, never whether the compile succeeds.
"""
spec_dir = Path(spec_dir)
output_dir = Path(output_dir)
# The pre-flight runs in THIS compile's mode (RM359). It used to run in best_effort whatever the
# mode, so it seeded `all_warnings` with every mode-ladder sentence and a `strict` compile then
# refused with the same finding in `errors`: one finding on two channels, counted twice by any
# consumer. In the compile's mode a ladder finding lands where its severity puts it, and only
# there, and every refusal the pre-flight can ask is `validate --strict`'s own. That includes the
# ladders that read RESOLVED rows (`locus_cannot_host_genotype`, `rsid_coordinate_disagrees`): since
# RM367 the pre-flight runs this compile's resolution step over the injected table, so both refuse
# on both paths. The one path it cannot follow is the deprecated `ensembl_cache` placing rows with
# no table beside the spec, which it is not handed (`legacy_reference`, below; `validate.py` says
# so at its resolution step). `resolve_with_ensembl` is passed for its
# own reason: it is not a severity, it decides whether the injected table is consulted at all, and
# a pre-flight answering that differently would seed `all_warnings` with a finding this compile
# contradicts.
# `legacy_reference` tells it the deprecated `ensembl_cache` will place rows it cannot see.
validation, validation_findings = _validate_spec(
spec_dir,
authority_keys,
strict=strict,
resolve_with_ensembl=resolve_with_ensembl,
legacy_reference=ensembl_cache is not None,
ba1_threshold=ba1_threshold,
log_files=log_files,
provenance_file=provenance_file,
logo_file=logo_file,
readme_file=readme_file,
)
if not validation.valid:
# The classified list, not `validation.warnings` — a failed compile publishes the same
# readable channel a successful one does, and the model's copy has lost its codes.
return CompilationResult(success=False, errors=validation.errors, warnings=validation_findings)
config, _, _ = _load_yaml(spec_dir / "module_spec.yaml", authority_keys)
assert config is not None
module_name = config.module.name
# A module composes from optional table kinds (RM2): load whatever is present.
variants: list[VariantRow] = []
if (spec_dir / "variants.csv").exists():
variants, _, _ = _load_csv_rows(
spec_dir / "variants.csv",
VariantRow,
"variants.csv",
genome_build=config.genome_build,
)
# Same re-stamp as in `validate_spec`; this function re-loads its own rows, so the fix has to
# happen on both copies or the artifact would carry the un-corrected keys.
_restamp_for_build(variants, config.genome_build)
studies: list[StudyRow] = []
if (spec_dir / "studies.csv").exists():
studies, _, _ = _load_csv_rows(
spec_dir / "studies.csv", StudyRow, "studies.csv", genome_build=config.genome_build
)
# The 0.4 table kinds, loaded here rather than at materialization time. Two reasons, and the
# second is the load-bearing one: the symbolic-allele check below has to reach them, and a check
# that can refuse must do so **before** `output_dir.mkdir()` — the placement the licence gate
# already has, for the same reason. (It also removes a second load of every table kind.)
kind_rows: dict[str, list[Any]] = {}
for csv_name, _parquet_name, model in _TABLE_KINDS:
if (spec_dir / csv_name).exists():
kind_rows[csv_name], _, _ = _load_csv_rows(
spec_dir / csv_name, model, csv_name, genome_build=config.genome_build
)
# Seeded from the classified list rather than from `validation.warnings`, whose members pydantic
# has already flattened to plain strings. The two lists carry identical text, so every `if w not
# in all_warnings` de-duplication below still compares exactly what it compared before.
all_warnings = list(validation_findings)
# Symbolic/structural alleles the module cannot apply (RM5). Runs before anything is written, and
# before resolution, because it decides which rows exist: under `best_effort` a row stating an
# unusable rule is **dropped** (the warning says so — `reverse` will not re-emit it), under
# `strict` the compile refuses. Re-run here because this pass also yields the DROPS the pre-flight
# only predicts; the warnings the pre-flight already produced are de-duplicated on the message, the
# way ploidy's and allele-membership's are.
symbolic_errors, symbolic_warnings, symbolic_drops = _check_symbolic_alleles(
{"variants.csv": variants, **kind_rows}, strict=strict
)
all_warnings.extend(w for w in symbolic_warnings if w not in all_warnings)
if symbolic_errors:
return CompilationResult(success=False, errors=symbolic_errors, warnings=all_warnings)
for table, drop_rows in symbolic_drops.items():
if table == "variants.csv":
variants = _apply_symbolic_drops(variants, drop_rows)
else:
kind_rows[table] = _apply_symbolic_drops(kind_rows[table], drop_rows)
if symbolic_drops:
# Re-derive the stats over what survived. `validate_spec` computed them from the full set, and
# `weights_rows` counts the parquet, so leaving them would publish a `variant_count` higher
# than the artifact holds (the RM44 class: a manifest number a catalog keys on and cannot
# check). **After the whole loop, not inside its `variants.csv` branch** — since S57 put the
# gene facets on every authored kind, a drop that empties the last row naming a gene in a
# *kind* table has to move `genes` too, and the old placement could not see one.
validation.stats.update(module_stats(variants, kind_rows))
# The meta-table (0.8), loaded and refused here, before `mkdir`, so a refusal leaves nothing written
# (the pre-flight has already refused anything these could, so the arms are defensive). It is
# EXPANDED further down, after the positional fill, against `authored_variants`: the post-drop,
# pre-resolution rows. That is the view `validate_spec` expands against (its survivors are filled
# too), so a member matches the same rows on both sides and each finding is one sentence.
# Every authored table that is not a table kind, loaded once by the one loader (RM357) and refused
# below in the order the checks always ran: the meta-table, the model-organism table, the overlay.
non_kind = {table.csv: load_authored_table(spec_dir, table.csv) for table in AUTHORED_NON_KIND_TABLES}
meta_rows, meta_errors, _ = non_kind[META_CSV]
meta_errors = meta_errors or meta_scale_errors(meta_rows, config.tier_scale)
if meta_errors:
return CompilationResult(success=False, errors=meta_errors, warnings=all_warnings)
authored_variants = variants
# The model-organism table (0.8, RM335), loaded and refused before `mkdir` on the meta-table's terms
# (the pre-flight has already refused anything this could). It is written verbatim further down.
model_organism_rows, model_organism_errors, model_organism_warnings = non_kind[MODEL_ORGANISMS_CSV]
# De-duplicated on the message: the pre-flight emitted these over the same bytes.
all_warnings.extend(w for w in model_organism_warnings if w not in all_warnings)
if model_organism_errors:
return CompilationResult(success=False, errors=model_organism_errors, warnings=all_warnings)
# The source-independent resolution table (0.5), if authored/produced beside the spec. When
# present it is the *preferred* resolution path: the compiler consumes already-resolved facts and
# owns no source convention (Ensembl/DuckDB/provisioning) — the strict inject-only end state
# (Principle 2). An injected `ensembl_cache` (the DuckDB path) is the superseded fallback (P3).
# The authored overlay (RM124), loaded before the first derived table it lies on. `validate_spec`
# above already reported every finding it has — this pass re-loads its own copy, the way it
# re-loads `variants.csv`, because the rows themselves are needed to build the artifact.
overrides, overlay_errors, overlay_warnings = non_kind[OVERRIDES_CSV]
if overlay_errors:
return CompilationResult(success=False, errors=overlay_errors, warnings=all_warnings)
# De-duplicated on the message like every other check that runs on both sides: the pre-flight
# already emitted these over the same bytes. Extended rather than discarded so a finding this
# loader gains later cannot go missing from a compile that never runs `validate` separately.
all_warnings.extend(w for w in overlay_warnings if w not in all_warnings)
# The citation view: the table kinds plus every authored non-kind table that cites, read off the
# registry (RM357), as `validate_spec` builds it. Every literature check below reads this rather than
# `kind_rows`.
citing_rows: dict[str, list[Any]] = {
**kind_rows,
**{table.csv: non_kind[table.csv][0] for table in AUTHORED_NON_KIND_TABLES if table.cites},
}
overlaid: set[str] = set()
#: RM137, the compile's half of the same stash: unmatched `update` targets for the two tables a
#: reverse rebuilds from something narrower, split below once `studies` and the citing tables are
#: in scope. Same reason as in `validate_spec`, and the same function does the splitting.
compile_deferred: dict[str, list[tuple[tuple[str, str], bool]]] = {}
# The source-independent resolution table (0.5), read by the reader `validate_spec` reads it with
# (RM367): located, parsed, overlaid (RM124), VRS-verified and counted. `injected.rows` is the ACTIVE
# view — empty under `--no-resolve` — and every consumer below reads it, so the switch reaches the
# membership table, the VRS counts, the signature, the parquets' ids and the licence cross-check at
# once. The pre-flight has already refused anything this can; the refusal here is defensive.
injected = read_resolution(
spec_dir, overrides, resolve=resolve_with_ensembl, genome_build=config.genome_build, strict=strict
)
all_warnings.extend(w for w in injected.warnings if w not in all_warnings)
if injected.errors:
return CompilationResult(success=False, errors=injected.errors, warnings=all_warnings)
if injected.present:
overlaid.add("resolution.csv")
if injected.deferred is not None:
compile_deferred["resolution.csv"] = injected.deferred
resolution_rows: list[ResolutionRow] = injected.rows
resolution_table: dict[str, list[ResolutionRow]] = injected.table
resolution_sources: list[str] = []
resolution_sig: str | None = None
# 0/0 for a module with no active resolution table: no allele identities were attempted, which is a
# different statement from "none were achieved" and is what the manifest should carry.
vrs_alleles, vrs_identified, _gaps = _vrs_coverage(resolution_rows)
# The pin (RM304): the rule `_verify_vrs_ids` compared every row against. `None` when no allele
# carries an id, since there is then nothing to pin.
vrs_version: str | None = VRS_SPEC_VERSION if vrs_identified else None
# The table's identity, stamped where the table is READ (a table-only module carries one too), and
# gated on the active view having ROWS: a header-only table hashes to the empty-set digest, which
# would cost `resolution_signature is not None` its meaning ("this module was resolved"), and under
# `--no-resolve` the view is empty, so no signature names a table that shaped nothing. The residual,
# measured: since RM364 every row filed under a subject the artifact carries rides on a variant
# parquet's `resolution_loci` (a proxy's on `studies.parquet.proxy_loci`, RM331), so the signature
# reproduces across reverse; a row about a variant the module does not carry has no parquet row to
# ride on, and is pinned in `test_resolution_matrix.py` (`Case.table_says_more`).
if resolution_rows:
resolution_sources = sorted({row.source for row in resolution_rows if row.source})
resolution_sig = _resolution_signature(resolution_rows)
# Do the alleles the module *states* exist at the loci it points at? Runs here, on the AUTHORED
# rows, because resolution may expand one rsid into several loci that share this genotype — after
# that expansion the check reports the siblings it was never about. See `_check_allele_membership`.
allele_errors, allele_warnings = _check_allele_membership(variants, resolution_table, strict=strict)
# De-duplicated on the message, the same way `_check_contig_ploidy` below is and for the same
# reason: `compile_module` runs `validate_spec` first, in this compile's mode (RM359), so under
# `best_effort` a check living in both places emits its warning twice. Under `strict` the pre-flight
# has already refused anything this could, so the re-run is defensive there; what is never wanted
# is the identical sentence printed a second time.
all_warnings.extend(w for w in allele_warnings if w not in all_warnings)
if allele_errors:
return CompilationResult(success=False, errors=allele_errors, warnings=all_warnings)
# RM91's study-side half, de-duplicated on the message for the same reason as the block above.
study_allele_errors, study_allele_warnings = _check_study_effect_alleles(
studies, resolution_table, strict=strict
)
all_warnings.extend(w for w in study_allele_warnings if w not in all_warnings)
if study_allele_errors:
return CompilationResult(success=False, errors=study_allele_errors, warnings=all_warnings)
# De-duplicated on the message, like the two blocks above (RM94). This one was the exception, and
# the reason it survived is worth keeping: `@no-rerun-with-counts` guards against a re-run whose
# message embeds a count, because the two copies then disagree and the manifest publishes two
# numbers. This message carries no count, so the copies agreed and the field was merely redundant
# rather than self-contradicting -- an untidiness nobody was looking for. The wider rule the
# neighbours already follow is the one to take from it: a check that runs on both sides dedupes on
# the message, and re-running it is the normal case rather than the exception.
#
# The re-run is kept as the compile's own word in its own mode. Since RM359 the pre-flight runs in
# this compile's mode too, so under `--strict` it has already refused; this pass is defensive.
p_value_errors, p_value_warnings = _check_p_value_num(studies, strict=strict)
all_warnings.extend(w for w in p_value_warnings if w not in all_warnings)
if p_value_errors:
return CompilationResult(success=False, errors=p_value_errors, warnings=all_warnings)
resolution_mode: str | None = None
# `None` until the injected-table path runs and reports them (S33): the deprecated `ensembl_cache`
# path cannot answer, and "not established" is not "no expansion".
expanded_keys: int | None = None
expanded_rows: int | None = None
# `--no-resolve` is the master switch for resolution of any kind, despite its name; its warning (the
# row count it skipped) came from `read_resolution` above, on both paths.
if resolve_with_ensembl and variants:
resolution_mode = "strict" if strict else "best_effort"
if resolution_table or ensembl_cache is None:
# The step `validate_spec` runs too (RM367): fill/expand from the active table, or say once
# that nothing was injected. Its refusals are the pre-flight's, so this return is defensive.
resolved = resolve_variants(variants, resolution_table, config.genome_build, strict=strict)
variants = resolved.variants
expanded_keys, expanded_rows = resolved.expanded_keys, resolved.expanded_rows
all_warnings.extend(w for w in resolved.warnings if w not in all_warnings)
if resolved.refusals:
return CompilationResult(success=False, errors=resolved.refusals, warnings=all_warnings)
else:
# DEPRECATED (removed at 1.0): the in-compiler DuckDB-reference path, routed to the enricher.
# The one resolution the pre-flight cannot run, because it is not handed the reference
# (`check_paths.COMPILE_ONLY`). Guarded optional import (CLAUDE.md).
warnings.warn(
"compile_module(ensembl_cache=...) / in-compiler DuckDB resolution is deprecated and "
"will be removed at 1.0. Produce a resolution.csv (e.g. `just-dna-enricher enrich`) and "
"the compiler consumes it with no reference and no network.",
DeprecationWarning,
stacklevel=2,
)
try:
from just_dna_enricher.resolver import resolve_variants as _legacy_resolve
except ImportError:
return CompilationResult(
success=False,
errors=[
"ensembl_cache resolution now lives in just-dna-enricher (the network tier). "
"Install just-dna-enricher, or precompute resolution.csv and recompile without "
"ensembl_cache."
],
warnings=all_warnings,
)
variants, legacy_warnings = _legacy_resolve(
variants, ensembl_cache, genome_build=config.genome_build
)
all_warnings.extend(w for w in legacy_warnings if w not in all_warnings)
# Resolution can change identity, so the resolved rows owe the identity checks again (a duplicate
# `(variant_key, genotype)`, a coordinate impossible in the build). The pre-flight asks the same
# function over the same rows; the return is defensive except on the `ensembl_cache` path.
post_errors = identity_after_resolution(variants, config.genome_build)
if post_errors:
return CompilationResult(success=False, errors=post_errors, warnings=all_warnings)
# The non-diploid guardrail where `chrom` is final for every row; the pre-flight asks it there too,
# and the sentences are identical, so the message-dedup collapses them.
all_warnings.extend(
w for w in ploidy_after_resolution(variants, config.genome_build) if w not in all_warnings
)
# Outcome axis (orthogonal to the requested `resolution_mode` policy, Principle 5): did every
# in-scope variant resolve to a genomic position? Vacuously true for a table-kind-only module —
# which is why the denominator travels beside it into the manifest (RM44). Both come from the same
# list, so the flag can never be published without the count that says what it quantified over.
fully_resolved = all(v.chrom is not None and v.start is not None for v in variants)
resolution_subjects = len(variants)
# Strict (all-or-nothing): refuse to write a partial artifact. A variant still missing its
# genomic position after resolution means the injected reference was incomplete or absent, so the
# coordinate-anchored parquet bytes (and `artifact.digest`) would not be reproducible — the
# failure mode behind "local hash differs from published". Best-effort (strict=False) leaves such
# rows unset with a warning instead. Scope is the SNP-core VariantRow; the 0.4 table kinds carry
# no positions. The pre-flight asks the same function over the same rows (RM367); it used to ask a
# predicate of its own, which agreed with this one only on GRCh38 with resolution switched on.
if strict:
unplaced = unplaced_refusal(variants)
if unplaced:
return CompilationResult(success=False, errors=unplaced, warnings=all_warnings)
# The wrong-build gate (S78, RM143), here for the same placement reason as the licence gate below
# and one line ahead of it: a refusal must leave nothing written. It reads the attestation the
# enricher wrote and acts on one recorded finding — see `build_disagreement_error` for why that
# one and no other, and why this does not move the strict line. The block is re-read rather than
# carried down from the pre-flight because a stale attestation is dropped by the reader, and the
# gate must see what the reader saw.
if strict:
build_error = build_disagreement_error(_verification_block(spec_dir)[0])
if build_error is not None:
return CompilationResult(success=False, errors=[build_error], warnings=all_warnings)
# Licensing gate, ahead of the other fact tables (which are read below, also before `mkdir`, since
# RM367): a refusal must leave nothing written. Purely computation over injected data: the compiler
# holds no source→licence map (Principle 2 — it owns no source convention) and only reads what the
# enricher recorded.
sources_path, gate_spelling_warnings, gate_spelling_errors = _locate_sidecar(spec_dir, SOURCES_CSV)
if gate_spelling_errors:
return CompilationResult(success=False, errors=gate_spelling_errors, warnings=all_warnings)
all_warnings.extend(w for w in gate_spelling_warnings if w not in all_warnings)
if sources_path is not None:
gate_rows, gate_load_errors, _ = _load_csv_rows(sources_path, SourceRow, sources_path.name)
if gate_load_errors:
return CompilationResult(success=False, errors=gate_load_errors, warnings=all_warnings)
gate_errors = _check_license_gate(gate_rows)
if gate_errors:
return CompilationResult(success=False, errors=gate_errors, warnings=all_warnings)
# The derived-fact sidecars are loaded, overlaid and checked HERE, before `mkdir`, and written after
# it (RM367). Their loop used to interleave the two, so a refusal from a sidecar's load, overlay or
# check returned with the SNP core and the table kinds already on disk. The pre-flight has refused
# anything these can (the returns below are defensive); the AST guard in `test_check_paths.py`
# holds every `CompilationResult(success=False, …)` in this function above the `mkdir`.
# 0.5 derived-fact sidecars: materialize each present CSV, and cross-check it against what the
# module actually contains. A row describing something the module never mentions is a warning, not
# an error — an over-broad sidecar is harmless (a stale gene left in after a variant was removed),
# while failing the compile over it would punish the author for the enricher's generosity.
# One branch per sidecar, keyed by model. A two-way `if/else` was fine for two tables and stops
# being readable at three, so each entry states its own checks and its own builder; the loop below
# stays generic. Errors are fatal, warnings accumulate — the same contract the SNP core uses.
#
# The positional fill (RM43), on the rows that survive. It runs **after** the symbolic-allele
# ladder (RM5) and before `_build_table`, and that order is the only correct one: the drop decides
# which rows exist, so filling first would resolve a coordinate onto a row about to be discarded
# and — worse — leave the joinability report below counting rows the artifact does not contain.
# The two never disagree because the drop happens up at load time, long before the resolution
# table is even read; this comment is here so the distance between them does not read as accident.
# Gated on `resolve_with_ensembl` like every other resolution, so `--no-resolve` consults nothing.
# Before `mkdir` since RM362 met RM367: the orphan checks below ask `module_subjects` over the
# filled positional tables, and those checks refuse before the first write.
fill_warnings, fill_applied = _apply_positional_resolution(
kind_rows, resolution_table, config.genome_build, resolve=resolve_with_ensembl
)
all_warnings.extend(w for w in fill_warnings if w not in all_warnings)
# "What the module carries" is asked once, of `module_subjects` (RM362), over the resolved variants
# and the filled positional tables — the rows the artifact holds — so every orphan check below, the
# manifest's `stats.genes` and the enricher's scopes give one answer. A load failure has already
# refused the compile, so no table is unreadable here. The injected table's placements of every
# carried identity are positions too (RM362, 2026-11-04), under the fill's own switch.
placements = resolution_table if resolve_with_ensembl else None
# Whether a table was read at all, never whether it has rows (RM451): `[]` in `resolution_loci` says it
# filed nothing under a key, and must read the same on the lap that re-reads a header-only table.
table_read = resolve_with_ensembl and injected.consulted
subjects = module_subjects(
{"variants.csv": variants, **kind_rows}, resolution=placements, genome_build=config.genome_build
)
def _frequency_checks(rows: list) -> tuple[list[str], list[str]]:
# De-duplicated on the message, the `_literature_checks` idiom eleven lines below and the
# `_check_contig_ploidy` one before it: `compile_module` runs `validate_spec`, which has run
# `_check_frequency_arithmetic` over these same injected rows since RM93 added it for parity,
# so an unfiltered re-run publishes the finding **twice** in `manifest.compilation.warnings`
# (RM44) and every consumer counting warnings overstates what is wrong with the module.
# Re-running is the normal case and is not the bug; not filtering is (`@no-rerun-with-counts`).
# Only the warnings need it — validate's errors abort the compile before this closure runs.
errors, warns = _check_frequency_arithmetic(rows)
warns = [w for w in warns if w not in all_warnings]
warns.extend(_cross_check_frequencies(rows, subjects))
warns.extend(
_check_ba1_lint(
rows,
variants,
threshold=ba1_threshold,
resolution=placements,
genome_build=config.genome_build,
)
)
return errors, warns
def _gene_metrics_checks(rows: list) -> tuple[list[str], list[str]]:
# Both have run in the pre-flight since RM211, so both arrive twice. The **extend site**
# below de-duplicates every fact-table check on the message — `@first-fact-check-on-both-
# sides`, dedupe where the results are collected rather than in each closure — which is why
# this one carries no filter of its own and `_gene_validity_checks` below carries none
# either. Re-running is the normal case; what would break it is a message embedding a count,
# and neither of these does.
return [], [*_check_gene_metrics_arithmetic(rows), *_cross_check_gene_metrics(rows, subjects)]
def _literature_checks(rows: list) -> tuple[list[str], list[str]]:
# De-duplicated on the message: `compile_module` runs `validate_spec`, which runs this same
# check, so a finding living in both places would otherwise print twice (the
# `_check_contig_ploidy` idiom).
return [], [w for w in _cross_check_literature(rows, studies, citing_rows) if w not in all_warnings]
def _gene_validity_checks(rows: list) -> tuple[list[str], list[str]]:
# Both warn in either mode (see `_check_gene_validity_currency`), so nothing here reads
# `strict` — the errors list stays empty by construction rather than by a branch.
return [], [
*_cross_check_gene_validity(rows, subjects),
*_check_gene_validity_currency(rows),
]
def _clinical_assertion_checks(rows: list) -> tuple[list[str], list[str]]:
return [], list(_cross_check_clinical_assertions(rows, subjects))
def _gwas_effect_checks(rows: list) -> tuple[list[str], list[str]]:
return [], list(_cross_check_gwas_effects(rows, subjects))
def _expression_effect_checks(rows: list) -> tuple[list[str], list[str]]:
"""No checks, and the absence is a decision rather than a gap (RM194/RM200).
**`_cross_check_gwas_effects`'s orphan warning deliberately does not transfer**, though the
two tables look alike enough that copying it would be the obvious move. A `GwasEffectRow` is
module-scoped by construction: the pass queries the Catalog *with the module's own rsIDs*, so
a row naming an identity the module does not carry really is the residue of a narrowed
variant list, which is what that warning is about.
`expression_effects.csv` is **locus-wide by construction instead**. The pass queries a
genomic interval and AlphaGenome answers for every scored variant in it, most of which the
module does not author — and finding those is the entire point of the item, because slicing
by gene position silently drops the promoters and enhancers that act on a gene without
sitting in it. Running the identical check here would fire on nearly every row of every
module, which is a warning that means "this table is working".
A gene-scoped variant of it — warn when `gene` names no gene the module annotates — was
considered and left unbuilt. A warning code is a permanent key (`@warning-code-names-the-
finding`), nobody has asked for this one, and minting one speculatively costs more than the
check would return. It is additive if a caller ever wants it.
Returning `([], [])` rather than reporting a zero is `@tautology-zero`: this is a table with
no check, not a check that always passes.
"""
return [], []
def _no_checks(rows: list) -> tuple[list[str], list[str]]:
"""A table with no check, not a check that always passes (`@tautology-zero`)."""
return [], []
def _concordance_checks(rows: list) -> tuple[list[str], list[str]]:
# De-duplicated on the message, the `_literature_checks` idiom: `compile_module` runs
# `validate_spec`, which emits the identical sentences over the identical post-overlay rows,
# and an unfiltered re-run would publish the count twice. Safe to run twice at all because
# no compile step between the two passes touches this table or the overlay above it — which
# is the standing test `@no-rerun-with-counts` actually applies.
warns = [w for w in _concordance_warnings(rows) if w not in all_warnings]
warns.extend(
w
for w in _cross_check_clin_sig_concordance(rows, variants, table="clin_sig_concordance.csv")
if w not in all_warnings
)
return [], warns
def _authority_call_checks(rows: list) -> tuple[list[str], list[str]]:
# No check that the subject also appears in the parent record, deliberately: an author who
# answers a contested subject suppresses the parent row through the overlay, and the calls
# behind it stay — the evidence outliving the question is correct, and warning about it would
# make answering a finding produce a finding.
return [], [
w
for w in _cross_check_clin_sig_concordance(rows, variants, table="clin_sig_authority_calls.csv")
if w not in all_warnings
]
def _sources_checks(rows: list) -> tuple[list[str], list[str]]:
# `sources.csv` is last in `_FACT_TABLES`, so the other sidecars are already parsed into
# `fact_rows` and their `source` values can be cross-checked here. Warnings only — the gate
# that can actually refuse already ran, before anything was written.
#
# The set is `sources_in_use`, which the pre-flight asks too (RM367), over every resolution row
# the file holds: a source row is about the module's files, whatever `--no-resolve` says.
# `fact_rows` holds every other sidecar by now, since `sources.csv` is last in `_FACT_TABLES`.
warns = _source_checks(rows, sources_in_use(injected.asserted, fact_rows))
warns.extend(_check_declared_license_agrees(rows, config.license if config else None))
return [], warns
_FACT_HANDLERS: dict[type, tuple[Callable, Callable]] = {
FrequencyRow: (_frequency_checks, lambda rows: _build_frequencies(rows, module_name)),
GeneMetricsRow: (_gene_metrics_checks, lambda rows: _build_table(rows, GeneMetricsRow, module_name)),
LiteratureRow: (_literature_checks, lambda rows: _build_table(rows, LiteratureRow, module_name)),
GeneValidityRow: (
_gene_validity_checks,
lambda rows: _build_table(rows, GeneValidityRow, module_name),
),
ClinicalAssertionRow: (
_clinical_assertion_checks,
lambda rows: _build_table(rows, ClinicalAssertionRow, module_name),
),
GwasEffectRow: (
_gwas_effect_checks,
lambda rows: _build_table(rows, GwasEffectRow, module_name),
),
ExpressionEffectRow: (
_expression_effect_checks,
lambda rows: _build_table(rows, ExpressionEffectRow, module_name),
),
# The Open Targets lane's three (RM333/RM236): no check, for `_expression_effect_checks`'
# reason turned around. Every row is the pass's answer about a variant the module asked for,
# so an orphan warning could fire only on a hand edit, and the overlay is where those go.
LocusToGeneRow: (_no_checks, lambda rows: _build_table(rows, LocusToGeneRow, module_name)),
ColocalisationRow: (_no_checks, lambda rows: _build_table(rows, ColocalisationRow, module_name)),
ConsequenceRow: (_no_checks, lambda rows: _build_table(rows, ConsequenceRow, module_name)),
# RM331, the same reason: every row is the `ld-pairs` pass's answer about a study row's pair.
LdPairRow: (_no_checks, lambda rows: _build_table(rows, LdPairRow, module_name)),
ClinSigConcordanceRow: (
_concordance_checks,
lambda rows: _build_table(rows, ClinSigConcordanceRow, module_name),
),
ClinSigAuthorityCallRow: (
_authority_call_checks,
lambda rows: _build_table(rows, ClinSigAuthorityCallRow, module_name),
),
SourceRow: (_sources_checks, lambda rows: _build_table(rows, SourceRow, module_name)),
}
fact_rows: dict[type, list] = {}
#: `(parquet, builder, post-overlay rows)` per present sidecar, written after `mkdir`.
fact_tables: list[tuple[str, Callable, list]] = []
for csv_name, parquet_name, model in _FACT_TABLES:
# Spelling collisions and the deprecation notice were both already surfaced by the licence
# gate above (for `sources.csv`) or are surfaced here for the first time; either way the
# notices de-duplicate, the way ploidy's and the VRS pass's already do.
fact_path, fact_spelling_warnings, fact_spelling_errors = _locate_sidecar(spec_dir, csv_name)
if fact_spelling_errors:
return CompilationResult(success=False, errors=fact_spelling_errors, warnings=all_warnings)
all_warnings.extend(w for w in fact_spelling_warnings if w not in all_warnings)
if fact_path is None:
continue
rows, fact_errors, _ = _load_csv_rows(fact_path, model, fact_path.name)
if fact_errors:
return CompilationResult(success=False, errors=fact_errors, warnings=all_warnings)
# The overlay, before the table's own check runs (RM124) — the check reports on what the
# module asserts, and the parquet, the fact signature and the manifest block are all built
# from the same post-overlay rows.
if csv_name in OVERRIDABLE_TABLES:
overlaid.add(csv_name)
lossy = csv_name in LOSSY_OVERLAY_TABLES or csv_name in VINDICATING_OVERLAY_TABLES
if lossy:
compile_deferred[csv_name] = update_targets(csv_name, rows, overrides)
rows, apply_errors, apply_warnings = apply_overrides(
csv_name, rows, overrides, defer_unmatched=lossy
)
if apply_errors:
return CompilationResult(success=False, errors=apply_errors, warnings=all_warnings)
all_warnings.extend(w for w in apply_warnings if w not in all_warnings)
check, build = _FACT_HANDLERS[model]
# **The check sees every row; everything after it sees the kept ones** (RM79). A literature
# row for a citation no study and no bin names joins to nothing, so carrying it into the
# parquet and the manifest is dead weight — but reporting it needs the full list, which is why
# the split happens here rather than at load. `literature.csv` itself is untouched: it is
# merge-not-clobber on purpose, and that pin is what makes a re-run cheap.
check_errors, check_warnings = check(rows)
if model is LiteratureRow:
rows, _dropped = split_cited_literature(rows, studies, citing_rows)
fact_rows[model] = rows
if check_errors:
return CompilationResult(success=False, errors=check_errors, warnings=all_warnings)
# De-duplicated on the message, like every other check that runs on both sides (RM94): a
# fact-table check the pre-flight also performs reaches the identical sentence twice, and a
# doubled line in `manifest.compilation.warnings` doubles `warnings_summary`'s count with it.
# Both passes read the same POST-OVERLAY rows — `validate_spec` applies the overlay in its own
# loop — so a message embedding a count says the same number on both sides and
# `@no-rerun-with-counts` is satisfied rather than dodged. The dedup was previously
# unnecessary here because no fact check ran in the pre-flight; RM108's currency check is the
# first, and stating the rule once is cheaper than remembering it for the second.
all_warnings.extend(w for w in check_warnings if w not in all_warnings)
fact_tables.append((parquet_name, build, rows))
# The files that ship beside the parquets, judged read-only before anything is written (RM367). They
# were judged while being copied, after the parquets, so a malformed `provenance.json` refused a
# compile that had already filled its output directory.
ancillary = ancillary_errors(
spec_dir, provenance_file=provenance_file, logo_file=logo_file, readme_file=readme_file
)
if ancillary:
return CompilationResult(success=False, errors=ancillary, warnings=all_warnings)
output_dir.mkdir(parents=True, exist_ok=True)
# The ids `resolution.csv` records, by allele, for every variant parquet (RM270). Gated like every
# resolution read: under `--no-resolve` the table was not read, so no id reaches the artifact.
vrs_ids = VrsIds.of(resolution_rows, config.genome_build) if resolve_with_ensembl else VrsIds()
# SNP core: weights/annotations only when the module actually has variants.
weights_df = (
_build_weights(variants, config, studies, vrs_ids, resolution=placements, table_read=table_read)
if variants
else None
)
annotations_df = _build_annotations(variants, module_name) if variants else None
studies_df = (
# Gated like every resolution read: under `--no-resolve` the table was not read, so no proxy's
# loci may reach the artifact either (RM331).
_build_studies(
studies, module_name, resolution_table if resolve_with_ensembl else None, config.genome_build
)
if studies
else None
)
# Names taken from `SNP_CORE_PARQUETS` rather than spelled again: this is the site the binding was
# extracted from, so a literal here would be the copy the constant exists to remove.
_weights_name, _annotations_name = SNP_CORE_PARQUETS["variants.csv"]
(_studies_name,) = SNP_CORE_PARQUETS["studies.csv"]
if weights_df is not None:
weights_df.write_parquet(output_dir / _weights_name, compression=compression)
if annotations_df is not None:
annotations_df.write_parquet(output_dir / _annotations_name, compression=compression)
if studies_df is not None:
studies_df.write_parquet(output_dir / _studies_name, compression=compression)
# The meta expansion (0.8), on the same view as the pre-flight's: the positional tables filled, and
# `variants` as authored (resolution copies the rows it changes, so `authored_variants` is untouched).
meta_build = (
build_meta(meta_rows, member_tables(authored_variants, kind_rows), config.tier_scale)
if meta_rows
else None
)
if meta_build is not None:
all_warnings.extend(w for w in meta_build.warnings if w not in all_warnings)
# 0.4 table kinds (RM1): materialize each present CSV via the generic materializer. The rows were
# loaded above, before `mkdir`, so the symbolic-allele check could refuse without leaving a
# half-written directory behind — and so a row it dropped is absent from the parquet here.
table_rows: dict[str, int] = {}
for csv_name, parquet_name, model in _TABLE_KINDS:
rows = kind_rows.get(csv_name)
if rows is None:
continue
table_df = _build_table(rows, model, module_name)
if csv_name in _VRS_BEARING_TABLES:
table_df = with_vrs_columns(table_df, vrs_ids, config.genome_build)
if csv_name in _POSITIONAL_TABLE_CSVS:
# RM364: what the table said under each row's key, so `reverse` rebuilds `resolution.csv`
# from the table's answer rather than from the row's own cells. Gated like every resolution
# read, and NOT on the fill having applied: a row the fill left alone is the reason for it.
table_df = with_resolution_loci(
table_df, placements or {}, config.genome_build, table_read=table_read
)
table_df.write_parquet(output_dir / parquet_name, compression=compression)
table_rows[parquet_name] = table_df.height
# Re-run here so the finding reaches a caller who compiles without validating first, and
# de-duplicated on the message the way `_check_contig_ploidy` is: `compile_module` runs
# `validate_spec` itself, so a check living in both places otherwise prints its sentence twice.
all_warnings.extend(
w
for w in _check_positional_joinability(
kind_rows, injected.named_table, config.genome_build, fill_applied=fill_applied
)
if w not in all_warnings
)
# The same facts as counts rather than as a sentence, for the catalog that was reading the
# sentence (S31). Computed here, beside the check, so the two cannot describe different row sets.
positional_rows, positional_rows_placed = positional_placement(kind_rows)
all_warnings.extend(w for w in _check_binning_grounding(kind_rows, studies) if w not in all_warnings)
# `kind_rows` is freshly loaded and never resolved, so re-running the binning check here produces
# the identical sentence and the message-dedup above does its job.
all_warnings.extend(w for w in _check_measure_shape(kind_rows) if w not in all_warnings)
all_warnings.extend(w for w in _check_binning_deprecations(kind_rows) if w not in all_warnings)
# **THREE checks of this round are deliberately NOT re-run here, and that is the fix rather than an
# omission.** `_check_missing_allele_marker`, `_check_qual_floor_on_reference_evidence` and `_check_vcf_pointers`
# all read authored cells (`alts`, `requires_callable` + `quality_from`, the pointer columns) that
# resolution never fills, so `validate_spec`'s pass — which runs before any expansion — already has
# the right answer, and it reaches this list through `all_warnings = list(validation.warnings)`
# above. Running any of them again on `variants` would count the *expanded* rows: by this point
# `variants` is `outcome.variants`, one row per resolved locus, so a one-to-many rsid becomes N rows
# carrying one authored genotype and the same finding is reported with a different count and
# different example keys. The de-duplication above keys on the **message**, so two sentences
# differing only in their count can never collapse, and both are published into
# `manifest.compilation.warnings` — a surface RM44 established that consumers parse. Measured twice,
# independently, on the way in: an rsid-only `requires_callable` row over a two-locus
# `resolution.csv` emitted "1 row(s) …" beside "2 row(s) …", and the pointer check emitted 328
# beside 337 on `pathogenic_clinvar`.
#
# Nothing is lost by staying behind resolution: all three are warning-only in both modes, so there
# is no severity for a re-run to recover — which is the whole reason the *mode-ladder* checks re-run
# at all. This is the mirror of the `_check_contig_ploidy` lesson rather than a contradiction of it:
# that warning had to **move** here because resolution fills its input; these three must stay behind
# it because resolution fills nothing they read. The rule that covers both: re-run a check after
# resolution exactly when resolution changes its input, and never when the message embeds a count.
# The derived-fact sidecars, written from the rows checked before `mkdir`, in `_FACT_TABLES` order.
for parquet_name, build, rows in fact_tables:
fact_df = build(rows)
fact_df.write_parquet(output_dir / parquet_name, compression=compression)
table_rows[parquet_name] = fact_df.height
# De-duplicated on the message like every other check that runs on both sides: `validate_spec`
# ran this over the same overlay and `all_warnings` was seeded from its result.
all_warnings.extend(w for w in _overlay_targets_missing(overrides, overlaid) if w not in all_warnings)
# And RM137's split, deferred from both overlay sites to here — `studies` and the citing tables are
# in scope now. De-duplicated on the message for the reason every both-sides check is: the
# pre-flight computed the identical sentence from the identical inputs.
all_warnings.extend(
w
for w in _classify_deferred_overlay_updates(
compile_deferred, studies, citing_rows, subjects, resolution_rows
)
if w not in all_warnings
)
# The overlay itself, materialized verbatim and in authored order — it is authored input, so it
# goes to parquet the way `variants.csv` does rather than being re-derived from what it changed.
# This is what `reverse_module` reads the corrections back out of (RM124).
if overrides:
overlay_df = _build_table(overrides, OverrideRow, module_name)
overlay_df.write_parquet(output_dir / OVERRIDES_PARQUET, compression=compression)
table_rows[OVERRIDES_PARQUET] = overlay_df.height
# The meta-table's pair (0.8), from the expansion computed before anything was written. Both are
# written whenever the module carries meta.csv, `meta_resolved.parquet` with zero rows when no
# one-member row reached anything, so a reader finds the file it was promised.
if meta_build is not None:
for parquet_name, frame in (
(META_PARQUET, meta_frame(meta_build, module_name)),
(META_RESOLVED_PARQUET, meta_resolved_frame(meta_build, module_name)),
):
frame.write_parquet(output_dir / parquet_name, compression=compression)
table_rows[parquet_name] = frame.height
# The model-organism table (0.8, RM335), verbatim and in authored order, through the generic
# materializer: no expansion and no derived column. `reverse_module` reads it back from here.
if model_organism_rows:
model_organism_df = _build_table(model_organism_rows, ModelOrganismRow, module_name)
model_organism_df.write_parquet(output_dir / MODEL_ORGANISMS_PARQUET, compression=compression)
table_rows[MODEL_ORGANISMS_PARQUET] = model_organism_df.height
frequency_rows: list[FrequencyRow] = fact_rows.get(FrequencyRow, [])
gene_metrics_rows: list[GeneMetricsRow] = fact_rows.get(GeneMetricsRow, [])
literature_rows: list[LiteratureRow] = fact_rows.get(LiteratureRow, [])
gene_validity_rows: list[GeneValidityRow] = fact_rows.get(GeneValidityRow, [])
clinical_assertion_rows: list[ClinicalAssertionRow] = fact_rows.get(ClinicalAssertionRow, [])
gwas_effect_rows: list[GwasEffectRow] = fact_rows.get(GwasEffectRow, [])
expression_effect_rows: list[ExpressionEffectRow] = fact_rows.get(ExpressionEffectRow, [])
locus_to_gene_rows: list[LocusToGeneRow] = fact_rows.get(LocusToGeneRow, [])
colocalisation_rows: list[ColocalisationRow] = fact_rows.get(ColocalisationRow, [])
consequence_rows: list[ConsequenceRow] = fact_rows.get(ConsequenceRow, [])
ld_pair_rows: list[LdPairRow] = fact_rows.get(LdPairRow, [])
concordance_rows: list[ClinSigConcordanceRow] = fact_rows.get(ClinSigConcordanceRow, [])
authority_call_rows: list[ClinSigAuthorityCallRow] = fact_rows.get(ClinSigAuthorityCallRow, [])
source_rows: list[SourceRow] = fact_rows.get(SourceRow, [])
# RM337: a file under `logs/` left behind is a warning in both modes, never a refusal — it changes
# no verdict, and the registry's server-side compile shows an uploader only this channel.
logs, log_warnings = _collect_logs(spec_dir, output_dir, log_files)
all_warnings.extend(w for w in log_warnings if w not in all_warnings)
# The ancillary files, judged by `ancillary_errors` before `mkdir`, so the collectors only copy here.
provenance = _collect_provenance(spec_dir, output_dir, provenance_file)
logo = _collect_logo(spec_dir, output_dir, logo_file)
readme = _collect_readme(spec_dir, output_dir, readme_file)
# The attestation, re-read here because the block is what gets stamped (`validate_spec` ran the
# same call for its warning and threw the block away). De-duplicated on the message for the
# standard reason: the pre-flight already emitted the identical sentence.
verification, verification_warnings = _verification_block(spec_dir)
all_warnings.extend(w for w in verification_warnings if w not in all_warnings)
# Content identity over the RAW authored data (re-read from disk, so pre-resolution and
# reference-independent — the in-scope `variants` here are already resolved). Out of
# `artifact.digest`; lets a registry dedup across recompile/metadata-strip.
manifest = _build_manifest(
config=config,
spec_dir=spec_dir,
output_dir=output_dir,
validation=validation,
weights_rows=weights_df.height if weights_df is not None else 0,
warnings=all_warnings,
dropped_rows={t: len(r) for t, r in sorted(symbolic_drops.items())},
compiled_by=compiled_by,
ensembl_reference=ensembl_reference,
logs=logs,
provenance=provenance,
logo=logo,
readme=readme,
content_sig=content_signature(spec_dir),
resolution_mode=resolution_mode,
fully_resolved=fully_resolved,
resolution_subjects=resolution_subjects,
expanded_keys=expanded_keys,
expanded_rows=expanded_rows,
positional_rows=positional_rows,
positional_rows_placed=positional_rows_placed,
vrs_alleles=vrs_alleles,
vrs_alleles_identified=vrs_identified,
vrs_version=vrs_version,
resolution_sig=resolution_sig,
resolution_sources=resolution_sources,
frequency=_frequency_block(frequency_rows),
gene_metrics=_gene_metrics_block(gene_metrics_rows),
gene_validity=_gene_validity_block(gene_validity_rows),
clinical_assertions=_clinical_assertions_block(clinical_assertion_rows),
gwas_effects=_gwas_effects_block(gwas_effect_rows),
expression_effects=_expression_effects_block(expression_effect_rows),
locus_to_gene=_locus_to_gene_block(locus_to_gene_rows),
colocalisations=_colocalisations_block(colocalisation_rows),
consequences=_consequences_block(consequence_rows),
ld_pairs=_ld_pairs_block(ld_pair_rows),
clin_sig_concordance=_clin_sig_concordance_block(concordance_rows, authority_call_rows),
literature=_literature_block(literature_rows),
sources=_sources_block(source_rows),
verification=verification,
meta_selectors=meta_build.selector_counts() if meta_build is not None else None,
)
write_manifest(manifest, output_dir / "manifest.json")
stats: dict[str, Any] = {
"module_name": module_name,
"weights_rows": weights_df.height if weights_df is not None else 0,
"annotations_rows": annotations_df.height if annotations_df is not None else 0,
"studies_rows": studies_df.height if studies_df is not None else 0,
"table_rows": table_rows,
}
return CompilationResult(
success=True,
output_dir=output_dir,
errors=[],
warnings=all_warnings,
stats=stats,
manifest=manifest,
)