def reverse_module(
parquet_dir: Path,
output_dir: Path,
module_name: str | None = None,
title: str | None = None,
description: str | None = None,
report_title: str | None = None,
icon: str | None = None,
color: str | None = None,
version: str | None = None,
write_resolution: bool = True,
genome_build: str | None = None,
) -> Path:
"""Reverse-engineer a parquet module back into the spec DSL (yaml + csv). Returns output_dir.
**Every `module_spec.yaml` field the artifact carries is written back (RM241)**, and
`REVERSE_RECONSTRUCTED` / `REVERSE_EXCLUDED` say which those are. The spec's metadata reaches the
artifact through `manifest.json` and no parquet column: the `module:` block's display fields
(`manifest.display`), `version` (`identity`), `genome_build`, the defaults' `curator`/`method`,
`license`, `weighting`, `authorship`, `authority_precedence` and `panel`. The README and logo the
compile shipped beside the parquets are copied back byte for byte, after a hash check against the
manifest. Each value is taken in the same order: **an explicit argument, then the manifest, then
the fallback** a bare parquet directory with no manifest has always had (name-derived titles,
`database`/`#6435c9`, the most common row curator/method). `icon` and `color` default to `None` so
"not passed" can be told apart from "passed the old default".
`version` is recovered as `identity.version_coerced_from` where the compile recorded one, falling
back to `identity.version` — the pre-coercion string, because re-emitting the coerced one gives the
next compile nothing to coerce and `version_coerced_from` then goes absent on lap 2 (RM103).
`genome_build` matters more than the rest. A wrong title is cosmetic; a wrong build relocates every
coordinate in the module. This used to be hardcoded `"GRCh38"`, so
`compile → reverse → compile` on a `genome_build: GRCh37` module re-emitted it as GRCh38 and the
recompile minted `ga4gh:VA.…` ids — GRCh38 allele identities for GRCh37 positions — moving
`artifact.digest` and asserting a variant at a base the module never named. Resolution order is
therefore: this argument, else the artifact's own `manifest.json`, else `"GRCh38"` for a bare
parquet directory that records nothing.
`write_resolution` (default True) also emits `resolution.csv` — the resolved facts recovered from
the artifact — so `reverse → compile` reproduces the identical `artifact.digest` with **no network
and no Ensembl reference** (Principle 7 hardened from reference-dependent to self-contained). A
coord-keyed row's resolved rsid, dropped from `variants.csv`, is carried here and restored on
recompile via `resolution.resolve_from_table`.
The authored tables are written under their one legal name at the root. The machine-written
sidecars go through `layout.sidecar_write_path`, so a fresh tree gets the **preferred** spelling
(`licensing.csv`, not the deprecated `sources.csv` `_FACT_TABLES` still names for its parquet) and
an output directory that already carries a copy has that copy overwritten rather than joined by a
second one. Reversing into a directory that already holds two copies of one sidecar raises
`layout.SidecarCollision`: which of two hand-editable claims to overwrite is not something this
function may decide silently."""
parquet_dir = Path(parquet_dir)
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# SNP core is optional (RM2): a module may have no weights.parquet.
weights_path = parquet_dir / "weights.parquet"
weights_df = pl.read_parquet(weights_path) if weights_path.is_file() else None
# Every sidecar destination is resolved BEFORE the first write, and only for the tables this
# artifact will actually produce. `sidecar_write_path` raises on an output directory that already
# holds two copies of one table, and resolving late would raise it *after* `module_spec.yaml` and
# the authored CSVs had been rewritten — a refusal that leaves a half-rebuilt spec behind. The
# collision is refused with nothing touched instead, which is what the rest of this layout does.
sidecar_paths: dict[str, Path] = {
csv_name: sidecar_write_path(output_dir, csv_name)
for csv_name, parquet_name, _ in _FACT_TABLES
if (parquet_dir / parquet_name).is_file()
}
# Not `and weights_df is not None`: since RM43 the lookup table is rebuilt from the positional
# parquets too, so a table-only module emits one and needs its path resolved the same way. The
# writer itself returns before creating a file when there is nothing to write, so resolving the
# path unconditionally cannot leave an empty `resolution.csv` behind.
if write_resolution:
sidecar_paths["resolution.csv"] = sidecar_write_path(output_dir, "resolution.csv")
manifest = _artifact_manifest(parquet_dir)
if module_name is None:
module_name = _module_name_from_parquets(parquet_dir) or parquet_dir.name
if genome_build is None:
genome_build = (manifest.genome_build if manifest is not None else None) or "GRCh38"
if version is None and manifest is not None:
version = manifest.identity.version_coerced_from or manifest.identity.version
# **The attestation cannot be carried, and the silence about that was the defect (RM45).**
# `verification.json` records checks the *enricher* put against sources this tier cannot reach,
# and it is bound to the authored bytes by a hash — so reverse has nothing to rebuild it from and
# must not invent one. What it can do is say so: without this line a module round-trips into a
# spec that recompiles to a manifest with no `verification` block at all, and
# `manifest.compilation.warnings` — a surface consumers parse (RM44) — differs between a module
# and its own round trip with nothing edited. Losing a record of what was checked is acceptable;
# losing it invisibly is the S16 silent-success shape.
dropped_verification = manifest.verification if manifest is not None else None
if dropped_verification is not None:
logger.warning(_verification_loss_notice(dropped_verification), output_dir)
default_curator = "unknown"
default_method = "unknown"
# `priority` is intentionally NOT defaulted. It is Optional with no `Defaults.priority` fallback
# ('ai-module-creator'/'literature-review' back curator/method, but priority defaults to None),
# so a null priority is *authored-absent*. Inferring a default from the mode would fabricate a
# value for rows that never set one — turning weights `['high', None]` into `['high', 'high']`
# on recompile (a Principle 7 idempotency break). It is written verbatim, per row, instead.
default_priority: str | None = None
# The manifest records the spec's own `defaults.curator`/`method` (RM241). The row-mode guess is
# the bare-directory fallback, and it was worse than lossy on a table-only module: with no
# `weights.parquet` there are no rows to take a mode over, so a PGx module's defaults came back as
# `unknown` and the recompiled manifest said so.
if manifest is not None and manifest.curator is not None:
default_curator = manifest.curator
elif weights_df is not None:
default_curator = _most_common(weights_df, "curator") or "unknown"
if manifest is not None and manifest.method is not None:
default_method = manifest.method
elif weights_df is not None:
default_method = _most_common(weights_df, "method") or "unknown"
defaults_dict: dict[str, Any] = {"curator": default_curator, "method": default_method}
display = manifest.display if manifest is not None else None
fallback_title = module_name.replace("_", " ").title()
module_block: dict[str, Any] = {
"name": module_name,
"title": title or (display.title if display else fallback_title),
"description": description
or (display.description if display else f"Annotation module: {module_name}"),
"report_title": report_title or (display.report_title if display else fallback_title),
"icon": icon or (display.icon if display else "database"),
"color": color or (display.color if display else "#6435c9"),
}
if display is not None:
module_block["icon_set"] = display.icon_set
if version is not None:
module_block["version"] = version
spec: dict[str, Any] = {
# The one value the spec model accepts, so writing the constant is reconstructing it.
"schema_version": SCHEMA_VERSION,
"module": module_block,
"defaults": defaults_dict,
"genome_build": genome_build,
}
# The advisory blocks, copied through from the manifest the compile copied them into, and each
# written only when the module carried it, so a module without one reverses to a spec without one.
if manifest is not None:
if manifest.panel is not None:
spec["panel"] = manifest.panel.model_dump(mode="json", exclude_none=True)
if manifest.authorship:
spec["authorship"] = [
entry.model_dump(mode="json", exclude_none=True) for entry in manifest.authorship
]
if manifest.license is not None:
spec["license"] = manifest.license
if manifest.weighting is not None:
spec["weighting"] = manifest.weighting.model_dump(mode="json", exclude_none=True)
if manifest.authority_precedence:
spec["authority_precedence"] = list(manifest.authority_precedence)
if manifest.tier_scale is not None:
spec["tier_scale"] = manifest.tier_scale
(output_dir / "module_spec.yaml").write_text(
yaml.dump(spec, default_flow_style=False, sort_keys=False, allow_unicode=True), encoding="utf-8"
)
if manifest is not None:
for shipped in shipped_files(manifest):
_copy_shipped_file(parquet_dir, output_dir, shipped)
# variants.csv + studies.csv only when the module has them.
if weights_df is not None:
ann_lookup: dict[tuple, dict[str, str]] = {}
ann_key_columns: tuple[str, ...] = ()
ann_path = parquet_dir / "annotations.parquet"
if ann_path.exists():
ann_df = pl.read_parquet(ann_path)
# Which columns follow `variant_key` in this artifact's annotation key, read off the
# artifact rather than assumed — three generations of the table are in the wild and each
# keyed differently: 0.6 on (variant_key, genotype, conclusion, negatives) (RM80), 0.5 on
# the variant-effect pair, and the oldest on variant_key alone. Detected once and used by
# both the lookup and the weights-side probe, so the two cannot key differently.
# Order is fixed here, not by column order in the parquet, or the two sides could agree
# on the members and disagree on the tuple.
ann_key_columns = tuple(
name for name in ("genotype", "conclusion", "negatives") if name in ann_df.columns
)
for row in ann_df.iter_rows(named=True):
# variant_key so position-only variants (rsid null) match; fall back to rsid for an
# older artifact compiled before the variant_key column existed.
base = row.get("variant_key") or row.get("rsid")
if base is None:
continue
key = (base, *(row.get(name) for name in ann_key_columns))
ann_lookup[key] = {
"gene": row.get("gene", ""),
"phenotype": row.get("phenotype", ""),
"category": row.get("category", ""),
}
_write_variants_csv(
weights_df,
ann_lookup,
ann_key_columns,
default_curator,
default_method,
default_priority,
output_dir / "variants.csv",
genome_build=genome_build,
)
studies_path = parquet_dir / "studies.parquet"
studies_df: pl.DataFrame | None = pl.read_parquet(studies_path) if studies_path.exists() else None
if studies_df is not None:
_write_studies_csv(studies_df, output_dir / "studies.csv")
# 0.4 table kinds (RM1): each present parquet → its authored CSV.
positional_frames: list[tuple[str, pl.DataFrame]] = []
for csv_name, parquet_name, model in _TABLE_KINDS:
kind_path = parquet_dir / parquet_name
if kind_path.is_file():
kind_df = pl.read_parquet(kind_path)
_write_table_csv(kind_df, model, output_dir / csv_name)
if csv_name in _POSITIONAL_TABLE_CSVS:
positional_frames.append((csv_name, kind_df))
# `resolution.csv` last, because it is rebuilt from everything above. It is written from the SNP
# core **and** the positional tables since 0.6 (RM43): once the compiler fills a resolved
# coordinate into `pharm_variants`/`haplotypes`/`heteroplasmy`, a reverse that dropped the lookup
# table would emit a spec whose recompile leaves those parquets unfilled — so `compile → reverse →
# compile` would stop reproducing the artifact, which is Principle 7.
# Through `sidecar_paths`, never `output_dir / "resolution.csv"` — the literal join RM51 abolishes.
# Worth spelling out because the two halves of this arrived in different lanes and nearly cancelled:
# RM43 *moved* this call out of the `weights_df is not None` block so a table-only module emits one,
# while RM51's repair was applied to the call site at its old address. Taking either side of that
# merge alone loses the other.
# RM364: a spec that carried no table gets none back. Rebuilding one from the authored cells a
# first compile never resolved is a table the author never wrote: the second lap then reads it, and
# its manifest publishes a `resolution_signature`, VRS coverage and expansion counts the first did not.
if write_resolution and not _no_table_was_read(manifest, weights_df, positional_frames):
_write_resolution_csv(
weights_df,
positional_frames,
sidecar_paths["resolution.csv"],
genome_build=genome_build,
studies_df=studies_df,
spec_carried_one=manifest is not None and _spec_carried_a_table(manifest),
)
# 0.5 derived-fact sidecars: same round-trip, minus the columns that are recomputed rather than
# stored. `_write_table_csv` drops any parquet column the model does not declare, so
# `allele_frequency` (derived on write, absent from `FrequencyRow`'s fields) falls away by
# construction rather than by a special case — re-deriving it on the next compile reproduces the
# identical parquet.
#
# The filename comes from `sidecar_paths` (resolved above), not `output_dir / csv_name`: `_FACT_TABLES`
# names the licence table by its *deprecated* spelling (the parquet and the manifest key keep it,
# since only a major may rename those), so joining that name on by hand emitted `sources.csv` and
# made `compile → reverse → compile` deprecation-warn on a module whose own compile is silent —
# and `manifest.compilation.warnings` is a published field (RM44), so the module and its own round
# trip disagreed on it. This is the same "write to the file you read" rule every other writer
# follows rather than an exception to it: reverse builds a spec tree from an artifact, so on a
# fresh directory there is nothing to follow and the rule yields the preferred spelling, while
# reversing over a tree that already carries the old name (or a `derived/` split) overwrites that
# copy instead of leaving a second one behind — which is the collision the rule exists to prevent.
for csv_name, parquet_name, model in _FACT_TABLES:
fact_path = parquet_dir / parquet_name
if fact_path.is_file():
_write_table_csv(pl.read_parquet(fact_path), model, sidecar_paths[csv_name])
# The authored tables that are not table kinds (RM357), each at the spec **root** under its one legal
# name, from the first of its parquets, which carries every authored cell verbatim. They are
# authored like `variants.csv`, not machine-written sidecars, so none goes through `sidecar_paths`.
# `_write_table_csv` writes the authored columns only, so what a compile derived beside them falls
# away and the recompile derives it again: the meta-table's `members_key` and matches, and the whole
# of `meta_resolved.parquet`, which has no CSV and is rebuilt from these rows and the tables they name.
#
# **The overlay: this emits the post-overlay derived tables above AND the overlay, so the overlay
# applies twice, and that is the design rather than an oversight.** The alternative — emitting the
# *pre*-overlay tables so the apply happens exactly once — needs the overlay to record the value
# it replaced, which is a derived cell inside an authored table and rots the moment the source
# moves. All three operations are idempotent set operations instead (an update to a value already
# present, an insert of a row already keyed, a suppress of a row already absent are each a
# no-op), so the second lap is a fixed point and buys the round trip at no schema cost. It is
# checked by test, never assumed — Principle 7 requires that of every derivation.
for table in AUTHORED_NON_KIND_TABLES:
authored_parquet = parquet_dir / table.parquets[0]
if authored_parquet.is_file():
_write_table_csv(pl.read_parquet(authored_parquet), table.model, output_dir / table.csv)
return output_dir