Fetch the Atlas .proto sources and generate the gRPC bindings from them (RM192, RM196).
The repository carries neither the sources nor the bindings, and that is the point. Both are
upstream's, both are reproducible from a pinned commit, and a copy of somebody else's file in a
repository is a copy that goes stale silently. What is committed is the pin — a commit id and a
sha256 per file, below — which is what makes a fetch verifiable rather than merely convenient.
Three stages, and each has its own failure:
- Resolve.
fetch_protos() downloads the five files from google-deepmind/alphagenome at
UPSTREAM_COMMIT and checks each against UPSTREAM_SHA256. A hash mismatch is a refusal, not a
warning: the whole reason to pin a commit and a digest is that a commit id proves what git had
and a digest proves what arrived.
- Generate.
generate() stages the sources under a private package path, rewrites their
import "…" lines to match, and runs protoc. The rewrite is the safety property — see
STAGE_PREFIX.
- Build.
enricher/hatch_build.py runs both at wheel-build time, so a released artifact
carries the sources and the bindings even though the repository carries neither. The protos are
git-ignored and deliberately not build-ignored.
Run it as just-dna-enricher atlas generate, or directly::
uv run --with grpcio-tools python -m just_dna_enricher.atlas_protos
grpcio-tools is build-time only and lives in [dev]; the runtime needs grpcio and protobuf,
which is the claim RM192 rests on.
ProtoFetchError
Bases: RuntimeError
The pinned sources could not be obtained, or arrived as something else.
missing_sources
missing_sources(
proto_dir: Path | None = None,
) -> tuple[str, ...]
Which pinned files are not on disk yet. Empty means generate() can run offline.
Source code in enricher/src/just_dna_enricher/atlas_protos.py
| def missing_sources(proto_dir: Path | None = None) -> tuple[str, ...]:
"""Which pinned files are not on disk yet. Empty means `generate()` can run offline."""
directory = proto_dir or PROTO_DIR
if not directory.is_dir():
return tuple(UPSTREAM_SHA256)
return tuple(name for name in UPSTREAM_SHA256 if not (directory / name).is_file())
|
fetch_protos
fetch_protos(
proto_dir: Path | None = None, *, force: bool = False
) -> Path
Download the pinned sources, verifying each against its digest. Idempotent.
A file already on disk and matching its pin is left alone, so a build is offline after the
first run and a developer's checkout does not re-fetch on every wheel. A file on disk that does
not match is re-fetched rather than trusted: the only thing worse than no pin is a pin nobody
acts on.
Source code in enricher/src/just_dna_enricher/atlas_protos.py
| def fetch_protos(proto_dir: Path | None = None, *, force: bool = False) -> Path:
"""Download the pinned sources, verifying each against its digest. Idempotent.
A file already on disk **and matching its pin** is left alone, so a build is offline after the
first run and a developer's checkout does not re-fetch on every wheel. A file on disk that does
*not* match is re-fetched rather than trusted: the only thing worse than no pin is a pin nobody
acts on.
"""
directory = proto_dir or PROTO_DIR
directory.mkdir(parents=True, exist_ok=True)
for name, expected in UPSTREAM_SHA256.items():
target = directory / name
if not force and target.is_file() and _digest(target) == expected:
continue
try:
with urllib.request.urlopen(_url(name), timeout=60) as response:
payload = response.read()
except (urllib.error.URLError, TimeoutError, OSError) as exc:
raise ProtoFetchError(
f"could not fetch {name} from {UPSTREAM_REPO}@{UPSTREAM_COMMIT[:7]}: {exc}. "
"The Atlas sources are not vendored in this repository (RM196); a build needs "
"network once, and then never again."
) from exc
got = hashlib.sha256(payload).hexdigest()
if got != expected:
raise ProtoFetchError(
f"{name} does not match its pin: expected {expected}, got {got}. A commit id says "
"what git had and a digest says what arrived; they disagree, so nothing is written."
)
target.write_bytes(payload)
return directory
|
generate
generate(
out_dir: Path | None = None,
*,
include_root: Path | None = None,
proto_dir: Path | None = None,
) -> Path
Build the bindings under a private package path, from the pinned sources.
The fetched .proto files stay byte-identical to upstream — the rewrite happens on staged
copies — so a re-pin is a diff against a known digest rather than a merge. Only the import "…"
lines move; the protobuf package declaration is untouched, so descriptor names, and therefore
the wire format, are unchanged.
Source code in enricher/src/just_dna_enricher/atlas_protos.py
| def generate(
out_dir: Path | None = None,
*,
include_root: Path | None = None,
proto_dir: Path | None = None,
) -> Path:
"""Build the bindings under a private package path, from the pinned sources.
The fetched `.proto` files stay **byte-identical to upstream** — the rewrite happens on staged
copies — so a re-pin is a diff against a known digest rather than a merge. Only the `import "…"`
lines move; the protobuf `package` declaration is untouched, so descriptor names, and therefore
the wire format, are unchanged.
"""
out = out_dir or OUT_DIR
sources = proto_dir or PROTO_DIR
absent = missing_sources(sources)
if absent:
fetch_protos(sources)
root = include_root or SRC_ROOT
package_dir = out.relative_to(root) / STAGE_PREFIX
staged_prefix = f"{package_dir.as_posix()}/"
staged = root / package_dir
if out.exists():
shutil.rmtree(out)
staged.mkdir(parents=True)
for name in PROTOS:
source = (sources / name).read_text()
rewritten = source.replace(f'import "{UPSTREAM_IMPORT_PREFIX}', f'import "{staged_prefix}')
if UPSTREAM_IMPORT_PREFIX in source and rewritten == source: # pragma: no cover
raise ProtoFetchError(f"{name}: import rewrite did not apply")
(staged / name).write_text(rewritten)
try:
subprocess.run(
[
sys.executable,
"-m",
"grpc_tools.protoc",
f"-I{root}",
f"--python_out={root}",
f"--grpc_python_out={root}",
*(f"{staged_prefix}{name}" for name in PROTOS),
],
check=True,
)
except FileNotFoundError as exc: # pragma: no cover - grpcio-tools absent
raise ProtoFetchError(
"grpcio-tools is not installed. It is build-time only and lives in the [dev] group: "
"`uv sync` from a checkout, or `pip install grpcio-tools`."
) from exc
for package in (out, staged):
(package / "__init__.py").touch()
# The channel reads this at connect time, so it sits where the runtime looks rather than where
# the sources happen to have been fetched to.
shutil.copyfile(sources / SERVICE_CONFIG_NAME, out / SERVICE_CONFIG_NAME)
shutil.copyfile(sources / LICENSE_NAME, out / LICENSE_NAME)
return out
|
gencode_protobuf_version
gencode_protobuf_version(
entry: Path | None = None,
) -> tuple[int, int, int] | None
The protobuf version the generated bindings were stamped with, read off the file — or None
when there are no bindings to read. Text, not an import: importing is what fails.
Source code in enricher/src/just_dna_enricher/atlas_protos.py
| def gencode_protobuf_version(entry: Path | None = None) -> tuple[int, int, int] | None:
"""The protobuf version the generated bindings were stamped with, read off the file — or `None`
when there are no bindings to read. Text, not an import: importing is what fails."""
path = entry or (OUT_DIR / STAGE_PREFIX / _GENERATED_ENTRY)
if not path.is_file():
return None
match = _GENCODE_STAMP.search(path.read_text(encoding="utf-8"))
return tuple(int(x) for x in match.groups()) if match else None
|
protobuf_runtime_version
protobuf_runtime_version() -> tuple[int, int, int] | None
The installed protobuf, from its distribution metadata — no import, for the reason above.
Source code in enricher/src/just_dna_enricher/atlas_protos.py
| def protobuf_runtime_version() -> tuple[int, int, int] | None:
"""The installed protobuf, from its distribution metadata — no import, for the reason above."""
try:
raw = importlib.metadata.version("protobuf")
except importlib.metadata.PackageNotFoundError:
return None
parts = re.match(r"(\d+)\.(\d+)(?:\.(\d+))?", raw)
return (int(parts[1]), int(parts[2]), int(parts[3] or 0)) if parts else None
|
client_absence
client_absence() -> str | None
Why atlas_client cannot be imported, in the words of the ONE fix that applies — or None.
Two absences with two different remedies, and folding them sends half the askers to the wrong
one (@specific-rejection: a generic rejection is a dead end where a specific one is a fix).
Reported by a consumer through S98's neighbourhood: a wheel user missing the [atlas] extra was
told to also run just-dna-enricher atlas generate, followed that instruction, and hit
"grpcio-tools is not installed" — a third error about a fourth thing, none of it their problem.
Their fix was one pip install.
- The extra is not installed.
grpcio/protobuf are absent, so import grpc is what fails.
One pip install fixes it, and atlas generate would not — there is nothing to generate
into a runtime that cannot load the result.
- The bindings have not been generated.
grpc imports and the generated package does not
exist. Only a checkout can fix this, because generating needs grpcio-tools (the [dev]
group) and the upstream .proto pins — an installed wheel carries neither and ships the
bindings prebuilt instead, which is RM196.
Lives here rather than in atlas_client for the obvious reason: atlas_client is the module
that fails to import, so it cannot be asked why. This one is stdlib-only on purpose and stays
importable when everything it describes is missing.
Checked with find_spec and a file test rather than by importing: asking the question must not
have the side effect of answering it differently, and importing grpc to discover whether
grpc is importable costs 19 MB of process to learn something a spec lookup already knows.
Source code in enricher/src/just_dna_enricher/atlas_protos.py
| def client_absence() -> str | None:
"""Why `atlas_client` cannot be imported, in the words of the ONE fix that applies — or `None`.
**Two absences with two different remedies, and folding them sends half the askers to the wrong
one** (`@specific-rejection`: a generic rejection is a dead end where a specific one is a fix).
Reported by a consumer through S98's neighbourhood: a wheel user missing the `[atlas]` extra was
told to *also* run `just-dna-enricher atlas generate`, followed that instruction, and hit
"grpcio-tools is not installed" — a third error about a fourth thing, none of it their problem.
Their fix was one `pip install`.
* **The extra is not installed.** `grpcio`/`protobuf` are absent, so `import grpc` is what fails.
One `pip install` fixes it, and `atlas generate` would not — there is nothing to generate
*into* a runtime that cannot load the result.
* **The bindings have not been generated.** `grpc` imports and the generated package does not
exist. Only a checkout can fix this, because generating needs `grpcio-tools` (the `[dev]`
group) and the upstream `.proto` pins — an installed wheel carries neither and ships the
bindings prebuilt instead, which is RM196.
Lives here rather than in `atlas_client` for the obvious reason: `atlas_client` is the module
that fails to import, so it cannot be asked why. This one is stdlib-only on purpose and stays
importable when everything it describes is missing.
Checked with `find_spec` and a file test rather than by importing: asking the question must not
have the side effect of answering it differently, and importing `grpc` to discover whether
`grpc` is importable costs 19 MB of process to learn something a spec lookup already knows.
"""
if importlib.util.find_spec("grpc") is None:
return (
"the `[atlas]` extra is not installed, so there is no Atlas client. Install it with "
"`pip install 'just-dna-enricher[atlas]'` — grpcio + protobuf, about 19 MB. Do NOT run "
"`atlas generate`; that builds bindings for a runtime that could not load them anyway."
)
if not (OUT_DIR / STAGE_PREFIX / _GENERATED_ENTRY).is_file():
return (
"the Atlas gRPC bindings have not been generated. Run `just-dna-enricher atlas generate` "
"from a **checkout** of just-dna-format — it needs grpcio-tools, which is in the [dev] "
"group. An installed wheel carries neither the .proto pins nor grpcio-tools and ships "
"the bindings prebuilt instead (RM196), so a wheel reporting this is a packaging bug "
"rather than something to generate your way out of."
)
# **The runtime protobuf is older than the gencode** — the third absence (S107, RM254). Neither
# of the first two: grpc is installed and the bindings exist, and `atlas_service_pb2` refuses at
# import with protobuf's `VersionError`. The remedy is a version, not a command, and the sentence
# names both numbers and the usual cause — a co-installed package pinning `protobuf<7`.
stamped, runtime = gencode_protobuf_version(), protobuf_runtime_version()
if stamped is not None and runtime is not None and runtime < stamped:
return (
f"the installed protobuf {'.'.join(map(str, runtime))} is older than the "
f"{'.'.join(map(str, stamped))} the Atlas bindings were generated with, so they refuse to "
f"load (protobuf's gencode/runtime rule). Something else in this environment pins protobuf "
f"below that — dagster pins `protobuf<7` — and the `[atlas]` extra declares "
f"`protobuf>={'.'.join(map(str, stamped))}` for exactly this reason: resolve the two in one "
f"environment, or run the enricher in its own."
)
return None
|