230 lines
10 KiB
Python
230 lines
10 KiB
Python
"""The package boundaries, held as a test rather than as a convention.
|
|
|
|
Adapted from news/tests/test_architecture.py. One tree of our own, `app`,
|
|
and three copied ones: `pipeline` and `auth` from the boilerplate, `desk` from
|
|
news. Which of those may import which is `tests/test_coupling.py`'s rule; this
|
|
file holds `app`'s own layers.
|
|
|
|
1. **Direction.** Within `app`, a package imports only from packages below
|
|
it in `LAYERS`. `desk`, `pipeline` and `auth` may be imported by any of them.
|
|
2. **Doors.** From outside a package, only `<tree>.<package>` is imported, and
|
|
only names it lists in `__all__`. `desk.tools` has no door of its own, so
|
|
there the door is the tool: `desk.tools.<tool>`, one level, no deeper.
|
|
3. **Plugins.** A file under `plugins/` imports `desk` and nothing else from
|
|
these trees. A plugin is app-agnostic; it never sees `app`.
|
|
4. **Readers.** A file of `plugins/readers/` imports the standard library,
|
|
`httpx`, `pydantic`, `desk` through its doors and, within the package,
|
|
`._contract`, `._shared` and `._countries` only: a reader shares no code
|
|
with another reader, nor with `_runner`, which reads the readers and so
|
|
alone may import the package's registry, `plugins.readers`.
|
|
|
|
Tests are held to none of this.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import importlib
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
#: Bottom to top. `places` and `seniority` read what a board wrote and know
|
|
#: no database, so storage may use them. `main` assembles the app from
|
|
#: everything below it; `cli` and `__main__` are the entry points and may
|
|
#: reach anything.
|
|
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "prep", "jobs", "web", "main", "cli",
|
|
"__main__"]
|
|
APP = "app"
|
|
EXTERNAL = {"desk", "pipeline", "auth"}
|
|
TREES = {APP} | EXTERNAL
|
|
|
|
|
|
def _is_module(name: str) -> bool:
|
|
try:
|
|
return importlib.util.find_spec(name) is not None
|
|
except ModuleNotFoundError:
|
|
return False
|
|
|
|
|
|
def _imports(source: str) -> list[tuple[int, str, list[str]]]:
|
|
"""Every import of one of `TREES`: (line, module, names). `from x import y`
|
|
where `x.y` is a module is folded into `x.y` with no names, since that is
|
|
what it reaches for."""
|
|
found: list[tuple[int, str, list[str]]] = []
|
|
for node in ast.walk(ast.parse(source)):
|
|
if isinstance(node, ast.Import):
|
|
found += [(node.lineno, a.name, []) for a in node.names
|
|
if a.name.split(".")[0] in TREES]
|
|
elif isinstance(node, ast.ImportFrom) and node.level == 0 \
|
|
and (module := node.module or "").split(".")[0] in TREES:
|
|
names = []
|
|
for a in node.names:
|
|
if _is_module(f"{module}.{a.name}"):
|
|
found.append((node.lineno, f"{module}.{a.name}", []))
|
|
else:
|
|
names.append(a.name)
|
|
if names:
|
|
found.append((node.lineno, module, names))
|
|
return found
|
|
|
|
|
|
def _door_violation(module: str, names: list[str]) -> str | None:
|
|
"""Why this import is not through a door, or None if it is."""
|
|
parts = module.split(".")
|
|
depth = 3 if parts[:2] == ["desk", "tools"] else 2
|
|
if len(parts) > depth:
|
|
return f"reaches past {'.'.join(parts[:depth])}"
|
|
exported = getattr(importlib.import_module(module), "__all__", None)
|
|
hidden = [n for n in names if exported is not None and n not in exported]
|
|
return f"{', '.join(hidden)} is not in {module}.__all__" if hidden else None
|
|
|
|
|
|
def _package(path: Path) -> str:
|
|
"""`app/storage/store.py` -> "storage"; `app/cli.py` -> "cli";
|
|
the tree's own `__init__.py` -> "" (a door, exempt from direction)."""
|
|
rel = path.relative_to(ROOT).parts
|
|
name = rel[1].removesuffix(".py") if len(rel) > 1 else ""
|
|
return "" if name == "__init__" else name
|
|
|
|
|
|
APP_FILES = sorted(p for p in (ROOT / APP).rglob("*.py") if "__pycache__" not in p.parts)
|
|
PLUGIN_FILES = sorted((ROOT / "plugins").glob("*.py"))
|
|
READER_FILES = sorted((ROOT / "plugins" / "readers").glob("*.py"))
|
|
|
|
#: What a reader may import beside the standard library and `desk`.
|
|
READER_THIRD_PARTY = frozenset({"httpx", "pydantic"})
|
|
|
|
#: The package's own modules a reader may import.
|
|
READER_SIBLINGS = frozenset({"_contract", "_shared", "_countries"})
|
|
|
|
|
|
def _outside(here: str, source: str):
|
|
"""Imports in a file of package `here` that cross a package boundary."""
|
|
for line, module, names in _imports(source):
|
|
parts = module.split(".")
|
|
if parts[0] == APP and len(parts) > 1 and parts[1] == here:
|
|
continue
|
|
yield line, module, names
|
|
|
|
|
|
def direction_violations(here: str, source: str) -> list[str]:
|
|
wrong = []
|
|
for line, module, _ in _outside(here, source):
|
|
parts = module.split(".")
|
|
if parts[0] != APP or not here or len(parts) < 2 or parts[1] not in LAYERS:
|
|
continue
|
|
if LAYERS.index(parts[1]) >= LAYERS.index(here):
|
|
wrong.append(f"line {line}: {module} is not below app.{here}")
|
|
return wrong
|
|
|
|
|
|
def plugin_violations(source: str) -> list[str]:
|
|
wrong = []
|
|
for line, module, names in _imports(source):
|
|
if module.split(".")[0] != "desk":
|
|
wrong.append(f"line {line}: {module} (a plugin imports desk only)")
|
|
elif reason := _door_violation(module, names):
|
|
wrong.append(f"line {line}: {module} ({reason})")
|
|
return wrong
|
|
|
|
|
|
def reader_violations(source: str, registry: bool = False) -> list[str]:
|
|
"""Every import in a file of `plugins/readers/` outside rule 4;
|
|
`registry` allows `from plugins import readers`, for `_runner`."""
|
|
wrong = []
|
|
for node in ast.walk(ast.parse(source)):
|
|
if registry and isinstance(node, ast.ImportFrom) and node.level == 0 \
|
|
and node.module == "plugins" and [a.name for a in node.names] == ["readers"]:
|
|
continue
|
|
if isinstance(node, ast.Import):
|
|
tops = [(a.name.split(".")[0], a.name) for a in node.names]
|
|
elif isinstance(node, ast.ImportFrom) and node.level == 0:
|
|
tops = [((node.module or "").split(".")[0], node.module or "")]
|
|
elif isinstance(node, ast.ImportFrom):
|
|
names = {node.module} if node.module else {a.name for a in node.names}
|
|
if node.level != 1 or not names <= READER_SIBLINGS:
|
|
wrong.append(f"line {node.lineno}: {'.' * node.level}{node.module or ''} "
|
|
f"{sorted(names)} (a reader imports {', '.join(sorted(READER_SIBLINGS))})")
|
|
continue
|
|
else:
|
|
continue
|
|
for top, module in tops:
|
|
if top == "desk":
|
|
if isinstance(node, ast.ImportFrom) and (
|
|
reason := _door_violation(module, [a.name for a in node.names
|
|
if not _is_module(f"{module}.{a.name}")])):
|
|
wrong.append(f"line {node.lineno}: {module} ({reason})")
|
|
elif top not in sys.stdlib_module_names and top not in READER_THIRD_PARTY:
|
|
wrong.append(f"line {node.lineno}: {module} (a reader imports the standard "
|
|
f"library, httpx, pydantic and desk)")
|
|
return wrong
|
|
|
|
|
|
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
|
|
def test_a_package_imports_only_from_below_it(path: Path) -> None:
|
|
wrong = direction_violations(_package(path), path.read_text(encoding="utf-8"))
|
|
assert not wrong, "\n".join(wrong)
|
|
|
|
|
|
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
|
|
def test_a_package_is_reached_only_through_its_door(path: Path) -> None:
|
|
source = path.read_text(encoding="utf-8")
|
|
wrong = [f"line {line}: {reason}"
|
|
for line, module, names in _outside(_package(path), source)
|
|
if (reason := _door_violation(module, names))]
|
|
assert not wrong, "\n".join(wrong)
|
|
|
|
|
|
@pytest.mark.parametrize("path", PLUGIN_FILES, ids=lambda p: p.name)
|
|
def test_a_plugin_knows_only_desk(path: Path) -> None:
|
|
wrong = plugin_violations(path.read_text(encoding="utf-8"))
|
|
assert not wrong, "\n".join(wrong)
|
|
|
|
|
|
@pytest.mark.parametrize("path", READER_FILES, ids=lambda p: p.name)
|
|
def test_a_reader_knows_only_its_contract_and_helpers(path: Path) -> None:
|
|
wrong = reader_violations(path.read_text(encoding="utf-8"), path.name == "_runner.py")
|
|
assert not wrong, "\n".join(wrong)
|
|
|
|
|
|
def test_the_rules_catch_what_they_are_for() -> None:
|
|
"""The checker itself, on sources that break each rule."""
|
|
assert plugin_violations("from app.storage import Store")
|
|
assert plugin_violations("from pipeline import job")
|
|
assert plugin_violations("from desk.sources.base import Source")
|
|
assert not plugin_violations("from desk.sources import Source, Item")
|
|
|
|
assert reader_violations("from plugins import ats")
|
|
assert reader_violations("from plugins import readers")
|
|
assert not reader_violations("from plugins import readers", registry=True)
|
|
assert reader_violations("from plugins import ats", registry=True)
|
|
assert reader_violations("from app.prep import words")
|
|
assert reader_violations("from . import _runner")
|
|
assert reader_violations("from ._runner import fetch")
|
|
assert reader_violations("from .lever import parse")
|
|
assert reader_violations("from .. import ats")
|
|
assert reader_violations("import requests")
|
|
assert reader_violations("from desk.sources.base import Source")
|
|
assert not reader_violations("import json\nfrom urllib.parse import quote\nimport httpx\n"
|
|
"from pydantic import ConfigDict\n"
|
|
"from desk.sources import strip_html\n"
|
|
"from . import _contract\nfrom ._shared import only\n"
|
|
"from ._countries import code_of")
|
|
|
|
assert direction_violations("storage", "from app.web import TABS")
|
|
assert direction_violations("web", "from app.main import create_app")
|
|
assert not direction_violations("main", "from app.web import pages_router")
|
|
assert direction_violations("storage", "from app import jobs")
|
|
assert not direction_violations("jobs", "from app.storage import Store")
|
|
assert not direction_violations("models", "from pipeline import job")
|
|
|
|
assert _door_violation("app.storage.store", ["Store"])
|
|
assert _door_violation("desk.tools.video.marks", [])
|
|
assert _door_violation("pipeline", ["not_exported_name"])
|
|
assert not _door_violation("pipeline.web", ["connect"])
|