335 lines
14 KiB
Python
335 lines
14 KiB
Python
"""The types of the reader contract, and the check of a module's declaration
|
|
against its functions. The prose is the package's docstring; this file is
|
|
what it names.
|
|
|
|
A reader declares `READS = Reads(...)`: what it reads (a board or a search),
|
|
how the listing arrives, where the text is, how completeness is known, what
|
|
it can filter at the source, how it states places and which fields it
|
|
states. The runner reads the declaration and does the rest. Values are
|
|
checked by pydantic where the module is declared, so a misspelt trait fails
|
|
the import; what depends on the module's functions, or on two traits at
|
|
once, is `problems()`, which the package's loader runs and turns into an
|
|
ImportError naming every problem.
|
|
|
|
Signatures of a module that declares `READS`, with `Get` and `Page` from
|
|
`._shared`:
|
|
|
|
def feed_url(slug: str, ask: Ask) -> str # listing="whole"
|
|
def parse(body: bytes, slug: str) -> list[Row] # listing="whole"
|
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page # listing="paged"
|
|
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
|
|
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
|
|
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
|
|
def clean_params(params: dict[str, Any]) -> dict[str, Any] # scope="search"
|
|
|
|
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
|
|
`country`, a place name for `place`, any word for `remote`. A dimension the
|
|
host has no facet for is left out, and the runner then reads it unfiltered;
|
|
a dimension present without a wanted value means the host has none of it.
|
|
|
|
A module without `READS` fails the import: every reader declares.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import inspect
|
|
import re
|
|
from collections.abc import Callable
|
|
from types import ModuleType
|
|
from typing import Any, Literal, Required, TypedDict
|
|
|
|
from pydantic import ConfigDict
|
|
|
|
from desk.models import Model
|
|
|
|
#: A filter dimension's reach at the source: `none`, not filtered; `one`, one
|
|
#: value a request (the runner makes one series per value and unions them by
|
|
#: id); `many`, a list in one request; `ids`, values the host names by its own
|
|
#: ids, which `facets` looks up.
|
|
Filter = Literal["none", "one", "many", "ids"]
|
|
|
|
#: Whether the reader states a field: `stated` from the host, `none` never.
|
|
Stated = Literal["stated", "none"]
|
|
|
|
#: A config entry name: `sources.search.<name>` in `config.yaml`.
|
|
_CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
|
|
|
|
#: An environment variable name.
|
|
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
|
|
|
|
#: The date a robots.txt record must carry: when the file was read.
|
|
_DATE = re.compile(r"20\d\d-\d\d-\d\d")
|
|
|
|
|
|
class _Frozen(Model):
|
|
"""A declaration: immutable, extra keys refused."""
|
|
|
|
model_config = ConfigDict(from_attributes=True, extra="forbid", frozen=True)
|
|
|
|
|
|
class Filters(_Frozen):
|
|
"""What the host filters at the source, per dimension. `country` takes
|
|
ISO 3166 alpha-2 codes, `place` place names, `remote` a bool, `query`
|
|
words. The runner never hands a reader a value on a dimension it
|
|
declared `none`. A board host with `query="one"` also takes the agents'
|
|
fetch terms, one term a series (`Ask.terms`, see `_runner.ask_for`)."""
|
|
|
|
country: Filter = "none"
|
|
place: Filter = "none"
|
|
remote: Filter = "none"
|
|
query: Filter = "none"
|
|
|
|
def declared(self) -> dict[str, Filter]:
|
|
"""The dimensions filtered at the source, `name -> reach`."""
|
|
return {name: reach for name, reach in self.model_dump().items() if reach != "none"}
|
|
|
|
|
|
class Fields(_Frozen):
|
|
"""Which fields the rows state, and how far they can be trusted.
|
|
|
|
`pay`: `figures` (`salary_min`, `salary_max`, `currency`), `line` (only
|
|
`salary_text`), `none`. `posted`: `trusted` (the host's date is the
|
|
posting date), `listed` (a date, but one the host moves, e.g. on an
|
|
edit), `none`. The rest: `stated` when the host states it, `none` when
|
|
the app has to infer it.
|
|
"""
|
|
|
|
pay: Literal["figures", "line", "none"] = "none"
|
|
posted: Literal["trusted", "listed", "none"] = "none"
|
|
remote: Stated = "none"
|
|
country: Stated = "none"
|
|
seniority: Stated = "none"
|
|
employment: Stated = "none"
|
|
|
|
|
|
class Key(_Frozen):
|
|
"""An API key the reader needs: the config entry `sources.search.<name>`
|
|
with `fields`, each falling back to the environment variable at the same
|
|
position in `env`. The runner resolves it and hands the reader the
|
|
values in `Ask.params["key"]`; a reader never reads a key itself."""
|
|
|
|
name: str
|
|
fields: tuple[str, ...]
|
|
env: tuple[str, ...] = ()
|
|
|
|
|
|
class Reads(_Frozen):
|
|
"""What a reader reads and how. One per module, as `READS`.
|
|
|
|
`scope`: `board` (one company's whole list) or `search` (many companies,
|
|
never a whole list). `listing`: `whole` (one answer) or `paged`.
|
|
`page_size` and `ceiling` (paged only): roles a page, and the offset
|
|
where the host stops answering whatever is asked (Workday 2,000).
|
|
`text`: `listing` (full text in the list), `excerpt` (a teaser in the
|
|
list, nothing more), `detail` (one request a role), `none`. `teaser`:
|
|
with `text="detail"`, the list also carries a teaser, handed over as
|
|
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
|
|
`whole` (the one answer), `never` (a search). `places`: `one` (the host
|
|
states one), `many` (a list per role), `copies` (the host lists one
|
|
posting per place, each with its own id), `none` (the host states no
|
|
place; every row has `places == []`). `robots`: the robots.txt
|
|
decision for the paths the reader requests, with the date the file was
|
|
read. `empty`: `suspect` when an unknown slug answers an empty list
|
|
rather than an error. `title`: `provisional` when the listing may not
|
|
state the title (a sitemap), which detail then fills; nothing reads it
|
|
yet, since `_runner.DETAIL_FIELDS` has no `title`.
|
|
"""
|
|
|
|
scope: Literal["board", "search"]
|
|
listing: Literal["whole", "paged"]
|
|
page_size: int | None = None
|
|
ceiling: int | None = None
|
|
text: Literal["listing", "excerpt", "detail", "none"]
|
|
teaser: bool = False
|
|
complete: Literal["stated", "end", "whole", "never"]
|
|
filters: Filters = Filters()
|
|
places: Literal["one", "many", "copies", "none"]
|
|
fields: Fields = Fields()
|
|
robots: str
|
|
empty: Literal["trusted", "suspect"] = "trusted"
|
|
key: Key | None = None
|
|
title: Literal["final", "provisional"] = "final"
|
|
|
|
|
|
class Ask(_Frozen):
|
|
"""What the runner wants from one series of requests. Only the
|
|
dimensions the reader declared are set; for a `one` filter each series
|
|
carries one value. With `ids`, `countries` and `places` hold the host's
|
|
ids from `facets`, not codes or names. `params` are a search's saved
|
|
params (and a resolved key under `"key"`). An empty `Ask()` is an
|
|
unfiltered read.
|
|
|
|
`remote=True` asks for remote roles. A reach of countries and remote
|
|
means "in these countries, or remote", which no host can ask in one
|
|
request (two facets intersect), so the runner makes the remote roles a
|
|
series of their own: a reader never gets both `remote=True` and a
|
|
country or a place. With `remote="ids"`, `remote_ids` holds the host's
|
|
ids of its remote values (`facets(...)["remote"]`, any key), and
|
|
`remote` stays True.
|
|
|
|
`terms` are the agents' fetch terms a board is narrowed by (see
|
|
`app/jobs/reach.py`). The runner makes one series per term, each with
|
|
that term as `query`, so a reader only ever reads `query`. A read with
|
|
terms is never the whole board: it closes nothing."""
|
|
|
|
countries: tuple[str, ...] = ()
|
|
places: tuple[str, ...] = ()
|
|
remote: bool | None = None
|
|
remote_ids: tuple[str, ...] = ()
|
|
query: str = ""
|
|
terms: tuple[str, ...] = ()
|
|
params: dict[str, Any] = {}
|
|
|
|
|
|
class Place(TypedDict):
|
|
"""One place of a role, as the host states it: `text` the host's own
|
|
words ("Berlin, Germany"), `remote` True for a remote place ("Remote
|
|
(DE)"), False for an office, None when the host does not say. Resolving
|
|
`text` to a city, region or country is the app's job, not the reader's."""
|
|
|
|
text: str
|
|
remote: bool | None
|
|
|
|
|
|
class Row(TypedDict, total=False):
|
|
"""One role as a reader hands it over.
|
|
|
|
`source_id` is the host's id, unique within the board or the search;
|
|
`url` a web URL (`_shared.is_web_url`); `places` every place the host
|
|
states, never multiplied into rows, never cut to one; `teaser` the
|
|
short text the list carries. `location` is the host's own one-line
|
|
place, kept while storage reads it. `posted` is an aware datetime or
|
|
None; `description` plain text; `remote` True, False, a word or None;
|
|
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
|
|
`employment` only where the host states them, as `Fields` declares.
|
|
`company` is the employer's name on a search row, which lists many
|
|
(`plugins/search.py` stores it); a board row carries none, the board is
|
|
the company. Any other stated key is kept in `raw` by the runner.
|
|
`source_id`, `title` and `url` come empty from a listing that lacks
|
|
them: the runner drops such a row and counts it (`_runner.valid`).
|
|
"""
|
|
|
|
source_id: Required[str]
|
|
title: Required[str]
|
|
url: Required[str]
|
|
places: Required[list[Place]]
|
|
teaser: str
|
|
location: str
|
|
posted: Any
|
|
department: str
|
|
description: str
|
|
remote: bool | str | None
|
|
salary_text: str
|
|
salary_min: int | None
|
|
salary_max: int | None
|
|
currency: str | None
|
|
country: str | None
|
|
seniority: str | None
|
|
employment: str | None
|
|
company: str
|
|
|
|
|
|
#: What a declared module's functions take, positionally.
|
|
SIGNATURES: dict[str, tuple[str, ...]] = {
|
|
"feed_url": ("slug", "ask"),
|
|
"parse": ("body", "slug"),
|
|
"page": ("get", "slug", "cursor", "ask"),
|
|
"rows": ("decoded", "slug"),
|
|
"detail": ("get", "slug", "row"),
|
|
"facets": ("get", "slug"),
|
|
"clean_params": ("params",),
|
|
}
|
|
|
|
|
|
def _takes(function: Callable[..., Any], count: int) -> bool:
|
|
"""Whether `function` accepts `count` positional arguments."""
|
|
try:
|
|
signature = inspect.signature(function)
|
|
except (TypeError, ValueError):
|
|
return True
|
|
try:
|
|
signature.bind(*([None] * count))
|
|
except TypeError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def problems(module: ModuleType) -> list[str]:
|
|
"""What `module`'s `READS` gets wrong, on its own or against the
|
|
module's functions; "no READS" for a module that declares none."""
|
|
if not hasattr(module, "READS"):
|
|
return ["no READS: every reader declares what it reads"]
|
|
reads = module.READS
|
|
if not isinstance(reads, Reads):
|
|
return ["READS is not a Reads"]
|
|
has = lambda name: callable(getattr(module, name, None)) # noqa: E731
|
|
out: list[str] = []
|
|
|
|
if reads.listing == "paged" and not (has("page") and has("rows")):
|
|
out.append('listing="paged" without page and rows')
|
|
if reads.listing == "whole" and not (has("feed_url") and has("parse")):
|
|
out.append('listing="whole" without feed_url and parse')
|
|
if reads.listing == "whole" and has("page"):
|
|
out.append('listing="whole" with page')
|
|
if reads.listing == "whole" and (reads.page_size is not None or reads.ceiling is not None):
|
|
out.append('page_size and ceiling are for listing="paged"')
|
|
if reads.page_size is not None and reads.page_size < 1:
|
|
out.append("page_size is not positive")
|
|
if reads.ceiling is not None and reads.ceiling < (reads.page_size or 1):
|
|
out.append("ceiling is below page_size")
|
|
if has("fallback_url") and reads.listing != "whole":
|
|
out.append('fallback_url without listing="whole"')
|
|
|
|
if reads.text == "detail" and not has("detail"):
|
|
out.append('text="detail" without detail')
|
|
if has("detail") and reads.text != "detail":
|
|
out.append(f'detail with text="{reads.text}"')
|
|
if reads.title == "provisional" and not has("detail"):
|
|
out.append('title="provisional" without detail to fill it')
|
|
if reads.teaser and reads.text != "detail":
|
|
out.append('teaser is for text="detail"; the list of "excerpt" is the teaser')
|
|
|
|
if (reads.complete == "whole") != (reads.listing == "whole"):
|
|
out.append('complete="whole" goes with listing="whole" and only with it')
|
|
if reads.scope == "search" and reads.complete not in ("never", "stated"):
|
|
out.append('scope="search" needs complete="never" or "stated"')
|
|
if reads.scope == "board" and reads.complete == "never":
|
|
out.append('scope="board" cannot be complete="never"')
|
|
|
|
ids = [name for name, reach in reads.filters.declared().items() if reach == "ids"]
|
|
if ids and not has("facets"):
|
|
out.append(f"an ids filter ({', '.join(ids)}) without facets")
|
|
if has("facets") and not ids:
|
|
out.append("facets without an ids filter")
|
|
if reads.filters.query == "ids":
|
|
out.append('query cannot be "ids"')
|
|
|
|
if key := reads.key:
|
|
if not _CONFIG_NAME.fullmatch(key.name):
|
|
out.append(f"key names no config entry ({key.name!r})")
|
|
if not key.fields:
|
|
out.append("key has no fields")
|
|
if key.env and len(key.env) != len(key.fields):
|
|
out.append("key.env does not match key.fields")
|
|
if any(not _ENV_NAME.fullmatch(name) for name in key.env):
|
|
out.append("key.env has a name that is not an environment variable")
|
|
|
|
if not reads.robots.strip():
|
|
out.append("robots is empty: record the robots.txt decision and its date")
|
|
elif not _DATE.search(reads.robots):
|
|
out.append("robots has no date: record when robots.txt was read (YYYY-MM-DD)")
|
|
|
|
if reads.scope == "search" and hasattr(module, "PARAMS"):
|
|
params = module.PARAMS
|
|
if not isinstance(params, dict) or not params:
|
|
out.append("PARAMS is not a dict of saved param -> filter dimension")
|
|
else:
|
|
dimensions = set(Filters.model_fields)
|
|
out += [f"PARAMS maps {param!r} to {dimension!r}, not a filter dimension"
|
|
for param, dimension in params.items() if dimension not in dimensions]
|
|
|
|
for name, params in SIGNATURES.items():
|
|
if has(name) and not _takes(getattr(module, name), len(params)):
|
|
out.append(f"{name} does not take ({', '.join(params)})")
|
|
return out
|