Require READS, search params and a dated robots record; record robots.txt per requested path

Every reader declares, so undeclared(), the scope() fallback, robots_page and
ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may
carry company, places gains none. Each robots record names the paths the
reader requests, from robots.txt read again on 2026-09-30. Platsbanken states
remote, HR Manager states no place.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:51:13 +02:00
1 parent 4e827665dd
commit 2349fc424b
46 files changed
+323 -100

No files matched your search

+14 -22
View File
@@ -241,7 +241,7 @@ from . import _contract
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
"Place", "Reads", "Row", "load", "problems", "scope", "undeclared"]
"Place", "Reads", "Row", "load", "problems"]
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
@@ -250,27 +250,28 @@ DEFAULT_RANK = 100
#: What every board module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
#: What a module of `scope="search"` defines.
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed")
#: What a module of `scope="search"` defines: `PARAMS` and `clean_params` are
#: what `app/jobs/sources.search_params` stores a search's params with.
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed", "PARAMS", "clean_params")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
"fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok",
"fallback_url", "detail", "facets", "clean_params", "probe_url", "probe_ok",
"probe_slugs")
def scope(module: ModuleType) -> str:
"""`board` or `search`; `board` for a module without `READS`."""
def _search(module: ModuleType) -> bool:
reads = getattr(module, "READS", None)
return reads.scope if isinstance(reads, Reads) else "board"
return isinstance(reads, Reads) and reads.scope == "search"
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
in the package, which `NAME` must equal. `READS` is checked when the
module declares one (`_contract.problems`)."""
required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED
in the package, which `NAME` must equal. A module without `READS` is
held to a board's names and fails on the missing declaration; the
declaration itself is `_contract.problems`."""
required = REQUIRED_SEARCH if _search(module) else REQUIRED
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
@@ -278,8 +279,6 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]:
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
out.append("ROBOTS_ANY_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
@@ -323,20 +322,13 @@ def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
return {module.NAME: module for module in modules}
def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]:
"""The names of the modules (of `READERS` by default) without `READS`,
read with the older signatures until they declare."""
return [name for name, module in (READERS if modules is None else modules).items()
if not hasattr(module, "READS")]
#: name -> module for every reader, in order of `(RANK, NAME)`.
READERS: dict[str, ModuleType] = load(__path__, __name__)
#: host -> its module: the readers of `scope="board"` and the undeclared.
#: host -> its module: the readers of `scope="board"`.
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if scope(module) == "board"}
if module.READS.scope == "board"}
#: provider -> its module: the readers of `scope="search"`.
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
if scope(module) == "search"}
if module.READS.scope == "search"}
+32 -12
View File
@@ -20,14 +20,14 @@ Signatures of a module that declares `READS`, with `Get` and `Page` from
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
def clean_params(params: dict[str, Any]) -> dict[str, Any] # scope="search"
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
`country`, a place name for `place`, any word for `remote`. A dimension the
host has no facet for is left out, and the runner then reads it unfiltered;
a dimension present without a wanted value means the host has none of it.
A module without `READS` keeps the signatures without `ask` until it
declares; `undeclared()` in the package lists those modules.
A module without `READS` fails the import: every reader declares.
"""
from __future__ import annotations
@@ -57,6 +57,9 @@ _CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
#: An environment variable name.
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
#: The date a robots.txt record must carry: when the file was read.
_DATE = re.compile(r"20\d\d-\d\d-\d\d")
class _Frozen(Model):
"""A declaration: immutable, extra keys refused."""
@@ -123,11 +126,13 @@ class Reads(_Frozen):
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
`whole` (the one answer), `never` (a search). `places`: `one` (the host
states one), `many` (a list per role), `copies` (the host lists one
posting per place, each with its own id). `robots`: the robots.txt
decision, with the date, taken when the reader was built. `empty`:
`suspect` when an unknown slug answers an empty list rather than an
error. `title`: `provisional` when the listing may not state the title
(a sitemap), which detail then fills.
posting per place, each with its own id), `none` (the host states no
place; every row has `places == []`). `robots`: the robots.txt
decision for the paths the reader requests, with the date the file was
read. `empty`: `suspect` when an unknown slug answers an empty list
rather than an error. `title`: `provisional` when the listing may not
state the title (a sitemap), which detail then fills; nothing reads it
yet, since `_runner.DETAIL_FIELDS` has no `title`.
"""
scope: Literal["board", "search"]
@@ -138,7 +143,7 @@ class Reads(_Frozen):
teaser: bool = False
complete: Literal["stated", "end", "whole", "never"]
filters: Filters = Filters()
places: Literal["one", "many", "copies"]
places: Literal["one", "many", "copies", "none"]
fields: Fields = Fields()
robots: str
empty: Literal["trusted", "suspect"] = "trusted"
@@ -196,8 +201,10 @@ class Row(TypedDict, total=False):
place, kept while storage reads it. `posted` is an aware datetime or
None; `description` plain text; `remote` True, False, a word or None;
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
`employment` only where the host states them, as `Fields` declares. Any
other stated key is kept in `raw` by the runner.
`employment` only where the host states them, as `Fields` declares.
`company` is the employer's name on a search row, which lists many
(`plugins/search.py` stores it); a board row carries none, the board is
the company. Any other stated key is kept in `raw` by the runner.
"""
source_id: Required[str]
@@ -217,6 +224,7 @@ class Row(TypedDict, total=False):
country: str | None
seniority: str | None
employment: str | None
company: str
#: What a declared module's functions take, positionally.
@@ -227,6 +235,7 @@ SIGNATURES: dict[str, tuple[str, ...]] = {
"rows": ("decoded", "slug"),
"detail": ("get", "slug", "row"),
"facets": ("get", "slug"),
"clean_params": ("params",),
}
@@ -245,9 +254,9 @@ def _takes(function: Callable[..., Any], count: int) -> bool:
def problems(module: ModuleType) -> list[str]:
"""What `module`'s `READS` gets wrong, on its own or against the
module's functions; empty for a module that declares none."""
module's functions; "no READS" for a module that declares none."""
if not hasattr(module, "READS"):
return []
return ["no READS: every reader declares what it reads"]
reads = module.READS
if not isinstance(reads, Reads):
return ["READS is not a Reads"]
@@ -305,6 +314,17 @@ def problems(module: ModuleType) -> list[str]:
if not reads.robots.strip():
out.append("robots is empty: record the robots.txt decision and its date")
elif not _DATE.search(reads.robots):
out.append("robots has no date: record when robots.txt was read (YYYY-MM-DD)")
if reads.scope == "search" and hasattr(module, "PARAMS"):
params = module.PARAMS
if not isinstance(params, dict) or not params:
out.append("PARAMS is not a dict of saved param -> filter dimension")
else:
dimensions = set(Filters.model_fields)
out += [f"PARAMS maps {param!r} to {dimension!r}, not a filter dimension"
for param, dimension in params.items() if dimension not in dimensions]
for name, params in SIGNATURES.items():
if has(name) and not _takes(getattr(module, name), len(params)):
+2 -1
View File
@@ -31,7 +31,8 @@ READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many", empty="suspect",
fields=Fields(pay="figures", posted="trusted", remote="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
robots=("read: api.ashbyhq.com answers /robots.txt with 401 and no rules, so "
"/posting-api/job-board/<slug> is not disallowed (checked 2026-09-30)"),
)
_API = "api.ashbyhq.com"
+3 -1
View File
@@ -55,7 +55,9 @@ READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
places="many", filters=Filters(country="one", query="one"),
fields=Fields(posted="listed", remote="stated"),
robots="read: robots.txt allows /careers (checked 2026-09-29)",
robots=("read: robots.txt disallows / and allows /api/pcsx, which covers "
"/api/pcsx/search and /api/pcsx/position_details (jobs.ericsson.com, "
"checked 2026-09-30)"),
)
NAME = "eightfold"
+2 -1
View File
@@ -35,7 +35,8 @@ READS = Reads(
scope="board", listing="paged", text="detail", complete="end",
places="many", empty="suspect",
fields=Fields(posted="listed", remote="stated", employment="stated"),
robots="read: robots.txt allows /<board> (checked 2026-09-29)",
robots=("read: POST /api/public/graphql on jobs.gem.com, whose /robots.txt answers "
"404, so nothing is disallowed (checked 2026-09-30)"),
)
NAME = "gem"
+2 -1
View File
@@ -29,7 +29,8 @@ READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="figures", posted="trusted"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
robots=("read: boards-api.greenhouse.io's robots.txt disallows /embed/ only, so "
"/v1/boards/<slug>/jobs is allowed (checked 2026-09-30)"),
)
_API = "boards-api.greenhouse.io"
+2 -2
View File
@@ -34,8 +34,8 @@ READS = Reads(
scope="board", listing="paged", text="listing", complete="end",
places="copies",
fields=Fields(pay="figures", posted="listed", remote="stated", employment="stated"),
robots="read: robots.txt allows the careers site; its robots.txt is the single-page "
"app's index page with no rules (checked 2026-09-29)",
robots=("read: /api/job-ad on <slug>.careers.hibob.com, whose /robots.txt answers the "
"single-page app's index page, no rules (dixa, checked 2026-09-30)"),
)
NAME = "hibob"
+10 -5
View File
@@ -14,9 +14,11 @@ everything for `*` except the application form. The feed is on the API host,
which has no robots.txt (a 404, which allows); it is the interface HR Manager
publishes for readers like this one, and the portal's HTML list, which the
robots.txt does shut out, is never read. The role's link is the application
form the robots.txt names as allowed, and is only shown, not fetched. The
declaration says "disallowed" all the same, and the owner decides whether the
feed is read.
form the robots.txt names as allowed, and is only shown, not fetched.
`READS.robots` records both hosts and the owner's decision.
The feed names no place (the title or the text may), so every row has
`places == []` (`places="none"`).
"""
from __future__ import annotations
@@ -34,9 +36,12 @@ from ._shared import Get, Page, is_web_url, only, parse_dt
READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="end",
places="one",
places="none",
fields=Fields(posted="listed"),
robots="disallowed by robots.txt; owner to decide",
robots=("read: the feed /JobPortal.svc/<customer>/PositionList/rss/ is on "
"api.hr-manager.net, whose /robots.txt answers 404 (checked 2026-09-30); the portal "
"candidate.hr-manager.net disallows all but /ApplicationInit.aspx? and is not "
"read: owner, 2026-09-29"),
)
NAME = "hrmanager"
+2 -1
View File
@@ -46,7 +46,8 @@ READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="stated",
places="many",
fields=Fields(posted="listed", country="stated", employment="stated"),
robots="read: robots.txt allows /jobs, with crawl-delay 5 not honoured (checked 2026-09-29)",
robots=("read: robots.txt allows /, so /api/jobs, with crawl-delay 5 not honoured "
"(jobs.booking.com, checked 2026-09-30)"),
)
NAME = "jibe"
+4 -1
View File
@@ -34,7 +34,10 @@ READS = Reads(
scope="board", listing="whole", text="detail", teaser=True, complete="whole",
places="many",
fields=Fields(posted="trusted", remote="stated", seniority="stated", employment="stated"),
robots="read: robots.txt allows /jobs/companies/ (checked 2026-09-29)",
robots=("read: robots.txt allows / for * and disallows /jobs/*/preview/, so "
"/jobs/companies/<id>/embed/v2/ on cdn.jobylon.com and /jobs/<id>-<name>/ on "
"emp.jobylon.com are allowed; it disallows / for named job crawlers (JobrapidoBot, "
"JoobleBot, AdzunaBot, ...), which this reader's agent is not (checked 2026-09-30)"),
)
NAME = "jobylon"
+2 -1
View File
@@ -32,7 +32,8 @@ READS = Reads(
places="many",
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated",
employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
robots=("read: robots.txt of api.lever.co and api.eu.lever.co allows /, so "
"/v0/postings/<site>, with crawl-delay 1, one request a board (checked 2026-09-30)"),
)
NAME = "lever"
+3 -2
View File
@@ -60,8 +60,9 @@ READS = Reads(
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
complete="stated", places="many", filters=Filters(country="ids", remote="one", query="one"),
fields=Fields(posted="trusted", remote="stated"),
robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites/<site>/jobs "
"(checked 2026-09-29)"),
robots=("read: /hcmRestApi/resources/latest/recruitingCEJobRequisitions and "
"recruitingCEJobRequisitionDetails on the tenant's host, whose /robots.txt answers "
"404, so nothing is disallowed (four seed tenants, checked 2026-09-30)"),
)
#: The page size asked for. 200 was answered in full by Nokia (600 roles in
+2 -1
View File
@@ -24,7 +24,8 @@ READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="figures", posted="trusted", seniority="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
robots=("read: /xml on <slug>.jobs.personio.de, whose /robots.txt answers 404, so "
"nothing is disallowed (penneo, checked 2026-09-30)"),
)
#: Personio's `salaryInformation/type` spellings, to the plain period word
+4 -1
View File
@@ -52,7 +52,10 @@ READS = Reads(
scope="board", listing="paged", page_size=100, text="detail", teaser=True,
complete="stated", places="many", filters=Filters(country="many", query="one"),
fields=Fields(posted="trusted", remote="stated"),
robots="read: robots.txt allows /<country>/<lang>/search-results (checked 2026-09-29)",
robots=("read: robots.txt has no rule on POST /widgets or /<country>/<lang>/job/<id>; "
"it disallows */apply, */px-widgets, */jobcart and the like (careers.king.com, "
"careers.justeattakeaway.com; www.careers.philips.com redirects /robots.txt to "
"its careers page; checked 2026-09-30)"),
)
#: The page size asked for. 100 is what the site's own search sends; `size`
+1 -1
View File
@@ -33,7 +33,7 @@ READS = Reads(
text="listing", complete="stated",
filters=Filters(query="one", place="one"),
places="many",
fields=Fields(posted="trusted", country="stated"),
fields=Fields(posted="trusted", remote="stated", country="stated"),
robots="read: the host publishes no robots.txt, /robots.txt answers 404 "
"(checked 2026-09-30)")
+2 -1
View File
@@ -26,7 +26,8 @@ READS = Reads(
places="many",
fields=Fields(pay="figures", posted="trusted", remote="stated",
seniority="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
robots=("read: <slug>.recruitee.com's robots.txt disallows /v/ only, so /api/offers/ "
"is allowed (famly, checked 2026-09-30)"),
)
#: What the board writes as a place for a role that is worked anywhere.
+3 -1
View File
@@ -49,7 +49,9 @@ READS = Reads(
places="one", empty="suspect", filters=Filters(country="one", query="one"),
fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated",
employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
robots=("read though api.smartrecruiters.com's robots.txt disallows / for every agent "
"but LinkedInBot, which alone is allowed /v1/companies/, the path of the listing "
"and of each posting: owner to decide, checked 2026-09-30"),
)
_API = "api.smartrecruiters.com"
+3 -1
View File
@@ -32,7 +32,9 @@ PATTERNS = (
READS = Reads(
scope="board", listing="paged", page_size=50, text="listing", complete="end",
places="many", fields=Fields(posted="trusted", remote="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
robots=("read: a career site's robots.txt disallows /app/, /messages/, /messenger/, "
"/facebook/tab/ and /jobs/internal/, so /jobs.rss is allowed (lunar, which "
"redirects to jobs.lunar.app, and careers.trackunit.com, checked 2026-09-30)"),
)
#: The page size, asked for as `per_page`. A feed with neither parameter
+3 -1
View File
@@ -37,7 +37,9 @@ READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
places="many",
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
robots=("read: apply.workable.com's robots.txt disallows nothing (an empty Disallow), so "
"POST /api/v3/accounts/<slug>/jobs and /api/v2/accounts/<slug>/jobs/<shortcode> "
"are allowed (checked 2026-09-30)"),
)
_API = "apply.workable.com"
+3 -1
View File
@@ -52,7 +52,9 @@ READS = Reads(
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
complete="stated", places="many", filters=Filters(country="ids", remote="ids", query="one"),
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots="read: robots.txt allows /<Site>/ (checked 2026-09-29)",
robots=("read: robots.txt allows /<Site>/ and disallows /refreshFacet/ and has no rule "
"on /wday/cxs/<tenant>/<site>/, the listing's POST .../jobs and each role's "
"JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"),
)
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins