From 2349fc424b7582f64bcc0fc2bb7ba68f832d92bb Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 15:51:13 +0200
Subject: [PATCH 1/9] Require READS, search params and a dated robots record;
record robots.txt per requested path
Every reader declares, so undeclared(), the scope() fallback, robots_page and
ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may
carry company, places gains none. Each robots record names the paths the
reader requests, from robots.txt read again on 2026-09-30. Platsbanken states
remote, HR Manager states no place.
Co-Authored-By: Claude Fable 5.1
---
plugins/readers/__init__.py | 36 ++++++-------
plugins/readers/_contract.py | 44 +++++++++++-----
plugins/readers/ashby.py | 3 +-
plugins/readers/eightfold.py | 4 +-
plugins/readers/gem.py | 3 +-
plugins/readers/greenhouse.py | 3 +-
plugins/readers/hibob.py | 4 +-
plugins/readers/hrmanager.py | 15 ++++--
plugins/readers/jibe.py | 3 +-
plugins/readers/jobylon.py | 5 +-
plugins/readers/lever.py | 3 +-
plugins/readers/oracle.py | 5 +-
plugins/readers/personio.py | 3 +-
plugins/readers/phenom.py | 5 +-
plugins/readers/platsbanken.py | 2 +-
plugins/readers/recruitee.py | 3 +-
plugins/readers/smartrecruiters.py | 4 +-
plugins/readers/teamtailor.py | 4 +-
plugins/readers/workable.py | 4 +-
plugins/readers/workday.py | 4 +-
tests/fixtures/ats/jobylon-detail-2.html | 55 ++++++++++++++++++++
tests/fixtures/ats/jobylon-detail.html | 28 +++++++++++
tests/fixtures/ats/rippling-detail-2.json | 53 ++++++++++++++++++++
tests/fixtures/ats/rippling-detail.json | 61 +++++++++++++++++++++++
tests/test_ats_ashby.py | 4 +-
tests/test_ats_bamboohr.py | 1 -
tests/test_ats_deel.py | 1 -
tests/test_ats_eightfold.py | 3 +-
tests/test_ats_gem.py | 3 +-
tests/test_ats_greenhouse.py | 4 +-
tests/test_ats_hibob.py | 3 +-
tests/test_ats_hrmanager.py | 5 +-
tests/test_ats_jibe.py | 1 -
tests/test_ats_jobylon.py | 11 ++--
tests/test_ats_lever.py | 4 +-
tests/test_ats_oracle.py | 1 -
tests/test_ats_personio.py | 3 +-
tests/test_ats_phenom.py | 1 -
tests/test_ats_pinpoint.py | 1 -
tests/test_ats_recruitee.py | 2 +-
tests/test_ats_rippling.py | 5 +-
tests/test_ats_smartrecruiters.py | 2 +-
tests/test_ats_successfactors.py | 1 -
tests/test_ats_teamtailor.py | 3 +-
tests/test_ats_workable.py | 2 +-
tests/test_ats_workday.py | 8 +--
46 files changed, 323 insertions(+), 100 deletions(-)
create mode 100644 tests/fixtures/ats/jobylon-detail-2.html
create mode 100644 tests/fixtures/ats/jobylon-detail.html
create mode 100644 tests/fixtures/ats/rippling-detail-2.json
create mode 100644 tests/fixtures/ats/rippling-detail.json
diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py
index 3866787..5e873fe 100644
--- a/plugins/readers/__init__.py
+++ b/plugins/readers/__init__.py
@@ -241,7 +241,7 @@ from . import _contract
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
- "Place", "Reads", "Row", "load", "problems", "scope", "undeclared"]
+ "Place", "Reads", "Row", "load", "problems"]
#: The rank of a module that states none: after the eight first hosts, whose
#: ranks (10 to 80) keep the order they had before there were modules.
@@ -250,27 +250,28 @@ DEFAULT_RANK = 100
#: What every board module defines.
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
-#: What a module of `scope="search"` defines.
-REQUIRED_SEARCH = ("NAME", "LABEL", "allowed")
+#: What a module of `scope="search"` defines: `PARAMS` and `clean_params` are
+#: what `app/jobs/sources.search_params` stores a search's params with.
+REQUIRED_SEARCH = ("NAME", "LABEL", "allowed", "PARAMS", "clean_params")
#: The functions a module may define; each must be callable if present.
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
- "fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok",
+ "fallback_url", "detail", "facets", "clean_params", "probe_url", "probe_ok",
"probe_slugs")
-def scope(module: ModuleType) -> str:
- """`board` or `search`; `board` for a module without `READS`."""
+def _search(module: ModuleType) -> bool:
reads = getattr(module, "READS", None)
- return reads.scope if isinstance(reads, Reads) else "board"
+ return isinstance(reads, Reads) and reads.scope == "search"
def problems(module: ModuleType, name: str | None = None) -> list[str]:
"""What `module` lacks or gets wrong against the contract, one phrase
each; empty for a module that keeps it. `name` is the module's own name
- in the package, which `NAME` must equal. `READS` is checked when the
- module declares one (`_contract.problems`)."""
- required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED
+ in the package, which `NAME` must equal. A module without `READS` is
+ held to a board's names and fails on the missing declaration; the
+ declaration itself is `_contract.problems`."""
+ required = REQUIRED_SEARCH if _search(module) else REQUIRED
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
if name is not None and getattr(module, "NAME", name) != name:
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
@@ -278,8 +279,6 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]:
out.append("LABEL is not a non-empty string")
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
out.append("KEEP_CASE is not a bool")
- if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
- out.append("ROBOTS_ANY_CASE is not a bool")
rank = getattr(module, "RANK", DEFAULT_RANK)
if not isinstance(rank, int) or isinstance(rank, bool):
out.append("RANK is not an int")
@@ -323,20 +322,13 @@ def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
return {module.NAME: module for module in modules}
-def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]:
- """The names of the modules (of `READERS` by default) without `READS`,
- read with the older signatures until they declare."""
- return [name for name, module in (READERS if modules is None else modules).items()
- if not hasattr(module, "READS")]
-
-
#: name -> module for every reader, in order of `(RANK, NAME)`.
READERS: dict[str, ModuleType] = load(__path__, __name__)
-#: host -> its module: the readers of `scope="board"` and the undeclared.
+#: host -> its module: the readers of `scope="board"`.
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
- if scope(module) == "board"}
+ if module.READS.scope == "board"}
#: provider -> its module: the readers of `scope="search"`.
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
- if scope(module) == "search"}
+ if module.READS.scope == "search"}
diff --git a/plugins/readers/_contract.py b/plugins/readers/_contract.py
index 4dfc671..7fd9706 100644
--- a/plugins/readers/_contract.py
+++ b/plugins/readers/_contract.py
@@ -20,14 +20,14 @@ Signatures of a module that declares `READS`, with `Get` and `Page` from
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
+ def clean_params(params: dict[str, Any]) -> dict[str, Any] # scope="search"
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
`country`, a place name for `place`, any word for `remote`. A dimension the
host has no facet for is left out, and the runner then reads it unfiltered;
a dimension present without a wanted value means the host has none of it.
-A module without `READS` keeps the signatures without `ask` until it
-declares; `undeclared()` in the package lists those modules.
+A module without `READS` fails the import: every reader declares.
"""
from __future__ import annotations
@@ -57,6 +57,9 @@ _CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
#: An environment variable name.
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
+#: The date a robots.txt record must carry: when the file was read.
+_DATE = re.compile(r"20\d\d-\d\d-\d\d")
+
class _Frozen(Model):
"""A declaration: immutable, extra keys refused."""
@@ -123,11 +126,13 @@ class Reads(_Frozen):
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
`whole` (the one answer), `never` (a search). `places`: `one` (the host
states one), `many` (a list per role), `copies` (the host lists one
- posting per place, each with its own id). `robots`: the robots.txt
- decision, with the date, taken when the reader was built. `empty`:
- `suspect` when an unknown slug answers an empty list rather than an
- error. `title`: `provisional` when the listing may not state the title
- (a sitemap), which detail then fills.
+ posting per place, each with its own id), `none` (the host states no
+ place; every row has `places == []`). `robots`: the robots.txt
+ decision for the paths the reader requests, with the date the file was
+ read. `empty`: `suspect` when an unknown slug answers an empty list
+ rather than an error. `title`: `provisional` when the listing may not
+ state the title (a sitemap), which detail then fills; nothing reads it
+ yet, since `_runner.DETAIL_FIELDS` has no `title`.
"""
scope: Literal["board", "search"]
@@ -138,7 +143,7 @@ class Reads(_Frozen):
teaser: bool = False
complete: Literal["stated", "end", "whole", "never"]
filters: Filters = Filters()
- places: Literal["one", "many", "copies"]
+ places: Literal["one", "many", "copies", "none"]
fields: Fields = Fields()
robots: str
empty: Literal["trusted", "suspect"] = "trusted"
@@ -196,8 +201,10 @@ class Row(TypedDict, total=False):
place, kept while storage reads it. `posted` is an aware datetime or
None; `description` plain text; `remote` True, False, a word or None;
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
- `employment` only where the host states them, as `Fields` declares. Any
- other stated key is kept in `raw` by the runner.
+ `employment` only where the host states them, as `Fields` declares.
+ `company` is the employer's name on a search row, which lists many
+ (`plugins/search.py` stores it); a board row carries none, the board is
+ the company. Any other stated key is kept in `raw` by the runner.
"""
source_id: Required[str]
@@ -217,6 +224,7 @@ class Row(TypedDict, total=False):
country: str | None
seniority: str | None
employment: str | None
+ company: str
#: What a declared module's functions take, positionally.
@@ -227,6 +235,7 @@ SIGNATURES: dict[str, tuple[str, ...]] = {
"rows": ("decoded", "slug"),
"detail": ("get", "slug", "row"),
"facets": ("get", "slug"),
+ "clean_params": ("params",),
}
@@ -245,9 +254,9 @@ def _takes(function: Callable[..., Any], count: int) -> bool:
def problems(module: ModuleType) -> list[str]:
"""What `module`'s `READS` gets wrong, on its own or against the
- module's functions; empty for a module that declares none."""
+ module's functions; "no READS" for a module that declares none."""
if not hasattr(module, "READS"):
- return []
+ return ["no READS: every reader declares what it reads"]
reads = module.READS
if not isinstance(reads, Reads):
return ["READS is not a Reads"]
@@ -305,6 +314,17 @@ def problems(module: ModuleType) -> list[str]:
if not reads.robots.strip():
out.append("robots is empty: record the robots.txt decision and its date")
+ elif not _DATE.search(reads.robots):
+ out.append("robots has no date: record when robots.txt was read (YYYY-MM-DD)")
+
+ if reads.scope == "search" and hasattr(module, "PARAMS"):
+ params = module.PARAMS
+ if not isinstance(params, dict) or not params:
+ out.append("PARAMS is not a dict of saved param -> filter dimension")
+ else:
+ dimensions = set(Filters.model_fields)
+ out += [f"PARAMS maps {param!r} to {dimension!r}, not a filter dimension"
+ for param, dimension in params.items() if dimension not in dimensions]
for name, params in SIGNATURES.items():
if has(name) and not _takes(getattr(module, name), len(params)):
diff --git a/plugins/readers/ashby.py b/plugins/readers/ashby.py
index c60efce..e25bf15 100644
--- a/plugins/readers/ashby.py
+++ b/plugins/readers/ashby.py
@@ -31,7 +31,8 @@ READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many", empty="suspect",
fields=Fields(pay="figures", posted="trusted", remote="stated", employment="stated"),
- robots="api: no robots.txt check needed (public board API), 2026-09-30",
+ robots=("read: api.ashbyhq.com answers /robots.txt with 401 and no rules, so "
+ "/posting-api/job-board/ is not disallowed (checked 2026-09-30)"),
)
_API = "api.ashbyhq.com"
diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py
index 00709c4..0d776f7 100644
--- a/plugins/readers/eightfold.py
+++ b/plugins/readers/eightfold.py
@@ -55,7 +55,9 @@ READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
places="many", filters=Filters(country="one", query="one"),
fields=Fields(posted="listed", remote="stated"),
- robots="read: robots.txt allows /careers (checked 2026-09-29)",
+ robots=("read: robots.txt disallows / and allows /api/pcsx, which covers "
+ "/api/pcsx/search and /api/pcsx/position_details (jobs.ericsson.com, "
+ "checked 2026-09-30)"),
)
NAME = "eightfold"
diff --git a/plugins/readers/gem.py b/plugins/readers/gem.py
index ee90a1e..1871833 100644
--- a/plugins/readers/gem.py
+++ b/plugins/readers/gem.py
@@ -35,7 +35,8 @@ READS = Reads(
scope="board", listing="paged", text="detail", complete="end",
places="many", empty="suspect",
fields=Fields(posted="listed", remote="stated", employment="stated"),
- robots="read: robots.txt allows / (checked 2026-09-29)",
+ robots=("read: POST /api/public/graphql on jobs.gem.com, whose /robots.txt answers "
+ "404, so nothing is disallowed (checked 2026-09-30)"),
)
NAME = "gem"
diff --git a/plugins/readers/greenhouse.py b/plugins/readers/greenhouse.py
index c79d31d..7d39b80 100644
--- a/plugins/readers/greenhouse.py
+++ b/plugins/readers/greenhouse.py
@@ -29,7 +29,8 @@ READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="figures", posted="trusted"),
- robots="api: no robots.txt check needed (public board API), 2026-09-30",
+ robots=("read: boards-api.greenhouse.io's robots.txt disallows /embed/ only, so "
+ "/v1/boards//jobs is allowed (checked 2026-09-30)"),
)
_API = "boards-api.greenhouse.io"
diff --git a/plugins/readers/hibob.py b/plugins/readers/hibob.py
index a1a9204..2d65444 100644
--- a/plugins/readers/hibob.py
+++ b/plugins/readers/hibob.py
@@ -34,8 +34,8 @@ READS = Reads(
scope="board", listing="paged", text="listing", complete="end",
places="copies",
fields=Fields(pay="figures", posted="listed", remote="stated", employment="stated"),
- robots="read: robots.txt allows the careers site; its robots.txt is the single-page "
- "app's index page with no rules (checked 2026-09-29)",
+ robots=("read: /api/job-ad on .careers.hibob.com, whose /robots.txt answers the "
+ "single-page app's index page, no rules (dixa, checked 2026-09-30)"),
)
NAME = "hibob"
diff --git a/plugins/readers/hrmanager.py b/plugins/readers/hrmanager.py
index 5bf54a8..e5c4836 100644
--- a/plugins/readers/hrmanager.py
+++ b/plugins/readers/hrmanager.py
@@ -14,9 +14,11 @@ everything for `*` except the application form. The feed is on the API host,
which has no robots.txt (a 404, which allows); it is the interface HR Manager
publishes for readers like this one, and the portal's HTML list, which the
robots.txt does shut out, is never read. The role's link is the application
-form the robots.txt names as allowed, and is only shown, not fetched. The
-declaration says "disallowed" all the same, and the owner decides whether the
-feed is read.
+form the robots.txt names as allowed, and is only shown, not fetched.
+`READS.robots` records both hosts and the owner's decision.
+
+The feed names no place (the title or the text may), so every row has
+`places == []` (`places="none"`).
"""
from __future__ import annotations
@@ -34,9 +36,12 @@ from ._shared import Get, Page, is_web_url, only, parse_dt
READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="end",
- places="one",
+ places="none",
fields=Fields(posted="listed"),
- robots="disallowed by robots.txt; owner to decide",
+ robots=("read: the feed /JobPortal.svc//PositionList/rss/ is on "
+ "api.hr-manager.net, whose /robots.txt answers 404 (checked 2026-09-30); the portal "
+ "candidate.hr-manager.net disallows all but /ApplicationInit.aspx? and is not "
+ "read: owner, 2026-09-29"),
)
NAME = "hrmanager"
diff --git a/plugins/readers/jibe.py b/plugins/readers/jibe.py
index 7e2e617..2e6d4f6 100644
--- a/plugins/readers/jibe.py
+++ b/plugins/readers/jibe.py
@@ -46,7 +46,8 @@ READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="stated",
places="many",
fields=Fields(posted="listed", country="stated", employment="stated"),
- robots="read: robots.txt allows /jobs, with crawl-delay 5 not honoured (checked 2026-09-29)",
+ robots=("read: robots.txt allows /, so /api/jobs, with crawl-delay 5 not honoured "
+ "(jobs.booking.com, checked 2026-09-30)"),
)
NAME = "jibe"
diff --git a/plugins/readers/jobylon.py b/plugins/readers/jobylon.py
index dc519b7..cf063bb 100644
--- a/plugins/readers/jobylon.py
+++ b/plugins/readers/jobylon.py
@@ -34,7 +34,10 @@ READS = Reads(
scope="board", listing="whole", text="detail", teaser=True, complete="whole",
places="many",
fields=Fields(posted="trusted", remote="stated", seniority="stated", employment="stated"),
- robots="read: robots.txt allows /jobs/companies/ (checked 2026-09-29)",
+ robots=("read: robots.txt allows / for * and disallows /jobs/*/preview/, so "
+ "/jobs/companies//embed/v2/ on cdn.jobylon.com and /jobs/-/ on "
+ "emp.jobylon.com are allowed; it disallows / for named job crawlers (JobrapidoBot, "
+ "JoobleBot, AdzunaBot, ...), which this reader's agent is not (checked 2026-09-30)"),
)
NAME = "jobylon"
diff --git a/plugins/readers/lever.py b/plugins/readers/lever.py
index 83bb2f5..90256f2 100644
--- a/plugins/readers/lever.py
+++ b/plugins/readers/lever.py
@@ -32,7 +32,8 @@ READS = Reads(
places="many",
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated",
employment="stated"),
- robots="api: no robots.txt check needed (public board API), 2026-09-30",
+ robots=("read: robots.txt of api.lever.co and api.eu.lever.co allows /, so "
+ "/v0/postings/, with crawl-delay 1, one request a board (checked 2026-09-30)"),
)
NAME = "lever"
diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py
index 02bd46f..c61b985 100644
--- a/plugins/readers/oracle.py
+++ b/plugins/readers/oracle.py
@@ -60,8 +60,9 @@ READS = Reads(
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
complete="stated", places="many", filters=Filters(country="ids", remote="one", query="one"),
fields=Fields(posted="trusted", remote="stated"),
- robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites//jobs "
- "(checked 2026-09-29)"),
+ robots=("read: /hcmRestApi/resources/latest/recruitingCEJobRequisitions and "
+ "recruitingCEJobRequisitionDetails on the tenant's host, whose /robots.txt answers "
+ "404, so nothing is disallowed (four seed tenants, checked 2026-09-30)"),
)
#: The page size asked for. 200 was answered in full by Nokia (600 roles in
diff --git a/plugins/readers/personio.py b/plugins/readers/personio.py
index 8768f13..bb34b4d 100644
--- a/plugins/readers/personio.py
+++ b/plugins/readers/personio.py
@@ -24,7 +24,8 @@ READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="figures", posted="trusted", seniority="stated", employment="stated"),
- robots="api: no robots.txt check needed (public board API), 2026-09-30",
+ robots=("read: /xml on .jobs.personio.de, whose /robots.txt answers 404, so "
+ "nothing is disallowed (penneo, checked 2026-09-30)"),
)
#: Personio's `salaryInformation/type` spellings, to the plain period word
diff --git a/plugins/readers/phenom.py b/plugins/readers/phenom.py
index d1dc5e0..944255a 100644
--- a/plugins/readers/phenom.py
+++ b/plugins/readers/phenom.py
@@ -52,7 +52,10 @@ READS = Reads(
scope="board", listing="paged", page_size=100, text="detail", teaser=True,
complete="stated", places="many", filters=Filters(country="many", query="one"),
fields=Fields(posted="trusted", remote="stated"),
- robots="read: robots.txt allows ///search-results (checked 2026-09-29)",
+ robots=("read: robots.txt has no rule on POST /widgets or ///job/; "
+ "it disallows */apply, */px-widgets, */jobcart and the like (careers.king.com, "
+ "careers.justeattakeaway.com; www.careers.philips.com redirects /robots.txt to "
+ "its careers page; checked 2026-09-30)"),
)
#: The page size asked for. 100 is what the site's own search sends; `size`
diff --git a/plugins/readers/platsbanken.py b/plugins/readers/platsbanken.py
index c876266..19e3183 100644
--- a/plugins/readers/platsbanken.py
+++ b/plugins/readers/platsbanken.py
@@ -33,7 +33,7 @@ READS = Reads(
text="listing", complete="stated",
filters=Filters(query="one", place="one"),
places="many",
- fields=Fields(posted="trusted", country="stated"),
+ fields=Fields(posted="trusted", remote="stated", country="stated"),
robots="read: the host publishes no robots.txt, /robots.txt answers 404 "
"(checked 2026-09-30)")
diff --git a/plugins/readers/recruitee.py b/plugins/readers/recruitee.py
index 36f1aab..cce230b 100644
--- a/plugins/readers/recruitee.py
+++ b/plugins/readers/recruitee.py
@@ -26,7 +26,8 @@ READS = Reads(
places="many",
fields=Fields(pay="figures", posted="trusted", remote="stated",
seniority="stated", employment="stated"),
- robots="api: no robots.txt check needed (public board API), 2026-09-30",
+ robots=("read: .recruitee.com's robots.txt disallows /v/ only, so /api/offers/ "
+ "is allowed (famly, checked 2026-09-30)"),
)
#: What the board writes as a place for a role that is worked anywhere.
diff --git a/plugins/readers/smartrecruiters.py b/plugins/readers/smartrecruiters.py
index 86b9079..f8bb5a9 100644
--- a/plugins/readers/smartrecruiters.py
+++ b/plugins/readers/smartrecruiters.py
@@ -49,7 +49,9 @@ READS = Reads(
places="one", empty="suspect", filters=Filters(country="one", query="one"),
fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated",
employment="stated"),
- robots="api: no robots.txt check needed (public board API), 2026-09-30",
+ robots=("read though api.smartrecruiters.com's robots.txt disallows / for every agent "
+ "but LinkedInBot, which alone is allowed /v1/companies/, the path of the listing "
+ "and of each posting: owner to decide, checked 2026-09-30"),
)
_API = "api.smartrecruiters.com"
diff --git a/plugins/readers/teamtailor.py b/plugins/readers/teamtailor.py
index 937d1f8..328fc6c 100644
--- a/plugins/readers/teamtailor.py
+++ b/plugins/readers/teamtailor.py
@@ -32,7 +32,9 @@ PATTERNS = (
READS = Reads(
scope="board", listing="paged", page_size=50, text="listing", complete="end",
places="many", fields=Fields(posted="trusted", remote="stated"),
- robots="api: no robots.txt check needed (public board API), 2026-09-30",
+ robots=("read: a career site's robots.txt disallows /app/, /messages/, /messenger/, "
+ "/facebook/tab/ and /jobs/internal/, so /jobs.rss is allowed (lunar, which "
+ "redirects to jobs.lunar.app, and careers.trackunit.com, checked 2026-09-30)"),
)
#: The page size, asked for as `per_page`. A feed with neither parameter
diff --git a/plugins/readers/workable.py b/plugins/readers/workable.py
index 8b0a968..36aa055 100644
--- a/plugins/readers/workable.py
+++ b/plugins/readers/workable.py
@@ -37,7 +37,9 @@ READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
places="many",
fields=Fields(posted="trusted", remote="stated", employment="stated"),
- robots="api: no robots.txt check needed (public board API), 2026-09-30",
+ robots=("read: apply.workable.com's robots.txt disallows nothing (an empty Disallow), so "
+ "POST /api/v3/accounts//jobs and /api/v2/accounts//jobs/ "
+ "are allowed (checked 2026-09-30)"),
)
_API = "apply.workable.com"
diff --git a/plugins/readers/workday.py b/plugins/readers/workday.py
index 875a983..02a253f 100644
--- a/plugins/readers/workday.py
+++ b/plugins/readers/workday.py
@@ -52,7 +52,9 @@ READS = Reads(
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
complete="stated", places="many", filters=Filters(country="ids", remote="ids", query="one"),
fields=Fields(posted="trusted", remote="stated", employment="stated"),
- robots="read: robots.txt allows // (checked 2026-09-29)",
+ robots=("read: robots.txt allows // and disallows /refreshFacet/ and has no rule "
+ "on /wday/cxs///, the listing's POST .../jobs and each role's "
+ "JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"),
)
#: One group for `.` and one for the site, which discovery joins
diff --git a/tests/fixtures/ats/jobylon-detail-2.html b/tests/fixtures/ats/jobylon-detail-2.html
new file mode 100644
index 0000000..d1da1cb
--- /dev/null
+++ b/tests/fixtures/ats/jobylon-detail-2.html
@@ -0,0 +1,55 @@
+
+CDL Driver Recruiter - Einride | Jobylon
+
+
diff --git a/tests/fixtures/ats/jobylon-detail.html b/tests/fixtures/ats/jobylon-detail.html
new file mode 100644
index 0000000..a03d447
--- /dev/null
+++ b/tests/fixtures/ats/jobylon-detail.html
@@ -0,0 +1,28 @@
+
+Financial Controller - Einride | Jobylon
+
+
diff --git a/tests/fixtures/ats/rippling-detail-2.json b/tests/fixtures/ats/rippling-detail-2.json
new file mode 100644
index 0000000..2bf5f4c
--- /dev/null
+++ b/tests/fixtures/ats/rippling-detail-2.json
@@ -0,0 +1,53 @@
+{
+ "uuid": "0fbcec57-3012-4271-96bf-12b145cf6dea",
+ "name": "General Applications: Engineering",
+ "description": {
+ "company": "
Not finding the perfect engineering role on our careers page?
We are constantly seeking talented individuals to join our team. If you are a Software Engineer and haven\u2019t found the perfect role on our careers page, don\u2019t worry \u2014 you\u2019ve come to the right place.
We strongly believe that each person has something unique to bring to the table. Whether you\u2019re a recent graduate or a seasoned professional, we\u2019re genuinely excited to hear from you. At Prisma, we work with modern technologies, including TypeScript",
+ "role": "
What is it like to work at Prisma?
You're likely to be a great fit for the team if you ...
... have an ownership mindset. As part of a small team, you'll have an outsize impact on decisions and the way things are executed.
... stay nimble. As a growing startup, the team and strategy evolves quickly. Be ready to re-calibrate often and stay flexible.
Rippling gives businesses one place to run HR, IT, and Finance. It brings together all of the workforce systems that are normally scattered across a company, like payroll, expenses, benefits, and computers. For the first time ever, you can manage and automate every part of the employee lifecycle in a single system.
Take onboarding, for example. With Rippling, you can hire a new employee anywhere in the world and set up their payroll, corporate card, computer, benefits, and even third-party apps like Slack and Microsoft 365\u2014all within 90 seconds.",
+ "role": "
About The Role
Rippling Infrastructure operates as a scaled engineering organization responsible for the mission-critical systems that power every product and platform across the company. We support nearly 1,000 engineers globally and deliver the foundational infrastructure that enables Rippling's products to scale reliably and efficiently.
"
+ },
+ "workLocations": [
+ "San Francisco, CA",
+ "Seattle, WA",
+ "New York, NY"
+ ],
+ "department": {
+ "name": "Platform",
+ "base_department": "Engineering",
+ "department_tree": [
+ "Engineering",
+ "Platform"
+ ]
+ },
+ "employmentType": {
+ "label": "SALARIED_FT",
+ "id": "Salaried, full-time"
+ },
+ "createdOn": "2025-04-07T12:29:19.348000-07:00",
+ "eeocQuestionnaireEnabled": true,
+ "eeocQuestionnaireEnabledForJobPost": true,
+ "url": "https://ats.rippling.com/rippling/jobs/2f2ae4c8-ec56-4d1f-8f2a-0d295674e19c",
+ "board": {
+ "boardType": "CUSTOM",
+ "slug": "rippling",
+ "banner": null,
+ "title": "Current Openings",
+ "subtitle": null,
+ "boardURL": "http://www.rippling.com/careers/open-roles",
+ "fontType": null,
+ "linkColor": null,
+ "buttonColor": null,
+ "buttonTextColor": null,
+ "noOpeningsMessage": null,
+ "groupJobsByLocation": false,
+ "showBoardLogoOnJobPost": false,
+ "showCompanyInfoUnderJobPost": false,
+ "companyName": "Rippling"
+ },
+ "payRangeDetails": [
+ {
+ "location": "US Tier 1",
+ "currency": "USD",
+ "frequency": "YEAR",
+ "rangeStart": 216000.0,
+ "rangeEnd": 378000.0,
+ "isRemote": false
+ }
+ ],
+ "applicationConfirmationTemplate": "6348729c5a9a6e6df0bb6f64",
+ "hasAIEvaluationsEnabled": true,
+ "companyName": "Rippling",
+ "unlistedFromSearch": false,
+ "jsonLd": null
+}
diff --git a/tests/test_ats_ashby.py b/tests/test_ats_ashby.py
index b776798..3cab493 100644
--- a/tests/test_ats_ashby.py
+++ b/tests/test_ats_ashby.py
@@ -35,10 +35,10 @@ def test_the_employment_type_is_the_hosts_word() -> None:
assert (first["employment"], second["employment"]) == ("FullTime", "Contract")
-def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
+def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
reads = ashby.READS
assert (reads.listing, reads.text, reads.places, reads.empty) == (
"whole", "listing", "many", "suspect")
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
- assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
+ assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
assert ashby.feed_url("acme", Ask()).endswith("/job-board/acme?includeCompensation=true")
diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py
index c7cd28a..cdb773d 100644
--- a/tests/test_ats_bamboohr.py
+++ b/tests/test_ats_bamboohr.py
@@ -156,7 +156,6 @@ def test_it_declares_what_it_reads() -> None:
assert (reads.listing, reads.text, reads.places, reads.fields.posted) == (
"whole", "detail", "one", "trusted")
assert reads.robots.startswith("read: robots.txt allows /careers")
- assert not hasattr(bamboohr, "robots_page")
def test_the_slug_the_urls_and_the_allow_list() -> None:
diff --git a/tests/test_ats_deel.py b/tests/test_ats_deel.py
index 1bb1127..7a313e4 100644
--- a/tests/test_ats_deel.py
+++ b/tests/test_ats_deel.py
@@ -113,7 +113,6 @@ def test_it_declares_a_whole_listing_read_from_the_page() -> None:
assert (reads.listing, reads.text, reads.places, reads.complete) == (
"whole", "listing", "many", "whole")
assert reads.fields.pay == "line" and "/api/ is disallowed and not read" in reads.robots
- assert not hasattr(deel, "robots_page")
# ------------------------------------------------------- slugs and hosts
diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py
index 7aa99a0..171ac68 100644
--- a/tests/test_ats_eightfold.py
+++ b/tests/test_ats_eightfold.py
@@ -210,5 +210,4 @@ def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request(
"paged", 10, "detail", "stated", "many")
assert reads.filters.declared() == {"country": "one", "query": "one"}
assert reads.fields.posted == "listed"
- assert reads.robots.startswith("read: robots.txt allows /careers")
- assert not hasattr(eightfold, "robots_page")
+ assert "/api/pcsx/search" in reads.robots and "/api/pcsx/position_details" in reads.robots
diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py
index 97395ca..dee67af 100644
--- a/tests/test_ats_gem.py
+++ b/tests/test_ats_gem.py
@@ -103,8 +103,7 @@ def test_it_declares_one_page_that_may_be_empty_by_mistake() -> None:
reads = gem.READS
assert (reads.listing, reads.text, reads.places, reads.empty) == (
"paged", "detail", "many", "suspect")
- assert reads.robots.startswith("read: robots.txt allows /")
- assert not hasattr(gem, "robots_page")
+ assert "POST /api/public/graphql" in reads.robots
@pytest.mark.parametrize(("body", "message"), [
diff --git a/tests/test_ats_greenhouse.py b/tests/test_ats_greenhouse.py
index 08594de..f49b2d4 100644
--- a/tests/test_ats_greenhouse.py
+++ b/tests/test_ats_greenhouse.py
@@ -43,10 +43,10 @@ def test_without_offices_the_name_is_the_place() -> None:
assert nothing["places"] == [] and nothing["location"] == ""
-def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
+def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
reads = greenhouse.READS
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
- assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
+ assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
assert greenhouse.feed_url("acme", Ask()).endswith("/boards/acme/jobs?content=true&pay_transparency=true")
assert greenhouse.fallback_url("acme").endswith("/boards/acme/jobs")
diff --git a/tests/test_ats_hibob.py b/tests/test_ats_hibob.py
index c73dded..a97e95d 100644
--- a/tests/test_ats_hibob.py
+++ b/tests/test_ats_hibob.py
@@ -163,8 +163,7 @@ def test_it_declares_copies_with_a_page_and_a_robots_decision() -> None:
reads = hibob.READS
assert (reads.listing, reads.text, reads.places, reads.complete) == (
"paged", "listing", "copies", "end")
- assert reads.robots.startswith("read: robots.txt allows")
- assert not hasattr(hibob, "robots_page")
+ assert reads.robots.startswith("read: /api/job-ad")
# ------------------------------------------------------- slugs and hosts
diff --git a/tests/test_ats_hrmanager.py b/tests/test_ats_hrmanager.py
index 4b57f93..1a1aa1e 100644
--- a/tests/test_ats_hrmanager.py
+++ b/tests/test_ats_hrmanager.py
@@ -122,9 +122,8 @@ def test_the_feed_states_no_place_and_the_robots_decision_is_the_owners() -> Non
assert all(r["places"] == [] for r in rows)
reads = hrmanager.READS
assert (reads.listing, reads.page_size, reads.text, reads.places) == (
- "paged", 100, "listing", "one")
- assert reads.robots == "disallowed by robots.txt; owner to decide"
- assert not hasattr(hrmanager, "robots_page")
+ "paged", 100, "listing", "none")
+ assert "api.hr-manager.net" in reads.robots and "owner, 2026-09-29" in reads.robots
# ------------------------------------------------------- slugs and hosts
diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py
index 0b03125..61e8bc7 100644
--- a/tests/test_ats_jibe.py
+++ b/tests/test_ats_jibe.py
@@ -184,4 +184,3 @@ def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
"paged", 100, "listing", "stated", "many")
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")
- assert not hasattr(jibe, "robots_page")
diff --git a/tests/test_ats_jobylon.py b/tests/test_ats_jobylon.py
index b744334..e4ce3a3 100644
--- a/tests/test_ats_jobylon.py
+++ b/tests/test_ats_jobylon.py
@@ -87,8 +87,8 @@ def _get(pages: dict[str, bytes], asked: list[str] | None = None) -> Any:
def test_detail_reads_the_role_pages_json_ld() -> None:
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
asked: list[str] = []
- pages = {ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
- ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
+ pages = {ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
+ ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
first = jobylon.detail(_get(pages, asked), "3143", rows["385191"])
assert asked == [ROLE]
assert set(first) <= set(ats.DETAIL_FIELDS) | {"places"}
@@ -116,8 +116,8 @@ def test_detail_without_a_job_posting_fails_the_read() -> None:
def test_the_board_is_one_request_and_each_role_is_read_on_its_own_page() -> None:
asked: list[str] = []
- pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
- ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
+ pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
+ ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
get = _get(pages, asked)
rows = jobylon.parse(get(jobylon.feed_url("3143", Ask())), "3143")
assert len(rows) == 5 and asked == [FEED]
@@ -150,8 +150,7 @@ def test_it_declares_a_whole_listing_with_a_teaser_and_detail() -> None:
assert (reads.listing, reads.text, reads.teaser, reads.places) == (
"whole", "detail", True, "many")
assert (reads.fields.posted, reads.fields.seniority) == ("trusted", "stated")
- assert reads.robots.startswith("read: robots.txt allows /jobs/companies/")
- assert not hasattr(jobylon, "robots_page")
+ assert "/jobs/companies//embed/v2/" in reads.robots
# ------------------------------------------------------- slugs and hosts
diff --git a/tests/test_ats_lever.py b/tests/test_ats_lever.py
index 1b4a2af..8377197 100644
--- a/tests/test_ats_lever.py
+++ b/tests/test_ats_lever.py
@@ -33,9 +33,9 @@ def test_the_commitment_is_the_hosts_word() -> None:
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
-def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
+def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
reads = lever.READS
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
assert reads.fields.posted == "trusted" and reads.fields.country == "stated"
- assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
+ assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
assert lever.feed_url("eu/tomtom", Ask()) == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
diff --git a/tests/test_ats_oracle.py b/tests/test_ats_oracle.py
index 4442319..aba4256 100644
--- a/tests/test_ats_oracle.py
+++ b/tests/test_ats_oracle.py
@@ -253,7 +253,6 @@ def test_the_careers_page_is_the_candidate_site() -> None:
url = f"https://{HOST}/hcmUI/CandidateExperience/en/sites/CX_1/jobs"
assert oracle.careers_url(SLUG) == ats.careers_url(f"oracle:{SLUG}") == url
assert oracle.allowed(SLUG)(httpx.URL(url).host)
- assert not hasattr(oracle, "robots_page") # decided when built, see READS.robots
def test_a_slug_written_with_capitals_names_the_same_pod() -> None:
diff --git a/tests/test_ats_personio.py b/tests/test_ats_personio.py
index e84f35c..d9aafdb 100644
--- a/tests/test_ats_personio.py
+++ b/tests/test_ats_personio.py
@@ -24,8 +24,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
reads = personio.READS
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
- assert reads.robots.startswith("api: no robots.txt check needed")
- assert not hasattr(personio, "robots_page")
+ assert reads.robots.startswith("read: /xml")
assert personio.feed_url("acme", Ask()) == "https://acme.jobs.personio.de/xml"
diff --git a/tests/test_ats_phenom.py b/tests/test_ats_phenom.py
index 0cd40c8..a6aee31 100644
--- a/tests/test_ats_phenom.py
+++ b/tests/test_ats_phenom.py
@@ -289,7 +289,6 @@ def test_the_careers_page_is_the_search_page() -> None:
url = "https://careers.king.com/us/en/search-results"
assert phenom.careers_url(SLUG) == ats.careers_url(f"phenom:{SLUG}") == url
assert phenom.allowed(SLUG)(httpx.URL(url).host)
- assert not hasattr(phenom, "robots_page") # decided when built, see READS.robots
assert phenom.careers_url("www.careers.philips.com/global/en") == \
"https://www.careers.philips.com/global/en/search-results"
diff --git a/tests/test_ats_pinpoint.py b/tests/test_ats_pinpoint.py
index e091caf..e568e3e 100644
--- a/tests/test_ats_pinpoint.py
+++ b/tests/test_ats_pinpoint.py
@@ -116,7 +116,6 @@ def test_the_board_is_read_through_the_app_and_is_complete() -> None:
def test_the_slug_the_urls_and_the_allow_list() -> None:
assert pinpoint.slug_ok("xeneta") and pinpoint.slug_ok("Xeneta") and pinpoint.slug_ok("a-b1")
assert pinpoint.careers_url("xeneta") == "https://xeneta.pinpointhq.com/"
- assert not hasattr(pinpoint, "robots_page") # decided when built, see READS.robots
assert pinpoint.feed_url("Xeneta") == "https://xeneta.pinpointhq.com/postings.json"
allow = pinpoint.allowed("Xeneta")
assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com")
diff --git a/tests/test_ats_recruitee.py b/tests/test_ats_recruitee.py
index cb20ea8..cd42328 100644
--- a/tests/test_ats_recruitee.py
+++ b/tests/test_ats_recruitee.py
@@ -25,7 +25,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
reads = recruitee.READS
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
assert reads.fields.posted == "trusted" and reads.fields.country == "none"
- assert reads.robots.startswith("api: no robots.txt check needed")
+ assert "/api/offers/ is allowed" in reads.robots
assert recruitee.feed_url("acme", Ask()) == "https://acme.recruitee.com/api/offers/"
diff --git a/tests/test_ats_rippling.py b/tests/test_ats_rippling.py
index 0f10598..ed5a89c 100644
--- a/tests/test_ats_rippling.py
+++ b/tests/test_ats_rippling.py
@@ -82,7 +82,7 @@ def _get(answer: bytes, asked: list[str] | None = None) -> Any:
def test_detail_reads_description_places_date_and_a_single_yearly_range() -> None:
row = {r["source_id"]: r for r in rippling.parse(BOARD, "rippling")}[DIRECTOR]
asked: list[str] = []
- found = rippling.detail(_get((FIXTURES / "rippling-role.json").read_bytes(), asked),
+ found = rippling.detail(_get((FIXTURES / "rippling-detail.json").read_bytes(), asked),
"rippling", row)
assert asked == [f"https://api.rippling.com/platform/api/ats/v1/board/rippling/jobs/{DIRECTOR}"]
assert found["description"].startswith("About The Role")
@@ -96,7 +96,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non
def test_detail_without_pay_leaves_the_pay_columns_out() -> None:
- found = rippling.detail(_get((FIXTURES / "rippling-role-2.json").read_bytes()),
+ found = rippling.detail(_get((FIXTURES / "rippling-detail-2.json").read_bytes()),
"prisma-careers", {"source_id": "x"})
assert found["description"].startswith("What is it like to work at Prisma?")
assert found["location"] == "Remote (Germany)"
@@ -165,7 +165,6 @@ def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
def test_the_urls() -> None:
assert rippling.feed_url("rippling") == FEED
assert rippling.careers_url("prisma-careers") == "https://ats.rippling.com/prisma-careers/jobs"
- assert not hasattr(rippling, "robots_page") # decided when built, see READS.robots
allow = rippling.allowed("rippling")
assert allow("api.rippling.com") and allow("ATS.RIPPLING.COM")
for host in ("rippling.com", "www.rippling.com", "api.rippling.com.evil.example", "localhost"):
diff --git a/tests/test_ats_smartrecruiters.py b/tests/test_ats_smartrecruiters.py
index c291f72..8a829ba 100644
--- a/tests/test_ats_smartrecruiters.py
+++ b/tests/test_ats_smartrecruiters.py
@@ -35,7 +35,7 @@ def test_the_declaration_says_what_the_list_gives() -> None:
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.empty) == \
("paged", 100, "one", "detail", "suspect")
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
- assert reads.robots.startswith("api: no robots.txt check needed")
+ assert reads.robots.startswith("read though") and "owner to decide" in reads.robots
def test_page_reads_by_offset_and_takes_the_ask() -> None:
diff --git a/tests/test_ats_successfactors.py b/tests/test_ats_successfactors.py
index ea237b7..9e0fb33 100644
--- a/tests/test_ats_successfactors.py
+++ b/tests/test_ats_successfactors.py
@@ -451,7 +451,6 @@ def test_careers_url_is_the_search_page() -> None:
page = successfactors.careers_url(slug)
assert httpx.URL(page).host == slug.split("/")[0]
assert successfactors.allowed(slug)(httpx.URL(page).host)
- assert not hasattr(successfactors, "robots_page") # decided when built, see READS.robots
def test_the_module_is_case_sensitive_about_the_prefix() -> None:
diff --git a/tests/test_ats_teamtailor.py b/tests/test_ats_teamtailor.py
index 46502ad..bd8ed66 100644
--- a/tests/test_ats_teamtailor.py
+++ b/tests/test_ats_teamtailor.py
@@ -24,8 +24,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.complete) == \
("paged", 50, "many", "listing", "end")
assert reads.fields.posted == "trusted" and reads.fields.remote == "stated"
- assert reads.robots.startswith("api: no robots.txt check needed")
- assert not hasattr(teamtailor, "robots_page")
+ assert "/jobs.rss is allowed" in reads.robots
def test_page_takes_the_ask_and_reads_the_same_url() -> None:
diff --git a/tests/test_ats_workable.py b/tests/test_ats_workable.py
index 021eb05..6a71de2 100644
--- a/tests/test_ats_workable.py
+++ b/tests/test_ats_workable.py
@@ -30,7 +30,7 @@ def test_the_declaration_says_what_the_list_gives() -> None:
assert (reads.listing, reads.page_size, reads.places, reads.text) == \
("paged", 10, "many", "detail")
assert reads.fields.posted == "trusted" and reads.fields.employment == "stated"
- assert reads.robots.startswith("api: no robots.txt check needed")
+ assert "/api/v3/accounts//jobs" in reads.robots
def test_page_takes_the_ask_and_posts_the_same_body() -> None:
diff --git a/tests/test_ats_workday.py b/tests/test_ats_workday.py
index f0e8dbe..a089e9b 100644
--- a/tests/test_ats_workday.py
+++ b/tests/test_ats_workday.py
@@ -446,10 +446,10 @@ def test_the_case_of_the_site_is_kept() -> None:
assert "workday" in ats.HOSTS and ats.HOSTS["workday"] == "Workday"
-def test_the_runtime_robots_check_has_left_the_module() -> None:
- """Decided when the reader was built (`READS.robots`): nothing to ask now."""
- assert not hasattr(workday, "robots_page") and not hasattr(workday, "ROBOTS_ANY_CASE")
- assert "robots.txt allows //" in workday.READS.robots
+def test_the_robots_record_names_the_api_path() -> None:
+ """Decided when the reader was built (`READS.robots`), for the paths it
+ requests, not only the career site's."""
+ assert "/wday/cxs///" in workday.READS.robots
# -------------------------------------------------------------- discovery
From 5f1b9e93e15158efd2c988d30603bebc4920931d Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 15:51:13 +0200
Subject: [PATCH 2/9] Check each declared field against the reader's fixtures
A missing fixture fails with the path to add. Detail fixtures are named
-detail*, the rows of a paged read state a total as complete says,
and a host that states words has a prep table for them.
Co-Authored-By: Claude Fable 5.1
---
tests/fixtures/ats/hibob-places.json | 2 +-
tests/fixtures/ats/jobylon-role-2.html | 55 ---
tests/fixtures/ats/jobylon-role.html | 28 --
tests/fixtures/ats/pinpoint-places.json | 2 +-
tests/fixtures/ats/rippling-role-2.json | 53 ---
tests/fixtures/ats/rippling-role.json | 61 ---
.../fixtures/ats/smartrecruiters-places.json | 4 +-
tests/test_readers_contract.py | 378 +++++++++++++-----
8 files changed, 294 insertions(+), 289 deletions(-)
delete mode 100644 tests/fixtures/ats/jobylon-role-2.html
delete mode 100644 tests/fixtures/ats/jobylon-role.html
delete mode 100644 tests/fixtures/ats/rippling-role-2.json
delete mode 100644 tests/fixtures/ats/rippling-role.json
diff --git a/tests/fixtures/ats/hibob-places.json b/tests/fixtures/ats/hibob-places.json
index 6948a15..1e0d6c1 100644
--- a/tests/fixtures/ats/hibob-places.json
+++ b/tests/fixtures/ats/hibob-places.json
@@ -1 +1 @@
-{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]}
\ No newline at end of file
+{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":50000,"payTransparencyMaxSalary":70000,"payTransparencySalaryCurrency":"EUR","payTransparencySalaryPayPeriod":"Yearly","payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]}
\ No newline at end of file
diff --git a/tests/fixtures/ats/jobylon-role-2.html b/tests/fixtures/ats/jobylon-role-2.html
deleted file mode 100644
index d1da1cb..0000000
--- a/tests/fixtures/ats/jobylon-role-2.html
+++ /dev/null
@@ -1,55 +0,0 @@
-
-CDL Driver Recruiter - Einride | Jobylon
-
-
diff --git a/tests/fixtures/ats/jobylon-role.html b/tests/fixtures/ats/jobylon-role.html
deleted file mode 100644
index a03d447..0000000
--- a/tests/fixtures/ats/jobylon-role.html
+++ /dev/null
@@ -1,28 +0,0 @@
-
-Financial Controller - Einride | Jobylon
-
-
diff --git a/tests/fixtures/ats/pinpoint-places.json b/tests/fixtures/ats/pinpoint-places.json
index 6cffdcb..3379649 100644
--- a/tests/fixtures/ats/pinpoint-places.json
+++ b/tests/fixtures/ats/pinpoint-places.json
@@ -1 +1 @@
-{"data":[{"id":"581878","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe will help you \u003cstrong\u003egrow your career:\u003c/strong\u003e We help you identify areas of growth for you, whether you want to grow as a specialist in your field or eventually want to go into a leadership/management role.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person from time to time.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will receive an\u003cstrong\u003e individual education budget \u003c/strong\u003eand can educate yourself during work hours: you can spend several hours each month attending courses, reading books, or getting involved in communities.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFull-time position, employed through GI Group, our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eLooking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Job Benefits","compensation":null,"compensation_minimum":null,"compensation_maximum":null,"compensation_currency":null,"compensation_frequency":null,"compensation_visible":false,"deadline_at":null,"description":"\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eData Detective Wanted to join our Data \u0026amp; Analytics Team\u003c/strong\u003e \u003cbr\u003e\u003cbr\u003e\u003cem\u003eLocation: Bucharest, Romania\u003c/em\u003e\u003cbr\u003e\u003cbr\u003eAs a \u003cstrong\u003eData Operations Analyst \u003c/strong\u003eat Xeneta, you’ll learn how to master complex data analytics within the world of supply chain and logistics. Xeneta is one of the world's leading data platforms providing market intelligence to the ocean and air freight industry. We have a big team of skilled Data Analysts that will help you learn the ropes from the ground up, while enabling you to collect and analyze data in one of the world’s greatest container shipping datasets. \u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eBring your analytical mindset and a genuine curiosity for how global shipping works and join our team!\u0026nbsp;\u003c/div\u003e","employment_type":"permanent_full_time","employment_type_text":"Permanent - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eJoin our Liner Data \u0026amp; Analytics Team (LDAT) and own a portfolio of container shipping services for the world's largest ocean carriers. Liner shipping data is messy, inconsistent, and constantly changing. You'll turn that chaos into the accurate database our customers rely on to make shipping decisions, working at the center of a fast-moving industry with direct ownership over the data that shapes how the world sees ocean freight.\u0026nbsp;\u003cbr\u003e\u003cbr\u003eYour key tasks will be:\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eReview and validate liner service data on a recurring cycle.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eMonitor vessel movements, deployment changes, blank sailings, and schedule updates for your portfolio, and update our platform to reflect the current state. \u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eInvestigate and resolve data quality exceptions: missing vessel assignments, stale voyage legs, unmatched data points and carrier discrepancies.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eQuery our database to dig into data issues and support your own analysis. \u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWork closely with Customer Success and Sales teams in answering customers´ data requests throughout their procurement life cycle journey.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eMaintain forecast accuracy for your portfolio. \u0026nbsp;\u003c/li\u003e\u003c/ul\u003e","key_responsibilities_header":"Key Responsibilities","reporting_to":"John Kingori","skills_knowledge_expertise":"\u003cdiv\u003e\u003c!--block--\u003eIn this role, we are looking for someone with the following background and skills:\u0026nbsp;\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eBachelor's degree in Business Administration, Economics, Mathematics, Engineering, Logistics, and related fields.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFluent in English, oral and written.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eHigh attention to detail; you catch discrepancies across large datasets.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eAnalytical and problem-solving mindset.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eComfortable working async across time zones in a distributed team.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eOutstanding team player, also fully capable of working independently.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou're curious by nature and learn new tools and systems quickly, always asking why the data looks the way it does, not just what it shows.\u0026nbsp;\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eBonus points for\u003c/strong\u003e\u0026nbsp;\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eExperience as a Data Analyst\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFamiliarity with container shipping, vessel schedules, or liner service structures.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eProficiency with AI tools to speed up data analysis, scripting, or documentation.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWorking knowledge of SQL; comfortable writing and troubleshooting queries against a relational database.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eComfortable building and maintaining dashboards in Power BI.\u0026nbsp;\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cem\u003ePlease note that English is our working language, so we ask that your application and CV are submitted in English.\u003c/em\u003e\u003c/div\u003e","skills_knowledge_expertise_header":"Skills, Knowledge \u0026 Expertise","title":"Data Operations Analyst - LSP","url":"https://xeneta.pinpointhq.com/en/postings/a85d5d45-b334-4f5c-aaea-d0b399b08a2e","path":"/en/postings/a85d5d45-b334-4f5c-aaea-d0b399b08a2e","workplace_type":"hybrid","workplace_type_text":"Hybrid","job":{"id":"582848","requisition_id":"REQ-0328","department":{"id":"51942","name":"RMT - LSP"},"division":null,"structure_custom_group_one":null},"location":{"id":"71041","city":"Bucharest","name":"Bucharest","postal_code":"","province":"Bucharest","street_address":""}},{"id":"587635","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eNote that the positions is employed through \u003cstrong\u003eGI Group,\u003c/strong\u003e our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. Looking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Benefits","compensation":null,"compensation_minimum":null,"compensation_maximum":null,"compensation_currency":null,"compensation_frequency":null,"compensation_visible":false,"deadline_at":"2026-10-09T23:59:59+02:00","description":"\u003cdiv\u003e\u003c!--block--\u003eWe are looking for a \u003cstrong\u003ePeople \u0026amp; Culture Assistant \u003c/strong\u003eto join our People Experience Team. This is a temporary, 12-month contract for a full-time role, as one of our team members is on leave.\u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u0026nbsp;\u003cbr\u003eCustomer centricity is a core principle in Xeneta. For the People \u0026amp; Culture team, our colleagues are our customers. As People and Culture Assistant, you will ensure our Executive Leadership's days run as smoothly and efficiently as possible. You will be their business partner who enables them to focus on their core responsibilities and helps them meet their business goals and objectives. You will work with our C-suite and thus get exposure to all the different areas of our business.\u0026nbsp; \u003cbr\u003e\u0026nbsp;\u003cbr\u003eWe're seeking a diligent and self-motivated person to join our People \u0026amp; Culture team. This is an exciting opportunity to work with various tasks and responsibilities in an international tech company!\u0026nbsp; \u003cbr\u003e\u003cbr\u003e\u003cem\u003eWe would prefer someone who can start quickly and we will call candidates in for interviews on a continuous basis, so please apply as soon as possible.\u0026nbsp;\u003c/em\u003e\u003c/div\u003e","employment_type":"fixed_term_full_time","employment_type_text":"Fixed Term - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eYour main responsibility will be to support the CEO, CPO, and CRO. As such, your key tasks will be: \u0026nbsp;\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eOwn calendar management: organizing inboxes, scheduling meetings, resolving scheduling conflicts, prioritizing calendar events, communicating urgent requests and changes to relevant stakeholders.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eManage Xeneta Board meeting logistics, including liaising with Board members, setting Board calendar, ensuring timely delivery of Board materials, and scheduling Board dinners.\u0026nbsp; \u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eArrange and coordinate travel plans, itineraries, and agendas.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eSupport with various administrative and ad hoc tasks, such as processing expense reimbursements.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eSchedule and coordinate various events for C-Suite and Extended Leadership Team members, such as offsite meetings, departmental events, team-building events, etc.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eSupport the People Experience Team on other global events and initiatives, such as supporting the Senior People Experience Partner on ticketing systems and ad hoc office management.\u003c/li\u003e\u003c/ul\u003e","key_responsibilities_header":"Key Responsibilities","reporting_to":"","skills_knowledge_expertise":"\u003cdiv\u003e\u003c!--block--\u003eWe are looking for someone who has the following experience and skills: \u0026nbsp;\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eIdeally 1-3 years of experience in a similar role, such as Scheduling Assistant, Personal Assistant, Coordinator, or Executive Assistant.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eBusiness fluent in English, oral and written.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eDemonstrated proficiency in Microsoft Suite (Outlook, Teams, Word, PowerPoint, Excel).\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eExcellent communication, interpersonal, and organizational skills.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eSelf-motivated; able to work independently and efficiently.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou thrive in an international, fast-working environment and enjoy making people's lives easier. You remain calm and composed, even while handling tasks that demand detail-focus and accuracy.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eAbility to quickly familiarize yourself with new topics. You like digging into new things and can deal with multiple issues simultaneously.\u0026nbsp;\u003c/li\u003e\u003c/ul\u003e","skills_knowledge_expertise_header":"Skills, Knowledge and Expertise","title":"People \u0026 Culture Assistant","url":"https://xeneta.pinpointhq.com/en/postings/ae6a3963-acf7-4ed0-8147-7138f7ebb21e","path":"/en/postings/ae6a3963-acf7-4ed0-8147-7138f7ebb21e","workplace_type":"hybrid","workplace_type_text":"Hybrid","job":{"id":"588176","requisition_id":"REQ-0331","department":{"id":"9357","name":"People \u0026 Culture"},"division":null,"structure_custom_group_one":null},"location":{"id":"71041","city":"Bucharest","name":"Bucharest","postal_code":"","province":"Bucharest","street_address":""}}]}
\ No newline at end of file
+{"data":[{"id":"581878","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe will help you \u003cstrong\u003egrow your career:\u003c/strong\u003e We help you identify areas of growth for you, whether you want to grow as a specialist in your field or eventually want to go into a leadership/management role.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person from time to time.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will receive an\u003cstrong\u003e individual education budget \u003c/strong\u003eand can educate yourself during work hours: you can spend several hours each month attending courses, reading books, or getting involved in communities.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFull-time position, employed through GI Group, our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eLooking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Job Benefits","compensation":null,"compensation_minimum":60000,"compensation_maximum":80000,"compensation_currency":"EUR","compensation_frequency":"annually","compensation_visible":true,"deadline_at":null,"description":"\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eData Detective Wanted to join our Data \u0026amp; Analytics Team\u003c/strong\u003e \u003cbr\u003e\u003cbr\u003e\u003cem\u003eLocation: Bucharest, Romania\u003c/em\u003e\u003cbr\u003e\u003cbr\u003eAs a \u003cstrong\u003eData Operations Analyst \u003c/strong\u003eat Xeneta, you’ll learn how to master complex data analytics within the world of supply chain and logistics. Xeneta is one of the world's leading data platforms providing market intelligence to the ocean and air freight industry. We have a big team of skilled Data Analysts that will help you learn the ropes from the ground up, while enabling you to collect and analyze data in one of the world’s greatest container shipping datasets. \u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eBring your analytical mindset and a genuine curiosity for how global shipping works and join our team!\u0026nbsp;\u003c/div\u003e","employment_type":"permanent_full_time","employment_type_text":"Permanent - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eJoin our Liner Data \u0026amp; Analytics Team (LDAT) and own a portfolio of container shipping services for the world's largest ocean carriers. Liner shipping data is messy, inconsistent, and constantly changing. You'll turn that chaos into the accurate database our customers rely on to make shipping decisions, working at the center of a fast-moving industry with direct ownership over the data that shapes how the world sees ocean freight.\u0026nbsp;\u003cbr\u003e\u003cbr\u003eYour key tasks will be:\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eReview and validate liner service data on a recurring cycle.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eMonitor vessel movements, deployment changes, blank sailings, and schedule updates for your portfolio, and update our platform to reflect the current state. \u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eInvestigate and resolve data quality exceptions: missing vessel assignments, stale voyage legs, unmatched data points and carrier discrepancies.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eQuery our database to dig into data issues and support your own analysis. \u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWork closely with Customer Success and Sales teams in answering customers´ data requests throughout their procurement life cycle journey.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eMaintain forecast accuracy for your portfolio. \u0026nbsp;\u003c/li\u003e\u003c/ul\u003e","key_responsibilities_header":"Key Responsibilities","reporting_to":"John Kingori","skills_knowledge_expertise":"\u003cdiv\u003e\u003c!--block--\u003eIn this role, we are looking for someone with the following background and skills:\u0026nbsp;\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eBachelor's degree in Business Administration, Economics, Mathematics, Engineering, Logistics, and related fields.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFluent in English, oral and written.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eHigh attention to detail; you catch discrepancies across large datasets.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eAnalytical and problem-solving mindset.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eComfortable working async across time zones in a distributed team.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eOutstanding team player, also fully capable of working independently.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou're curious by nature and learn new tools and systems quickly, always asking why the data looks the way it does, not just what it shows.\u0026nbsp;\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eBonus points for\u003c/strong\u003e\u0026nbsp;\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eExperience as a Data Analyst\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFamiliarity with container shipping, vessel schedules, or liner service structures.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eProficiency with AI tools to speed up data analysis, scripting, or documentation.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWorking knowledge of SQL; comfortable writing and troubleshooting queries against a relational database.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eComfortable building and maintaining dashboards in Power BI.\u0026nbsp;\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cem\u003ePlease note that English is our working language, so we ask that your application and CV are submitted in English.\u003c/em\u003e\u003c/div\u003e","skills_knowledge_expertise_header":"Skills, Knowledge \u0026 Expertise","title":"Data Operations Analyst - LSP","url":"https://xeneta.pinpointhq.com/en/postings/a85d5d45-b334-4f5c-aaea-d0b399b08a2e","path":"/en/postings/a85d5d45-b334-4f5c-aaea-d0b399b08a2e","workplace_type":"hybrid","workplace_type_text":"Hybrid","job":{"id":"582848","requisition_id":"REQ-0328","department":{"id":"51942","name":"RMT - LSP"},"division":null,"structure_custom_group_one":null},"location":{"id":"71041","city":"Bucharest","name":"Bucharest","postal_code":"","province":"Bucharest","street_address":""}},{"id":"587635","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eNote that the positions is employed through \u003cstrong\u003eGI Group,\u003c/strong\u003e our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. Looking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Benefits","compensation":null,"compensation_minimum":null,"compensation_maximum":null,"compensation_currency":null,"compensation_frequency":null,"compensation_visible":false,"deadline_at":"2026-10-09T23:59:59+02:00","description":"\u003cdiv\u003e\u003c!--block--\u003eWe are looking for a \u003cstrong\u003ePeople \u0026amp; Culture Assistant \u003c/strong\u003eto join our People Experience Team. This is a temporary, 12-month contract for a full-time role, as one of our team members is on leave.\u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u0026nbsp;\u003cbr\u003eCustomer centricity is a core principle in Xeneta. For the People \u0026amp; Culture team, our colleagues are our customers. As People and Culture Assistant, you will ensure our Executive Leadership's days run as smoothly and efficiently as possible. You will be their business partner who enables them to focus on their core responsibilities and helps them meet their business goals and objectives. You will work with our C-suite and thus get exposure to all the different areas of our business.\u0026nbsp; \u003cbr\u003e\u0026nbsp;\u003cbr\u003eWe're seeking a diligent and self-motivated person to join our People \u0026amp; Culture team. This is an exciting opportunity to work with various tasks and responsibilities in an international tech company!\u0026nbsp; \u003cbr\u003e\u003cbr\u003e\u003cem\u003eWe would prefer someone who can start quickly and we will call candidates in for interviews on a continuous basis, so please apply as soon as possible.\u0026nbsp;\u003c/em\u003e\u003c/div\u003e","employment_type":"fixed_term_full_time","employment_type_text":"Fixed Term - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eYour main responsibility will be to support the CEO, CPO, and CRO. As such, your key tasks will be: \u0026nbsp;\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eOwn calendar management: organizing inboxes, scheduling meetings, resolving scheduling conflicts, prioritizing calendar events, communicating urgent requests and changes to relevant stakeholders.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eManage Xeneta Board meeting logistics, including liaising with Board members, setting Board calendar, ensuring timely delivery of Board materials, and scheduling Board dinners.\u0026nbsp; \u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eArrange and coordinate travel plans, itineraries, and agendas.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eSupport with various administrative and ad hoc tasks, such as processing expense reimbursements.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eSchedule and coordinate various events for C-Suite and Extended Leadership Team members, such as offsite meetings, departmental events, team-building events, etc.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eSupport the People Experience Team on other global events and initiatives, such as supporting the Senior People Experience Partner on ticketing systems and ad hoc office management.\u003c/li\u003e\u003c/ul\u003e","key_responsibilities_header":"Key Responsibilities","reporting_to":"","skills_knowledge_expertise":"\u003cdiv\u003e\u003c!--block--\u003eWe are looking for someone who has the following experience and skills: \u0026nbsp;\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eIdeally 1-3 years of experience in a similar role, such as Scheduling Assistant, Personal Assistant, Coordinator, or Executive Assistant.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eBusiness fluent in English, oral and written.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eDemonstrated proficiency in Microsoft Suite (Outlook, Teams, Word, PowerPoint, Excel).\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eExcellent communication, interpersonal, and organizational skills.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eSelf-motivated; able to work independently and efficiently.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou thrive in an international, fast-working environment and enjoy making people's lives easier. You remain calm and composed, even while handling tasks that demand detail-focus and accuracy.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eAbility to quickly familiarize yourself with new topics. You like digging into new things and can deal with multiple issues simultaneously.\u0026nbsp;\u003c/li\u003e\u003c/ul\u003e","skills_knowledge_expertise_header":"Skills, Knowledge and Expertise","title":"People \u0026 Culture Assistant","url":"https://xeneta.pinpointhq.com/en/postings/ae6a3963-acf7-4ed0-8147-7138f7ebb21e","path":"/en/postings/ae6a3963-acf7-4ed0-8147-7138f7ebb21e","workplace_type":"hybrid","workplace_type_text":"Hybrid","job":{"id":"588176","requisition_id":"REQ-0331","department":{"id":"9357","name":"People \u0026 Culture"},"division":null,"structure_custom_group_one":null},"location":{"id":"71041","city":"Bucharest","name":"Bucharest","postal_code":"","province":"Bucharest","street_address":""}}]}
\ No newline at end of file
diff --git a/tests/fixtures/ats/rippling-role-2.json b/tests/fixtures/ats/rippling-role-2.json
deleted file mode 100644
index 2bf5f4c..0000000
--- a/tests/fixtures/ats/rippling-role-2.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
- "uuid": "0fbcec57-3012-4271-96bf-12b145cf6dea",
- "name": "General Applications: Engineering",
- "description": {
- "company": "
Not finding the perfect engineering role on our careers page?
We are constantly seeking talented individuals to join our team. If you are a Software Engineer and haven\u2019t found the perfect role on our careers page, don\u2019t worry \u2014 you\u2019ve come to the right place.
We strongly believe that each person has something unique to bring to the table. Whether you\u2019re a recent graduate or a seasoned professional, we\u2019re genuinely excited to hear from you. At Prisma, we work with modern technologies, including TypeScript",
- "role": "
What is it like to work at Prisma?
You're likely to be a great fit for the team if you ...
... have an ownership mindset. As part of a small team, you'll have an outsize impact on decisions and the way things are executed.
Rippling gives businesses one place to run HR, IT, and Finance. It brings together all of the workforce systems that are normally scattered across a company, like payroll, expenses, benefits, and computers. For the first time ever, you can manage and automate every part of the employee lifecycle in a single system.
Take onboarding, for example. With Rippling, you can hire a new employee anywhere in the world and set up their payroll, corporate card, computer, benefits, and even third-party apps like Slack and Microsoft 365\u2014all within 90 seconds.",
- "role": "
About The Role
Rippling Infrastructure operates as a scaled engineering organization responsible for the mission-critical systems that power every product and platform across the company. We support nearly 1,000 engineers globally and deliver the foundational infrastructure that enables Rippling's products to scale reliably and efficiently.
"
- },
- "workLocations": [
- "San Francisco, CA",
- "Seattle, WA",
- "New York, NY"
- ],
- "department": {
- "name": "Platform",
- "base_department": "Engineering",
- "department_tree": [
- "Engineering",
- "Platform"
- ]
- },
- "employmentType": {
- "label": "SALARIED_FT",
- "id": "Salaried, full-time"
- },
- "createdOn": "2025-04-07T12:29:19.348000-07:00",
- "eeocQuestionnaireEnabled": true,
- "eeocQuestionnaireEnabledForJobPost": true,
- "url": "https://ats.rippling.com/rippling/jobs/2f2ae4c8-ec56-4d1f-8f2a-0d295674e19c",
- "board": {
- "boardType": "CUSTOM",
- "slug": "rippling",
- "banner": null,
- "title": "Current Openings",
- "subtitle": null,
- "boardURL": "http://www.rippling.com/careers/open-roles",
- "fontType": null,
- "linkColor": null,
- "buttonColor": null,
- "buttonTextColor": null,
- "noOpeningsMessage": null,
- "groupJobsByLocation": false,
- "showBoardLogoOnJobPost": false,
- "showCompanyInfoUnderJobPost": false,
- "companyName": "Rippling"
- },
- "payRangeDetails": [
- {
- "location": "US Tier 1",
- "currency": "USD",
- "frequency": "YEAR",
- "rangeStart": 216000.0,
- "rangeEnd": 378000.0,
- "isRemote": false
- }
- ],
- "applicationConfirmationTemplate": "6348729c5a9a6e6df0bb6f64",
- "hasAIEvaluationsEnabled": true,
- "companyName": "Rippling",
- "unlistedFromSearch": false,
- "jsonLd": null
-}
diff --git a/tests/fixtures/ats/smartrecruiters-places.json b/tests/fixtures/ats/smartrecruiters-places.json
index 46efc4f..fbf3b88 100644
--- a/tests/fixtures/ats/smartrecruiters-places.json
+++ b/tests/fixtures/ats/smartrecruiters-places.json
@@ -16,7 +16,9 @@
"fullLocation": "Berlin, Germany"
},
"department": {"id": "2294038", "label": "Engineering, Infrastructure and Operations"},
- "function": {"id": "engineering", "label": "Engineering"}
+ "function": {"id": "engineering", "label": "Engineering"},
+ "typeOfEmployment": {"id": "permanent", "label": "Full-time"},
+ "experienceLevel": {"id": "director", "label": "Director"}
},
{
"id": "744000147485112",
diff --git a/tests/test_readers_contract.py b/tests/test_readers_contract.py
index 4dd2ca9..859b060 100644
--- a/tests/test_readers_contract.py
+++ b/tests/test_readers_contract.py
@@ -1,29 +1,32 @@
"""The reader contract, held for every module of `plugins/readers/`.
Per module: it declares `READS`, the declaration matches its functions, and
-its rows keep what the contract says they keep. No network: fixtures stand
-in for the host.
-
-`UNDECLARED` lists the modules that do not declare `READS` yet; their tests
-are marked xfail (not strict), and the planner removes a name once the
-module declares. A module not in the list must pass.
+its rows state what the declaration says and nothing it denies. No network:
+fixtures stand in for the host. A fixture a declaration needs and that is
+missing fails the test with the path to add.
Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/`
for searches, named after the module:
-- `-places.`: one listing answer with at least one role at two
- or more places. For `listing="whole"` it is the body `parse` gets; for
- `listing="paged"` it is what every request of the first `page` call
- returns, and `rows` reads that page. Required by the test of a module
- that declares `places="many"`, which is skipped until the fixture exists.
-- `-places-detail.`: optional, for a host whose places come from
- the role's own page: what `detail`'s requests return; its `places` replace
- the row's.
-- `-teaser.`: one listing answer with a teaser, read the same
- way. Required by the test of a module with `text="excerpt"` or
- `teaser=True`, skipped until it exists.
+- `-places.`: one listing answer, required of every reader. For
+ `listing="whole"` it is the body `parse` gets; for `listing="paged"` it is
+ what every request of the first `page` call returns, and `rows` reads
+ that page. Under `places="many"` at least one role is at two or more
+ places.
+- `.` and `-teaser.`: more listing answers, read the
+ same way; the teaser one is required under `text="excerpt"` or
+ `teaser=True`.
+- `-detail*.` and `-places-detail.`: what `detail`'s
+ requests return, one of them required under `text="detail"`. Only
+ `_runner.DETAIL_FIELDS` of the answer count, as at run time; the places of
+ `-places-detail` replace those of the `-places` rows.
- `.slug`: optional, the slug the fixtures are read with, one line;
- `acme` without it.
+ `acme` without it. A search's saved params are `SEARCH_PARAMS` below.
+
+Between them the listing and detail answers show every field the reader
+declares `stated` (or `figures`, `line`, `trusted`, `listed`): a real
+fixture that happens to lack one is filled from what the per-host test
+already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels).
"""
from __future__ import annotations
@@ -41,42 +44,53 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
+from app.prep.hosts import TABLES # noqa: E402
from plugins import readers # noqa: E402
from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402
from plugins.readers import _contract # noqa: E402
+from plugins.readers._runner import DETAIL_FIELDS # noqa: E402
-FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" / "search")
+BOARD_FIXTURES = ROOT / "tests" / "fixtures" / "ats"
+SEARCH_FIXTURES = ROOT / "tests" / "fixtures" / "search"
-#: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and
-#: C (the search providers, which arrive declared) land.
-UNDECLARED: frozenset[str] = frozenset()
+#: A search's saved params, which its `page` reads; `{}` for the rest.
+SEARCH_PARAMS: dict[str, dict[str, Any]] = {"adzuna": {"q": "backend", "country": "de"}}
+
+#: The readers whose listing states no date and whose `posted` comes from
+#: `detail` alone: the date exists only for roles the enrich stage read.
+POSTED_BY_DETAIL = frozenset({"bamboohr", "gem", "rippling"})
+
+#: The fields a host states in its own words, which `app.prep` maps through
+#: the host's table in `app/prep/hosts/.py`.
+WORD_FIELDS = ("seniority", "employment")
-def _params() -> list[Any]:
- return [pytest.param(name, marks=pytest.mark.xfail(reason="no READS yet", strict=False))
- if name in UNDECLARED else name for name in readers.READERS]
+def _folder(name: str) -> Path:
+ return SEARCH_FIXTURES if name in readers.PROVIDERS else BOARD_FIXTURES
-def _declared(name: str) -> tuple[types.ModuleType, Reads]:
- module = readers.READERS[name]
- reads = getattr(module, "READS", None)
- assert isinstance(reads, Reads), f"{name} declares no READS"
- return module, reads
+def _fixtures(name: str, pattern: str) -> list[Path]:
+ return sorted(p for p in _folder(name).glob(pattern.format(name=name))
+ if p.suffix not in (".slug",))
def _fixture(name: str, kind: str) -> Path | None:
- for folder in FIXTURE_DIRS:
- found = sorted(folder.glob(f"{name}-{kind}.*"))
- if found:
- return found[0]
- return None
+ found = _fixtures(name, f"{{name}}-{kind}.*")
+ return found[0] if found else None
+
+
+def _need(name: str, kind: str, why: str) -> Path:
+ """The fixture `-.*`, or a failure naming the file to add."""
+ found = _fixture(name, kind)
+ if found is None:
+ folder = _folder(name).relative_to(ROOT)
+ pytest.fail(f"add {folder}/{name}-{kind}.: {why}")
+ return found
def _slug(name: str) -> str:
- for folder in FIXTURE_DIRS:
- if (path := folder / f"{name}.slug").exists():
- return path.read_text(encoding="utf-8").strip()
- return "acme"
+ path = _folder(name) / f"{name}.slug"
+ return path.read_text(encoding="utf-8").strip() if path.exists() else "acme"
def _answer(body: bytes) -> Callable[..., bytes]:
@@ -84,7 +98,7 @@ def _answer(body: bytes) -> Callable[..., bytes]:
def read_fixture(module: types.ModuleType, body: bytes, slug: str,
- detail_body: bytes | None = None) -> list[dict[str, Any]]:
+ detail_body: bytes | None = None, ask: Ask | None = None) -> list[dict[str, Any]]:
"""The rows `module` makes of one listing answer, as the runner reads
them: `parse` for a whole listing, the first `page` and its `rows` for a
paged one, and, given `detail_body`, each row's `places` from `detail`."""
@@ -92,7 +106,7 @@ def read_fixture(module: types.ModuleType, body: bytes, slug: str,
if reads.listing == "whole":
rows = list(module.parse(body, slug))
else:
- decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, Ask())
+ decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, ask or Ask())
rows = list(module.rows(decoded, slug))
if detail_body is not None:
for row in rows:
@@ -102,6 +116,38 @@ def read_fixture(module: types.ModuleType, body: bytes, slug: str,
return rows
+def _ask(name: str) -> Ask:
+ return Ask(params=SEARCH_PARAMS.get(name, {}))
+
+
+def listing_rows(name: str) -> list[dict[str, Any]]:
+ """Every row of the reader's listing fixtures; the `-places` rows with
+ the places of `-places-detail` where there is one."""
+ module, slug = readers.READERS[name], _slug(name)
+ places = _need(name, "places", "one listing answer of the host")
+ detail = _fixture(name, "places-detail")
+ rows = read_fixture(module, places.read_bytes(), slug,
+ detail.read_bytes() if detail else None, _ask(name))
+ for path in _fixtures(name, "{name}.*") + _fixtures(name, "{name}-teaser.*"):
+ rows += read_fixture(module, path.read_bytes(), slug, ask=_ask(name))
+ return rows
+
+
+def detail_rows(name: str, listed: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """What `detail` hands over for the first listed role, once per detail
+ fixture, cut to `DETAIL_FIELDS` as the runner cuts it."""
+ module = readers.READERS[name]
+ if module.READS.text != "detail":
+ return []
+ paths = _fixtures(name, "{name}-detail*.*") + _fixtures(name, "{name}-places-detail.*")
+ if not paths:
+ _need(name, "detail", "what the role's own page answers, for text=\"detail\"")
+ assert listed, f"{name}: the listing fixtures gave no role to read in detail"
+ return [{key: value for key, value in
+ module.detail(_answer(path.read_bytes()), _slug(name), dict(listed[0])).items()
+ if key in DETAIL_FIELDS} for path in paths]
+
+
def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
"""What rows get wrong about `places` under the declared trait."""
out = []
@@ -118,60 +164,138 @@ def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
out.append(f"{rid}: {place!r} is not a Place")
if places in ("one", "copies") and len(found) > 1:
out.append(f'{rid}: {len(found)} places under places="{places}"')
+ if places == "none" and found:
+ out.append(f'{rid}: {len(found)} places under places="none"')
if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows):
out.append('no row has two places: a reader with places="many" keeps every place')
+ elif places != "none" and not any(row.get("places") for row in rows):
+ out.append(f'no row has a place under places="{places}": declare "none"')
+ return out
+
+
+def _states(rows: list[dict[str, Any]], key: str) -> bool:
+ """Whether some row carries `key` with a value (False counts, "" does not)."""
+ return any(row.get(key) not in (None, "", [], {}) for row in rows)
+
+
+def field_problems(reads: Reads, listing: list[dict[str, Any]], detail: list[dict[str, Any]],
+ posted_by_detail: bool = False) -> list[str]:
+ """What the rows get wrong about the declared `fields`, `text` and
+ `teaser`: a trait that promises a key no row carries, or denies one a
+ row carries."""
+ rows = listing + detail
+ fields = reads.fields
+ out: list[str] = []
+ figures = _states(rows, "salary_min") or _states(rows, "salary_max")
+ line = _states(rows, "salary_text")
+ if fields.pay == "figures" and not figures:
+ out.append('pay="figures" and no row has salary_min or salary_max')
+ if fields.pay == "line" and (figures or not line):
+ out.append('pay="line" wants salary_text on some row and figures on none')
+ if fields.pay == "none" and (figures or line):
+ out.append('pay="none" and a row has pay')
+
+ if fields.posted == "none" and _states(rows, "posted"):
+ out.append('posted="none" and a row has posted')
+ elif fields.posted != "none" and posted_by_detail:
+ if _states(listing, "posted") or not _states(detail, "posted"):
+ out.append(f'posted="{fields.posted}" by detail wants posted from detail only')
+ elif fields.posted != "none" and not _states(listing, "posted"):
+ out.append(f'posted="{fields.posted}" and no listed row has posted')
+
+ for key in ("remote", "country", "seniority", "employment"):
+ stated = getattr(fields, key) == "stated"
+ if stated != _states(rows, key):
+ out.append(f'{key}="{getattr(fields, key)}" and '
+ + ("no row has it" if stated else "a row has it"))
+
+ text = {"listing": _states(listing, "description"),
+ "detail": _states(detail, "description"),
+ "excerpt": _states(listing, "teaser") and not _states(listing, "description"),
+ "none": not _states(rows, "description")}[reads.text]
+ if not text:
+ out.append(f'text="{reads.text}" and the rows do not show it')
+ teaser = reads.teaser or reads.text == "excerpt"
+ if teaser != _states(listing if teaser else rows, "teaser"):
+ out.append("teaser declared and no listed row has one" if teaser
+ else "a row has a teaser the declaration does not name")
return out
# ------------------------------------------------------------ every module
-def test_the_undeclared_list_names_only_readers() -> None:
- assert set(readers.READERS) >= UNDECLARED
-
-
-def test_undeclared_lists_the_modules_without_reads() -> None:
- assert set(readers.undeclared()) == {name for name, module in readers.READERS.items()
- if not hasattr(module, "READS")}
-
-
-@pytest.mark.parametrize("name", _params())
-def test_a_reader_declares_reads(name: str) -> None:
- _declared(name)
-
-
-@pytest.mark.parametrize("name", _params())
+@pytest.mark.parametrize("name", readers.READERS)
def test_a_declaration_matches_the_functions(name: str) -> None:
- module, _reads = _declared(name)
+ module = readers.READERS[name]
assert readers.problems(module, name) == []
- expected = "search" if module.READS.scope == "search" else "board"
- assert name in (readers.PROVIDERS if expected == "search" else readers.HOSTS)
+ assert name in (readers.PROVIDERS if module.READS.scope == "search" else readers.HOSTS)
-@pytest.mark.parametrize("name", _params())
+@pytest.mark.parametrize("name", readers.READERS)
def test_a_reader_keeps_every_place(name: str) -> None:
- module, reads = _declared(name)
- fixture = _fixture(name, "places")
- if fixture is None:
- if reads.places == "many":
- pytest.skip(f"no fixture {name}-places.* yet")
- pytest.skip(f'places="{reads.places}" and no fixture')
- detail = _fixture(name, "places-detail")
- rows = read_fixture(module, fixture.read_bytes(), _slug(name),
- detail.read_bytes() if detail else None)
- assert rows, f"{fixture.name} gave no rows"
- assert place_problems(rows, reads.places) == []
+ listed = listing_rows(name)
+ assert listed, f"{name}: the listing fixtures gave no rows"
+ answered = [{"source_id": f"detail {i}", "places": row["places"]}
+ for i, row in enumerate(detail_rows(name, listed)) if "places" in row]
+ assert place_problems(listed + answered, readers.READERS[name].READS.places) == []
-@pytest.mark.parametrize("name", _params())
-def test_a_reader_keeps_the_teaser(name: str) -> None:
- module, reads = _declared(name)
- if not (reads.text == "excerpt" or reads.teaser):
- pytest.skip("the list carries no teaser")
- fixture = _fixture(name, "teaser")
- if fixture is None:
- pytest.skip(f"no fixture {name}-teaser.* yet")
- rows = read_fixture(module, fixture.read_bytes(), _slug(name))
- assert any(isinstance(row.get("teaser"), str) and row["teaser"].strip() for row in rows)
+@pytest.mark.parametrize("name", readers.READERS)
+def test_a_reader_states_what_it_declares(name: str) -> None:
+ reads = readers.READERS[name].READS
+ if reads.text == "excerpt" or reads.teaser:
+ _need(name, "teaser", "a listing answer with a teaser")
+ listed = listing_rows(name)
+ assert field_problems(reads, listed, detail_rows(name, listed),
+ name in POSTED_BY_DETAIL) == []
+
+
+@pytest.mark.parametrize("name", [name for name, module in readers.READERS.items()
+ if module.READS.listing == "paged"
+ and module.READS.complete in ("stated", "end")])
+def test_a_paged_reader_states_a_total_as_declared(name: str) -> None:
+ """`complete="stated"`: the first page's total is an int; `"end"`: None,
+ since the read ends on an empty page."""
+ module = readers.READERS[name]
+ body = _need(name, "places", "one listing answer of the host").read_bytes()
+ _decoded, _results, total, _cursor = module.page(_answer(body), _slug(name), None,
+ _ask(name))
+ if module.READS.complete == "stated":
+ assert isinstance(total, int) and not isinstance(total, bool), total
+ else:
+ assert total is None, total
+
+
+def test_the_posted_by_detail_list_names_only_readers_that_date_by_detail() -> None:
+ for name in POSTED_BY_DETAIL:
+ reads = readers.READERS[name].READS
+ assert reads.text == "detail" and reads.fields.posted != "none", name
+
+
+# ------------------------------------------------------------ prep tables
+
+def test_every_prep_table_is_a_readers() -> None:
+ assert set(TABLES) <= set(readers.READERS), set(TABLES) - set(readers.READERS)
+
+
+@pytest.mark.parametrize("name", readers.READERS)
+def test_a_reader_stating_words_has_a_table_for_them(name: str) -> None:
+ """A host that states seniority or employment in its own words has a
+ table for that field in `app/prep/hosts/.py`, even an empty one
+ for a host whose words are already the app's (`words.lookup` then
+ takes the word as it is). The table is where a word is known, so a
+ host without one lists every word as unmapped. A table for a field the
+ reader declares `none` is dead."""
+ fields = readers.READERS[name].READS.fields
+ tables = TABLES.get(name, {})
+ for field in WORD_FIELDS:
+ has = field in tables or (field == "employment" and "match" in tables)
+ stated = getattr(fields, field) == "stated"
+ assert has == stated, (f"{name}: {field}={getattr(fields, field)!r} "
+ f"{'without' if stated else 'with'} a table in "
+ f"app/prep/hosts/{name}.py")
+ if "remote" in tables:
+ assert fields.remote == "stated", f"{name}: a remote table and remote=\"none\""
# ------------------------------------------------------------ the checks
@@ -219,11 +343,10 @@ def test_a_good_declaration_has_no_problems() -> None:
assert readers.problems(module, "fake") == []
-def test_a_module_without_reads_is_checked_as_before() -> None:
+def test_a_module_without_reads_fails() -> None:
module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: [])
- assert readers.problems(module, "fake") == []
- assert readers.scope(module) == "board"
- assert readers.undeclared({"fake": module}) == ["fake"]
+ assert readers.problems(module, "fake") == [
+ "no READS: every reader declares what it reads"]
@pytest.mark.parametrize(("module", "reads", "problem"), [
@@ -256,6 +379,7 @@ def test_a_module_without_reads_is_checked_as_before() -> None:
env=("ADZUNA_APP_ID",))},
"key.env does not match key.fields"),
(_whole(), {"robots": " "}, "robots is empty"),
+ (_whole(), {"robots": "read: robots.txt allows /api/"}, "robots has no date"),
(_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {},
"feed_url does not take (slug, ask)"),
(_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []),
@@ -274,22 +398,48 @@ def test_reads_that_is_not_a_reads_is_refused() -> None:
assert "READS is not a Reads" in readers.problems(module, "fake")
-def test_a_search_needs_fewer_names_than_a_board() -> None:
+def _search() -> Any:
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL = "fake", "Fake"
module.allowed = lambda slug: lambda host: True
module.page = lambda get, slug, cursor, ask: (None, [], 0, None)
module.rows = lambda decoded, slug: []
+ module.PARAMS = {"q": "query", "where": "place"}
+ module.clean_params = lambda params: dict(params)
module.READS = _reads(scope="search", listing="paged", complete="never",
- filters=Filters(query="one"),
+ filters=Filters(query="one", place="one"),
key=Key(name="adzuna", fields=("app_id", "app_key"),
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")))
+ return module
+
+
+def test_a_search_needs_fewer_names_than_a_board() -> None:
+ module = _search()
assert readers.problems(module, "fake") == []
- assert readers.scope(module) == "search"
del module.READS
assert "no careers_url" in readers.problems(module, "fake")
+def test_a_search_names_its_params_and_cleans_them() -> None:
+ module = _search()
+ del module.PARAMS, module.clean_params
+ assert {"no PARAMS", "no clean_params"} <= set(readers.problems(module, "fake"))
+ module = _search()
+ module.PARAMS = {"q": "query", "area": "town"}
+ assert readers.problems(module, "fake") == [
+ "PARAMS maps 'area' to 'town', not a filter dimension"]
+ module = _search()
+ module.clean_params = lambda: {}
+ assert readers.problems(module, "fake") == ["clean_params does not take (params)"]
+
+
+def test_every_search_names_its_params() -> None:
+ for name, module in readers.PROVIDERS.items():
+ declared = module.READS.filters.declared()
+ assert "query" in module.PARAMS.values(), name
+ assert set(module.PARAMS.values()) <= set(declared), name
+
+
def test_a_misspelt_trait_fails_where_it_is_declared() -> None:
with pytest.raises(pydantic.ValidationError):
_reads(listing="pages")
@@ -349,12 +499,20 @@ def test_the_place_check_passes_a_role_at_two_places() -> None:
assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == []
+def test_the_place_check_holds_a_reader_that_states_none_to_none() -> None:
+ assert place_problems([{"source_id": "1", "places": []}], "none") == []
+ assert place_problems([{"source_id": "1", "places": []}], "one") == [
+ 'no row has a place under places="one": declare "none"']
+ assert place_problems([{"source_id": "2", "places": [{"text": "Oslo", "remote": None}]}],
+ "none") == ['2: 1 places under places="none"']
+
+
def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None:
joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}]
assert place_problems(joined, "many") == [
'no row has two places: a reader with places="many" keeps every place']
assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [
- "2: places is not a list"]
+ "2: places is not a list", 'no row has a place under places="one": declare "none"']
assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [
"3: ('Berlin', False) is not a Place"]
@@ -380,3 +538,45 @@ def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
def test_the_contract_types_are_what_the_package_exports() -> None:
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
+
+
+# ------------------------------------------------------------ the field check
+
+def test_the_field_check_passes_rows_that_state_what_is_declared() -> None:
+ reads = _reads(text="detail", teaser=True, listing="whole",
+ fields=Fields(pay="figures", posted="trusted", remote="stated"))
+ listed = [{"source_id": "1", "teaser": "Short.", "remote": False, "posted": "2026-09-30"}]
+ detail = [{"description": "Long.", "salary_min": 50000}]
+ assert field_problems(reads, listed, detail) == []
+
+
+def test_the_field_check_names_a_promise_no_row_keeps_and_a_denial_a_row_breaks() -> None:
+ reads = _reads(fields=Fields(pay="line", employment="stated"))
+ listed = [{"source_id": "1", "description": "Text.", "salary_min": 1, "remote": "hybrid",
+ "posted": "2026-09-30"}]
+ assert field_problems(reads, listed, []) == [
+ 'pay="line" wants salary_text on some row and figures on none',
+ 'posted="none" and a row has posted',
+ 'remote="none" and a row has it',
+ 'employment="stated" and no row has it']
+
+
+def test_the_field_check_takes_a_date_from_detail_only_where_it_is_said() -> None:
+ reads = _reads(text="detail", fields=Fields(posted="trusted"))
+ listed = [{"source_id": "1"}]
+ detail = [{"description": "Text.", "posted": "2026-09-30"}]
+ assert field_problems(reads, listed, detail, posted_by_detail=True) == []
+ assert field_problems(reads, listed, detail) == [
+ 'posted="trusted" and no listed row has posted']
+ assert field_problems(reads, [*listed, {"posted": "2026-09-30"}], detail,
+ posted_by_detail=True) == [
+ 'posted="trusted" by detail wants posted from detail only']
+
+
+def test_the_field_check_holds_text_and_teaser_to_the_declaration() -> None:
+ excerpt = _reads(text="excerpt")
+ assert field_problems(excerpt, [{"teaser": "Short."}], []) == []
+ assert field_problems(excerpt, [{"teaser": "Short.", "description": "Long."}], []) == [
+ 'text="excerpt" and the rows do not show it']
+ assert field_problems(_reads(), [{"description": "Long.", "teaser": "Short."}], []) == [
+ "a row has a teaser the declaration does not name"]
From 531c8bc337f6dc6442fb738a854e33b56a69336e Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 15:51:48 +0200
Subject: [PATCH 3/9] Delete the _UNREADABLE entries of hosts that have a
module
Ten entries were shadowed by their modules. A test now holds that no entry
names a reader, which replaces the per-host checks.
Co-Authored-By: Claude Fable 5.1
---
app/jobs/discover.py | 17 ++++-------------
tests/test_ats_bamboohr.py | 1 -
tests/test_ats_eightfold.py | 1 -
tests/test_ats_gem.py | 1 -
tests/test_ats_jibe.py | 1 -
tests/test_ats_pinpoint.py | 1 -
tests/test_discover.py | 7 +++++++
7 files changed, 11 insertions(+), 18 deletions(-)
diff --git a/app/jobs/discover.py b/app/jobs/discover.py
index c4bd671..121c282 100644
--- a/app/jobs/discover.py
+++ b/app/jobs/discover.py
@@ -10,7 +10,8 @@ parses) becomes a `board` target the pull can read; any other host is
recorded as `ats`, with its slug where the URL carries one, so the company is
kept as one the app cannot read yet rather than dropped. The patterns and
probes of a readable host are its module's; the hosts only named are in
-`_UNREADABLE` here, and one of them that gains a module is read from then on.
+`_UNREADABLE` here, and a host that gains a module loses its entry there
+(`tests/test_discover.py` holds that no entry names a reader).
A Teamtailor site on the company's own domain names no slug: the page only
loads Teamtailor's CDN. The board there is the site's own host, and it is
@@ -65,27 +66,17 @@ READABLE: tuple[str, ...] = tuple(readers.HOSTS)
#: company by, kept for the day a parser exists; a host with no group is
#: recognised but its board has no slug the app could use.
_UNREADABLE: tuple[tuple[str, str], ...] = (
- ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
- ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
- ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
- ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
- ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
- ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
- ("eightfold", r"\.eightfold\.ai"),
- ("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"),
- ("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
- ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
)
@@ -94,8 +85,8 @@ def _hosts(modules: Mapping[str, ModuleType],
"""Every host's patterns, compiled: the readable hosts' from their
modules first, so the first match at one place on a page is the one the
app can read, then the unreadable ones. An unreadable host that has
- gained a module is left out here, since its module's patterns replace
- it."""
+ a module is left out, so its module's patterns win even before its
+ entry is deleted."""
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py
index cdb773d..09f17d4 100644
--- a/tests/test_ats_bamboohr.py
+++ b/tests/test_ats_bamboohr.py
@@ -187,4 +187,3 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None:
'xy'
'z')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
- assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py
index 171ac68..266098b 100644
--- a/tests/test_ats_eightfold.py
+++ b/tests/test_ats_eightfold.py
@@ -201,7 +201,6 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None
found = [(h, s) for h, s, _ in discover.boards_in(page)]
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
assert ("eightfold", "") in found
- assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None:
diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py
index dee67af..c5f76bf 100644
--- a/tests/test_ats_gem.py
+++ b/tests/test_ats_gem.py
@@ -178,4 +178,3 @@ def test_discovery_finds_the_board_keeping_its_case() -> None:
'y')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
("gem", "Productboard"), ("gem", "sennder-gmbh")]
- assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py
index 61e8bc7..05568c4 100644
--- a/tests/test_ats_jibe.py
+++ b/tests/test_ats_jibe.py
@@ -176,7 +176,6 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
page = (''
'apply')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
- assert "jibe" not in {h for h, _ in discover._UNREADABLE}
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
diff --git a/tests/test_ats_pinpoint.py b/tests/test_ats_pinpoint.py
index e568e3e..b3741de 100644
--- a/tests/test_ats_pinpoint.py
+++ b/tests/test_ats_pinpoint.py
@@ -145,7 +145,6 @@ def test_discovery_finds_the_subdomain() -> None:
'')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
assert discover.boards_in("https://pinpointhq.com/") == []
- assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_a_pinpoint_board_is_not_probed() -> None:
diff --git a/tests/test_discover.py b/tests/test_discover.py
index 2e072f4..2440f25 100644
--- a/tests/test_discover.py
+++ b/tests/test_discover.py
@@ -17,6 +17,7 @@ from app.jobs import discover as mod
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
from app.jobs.sources import seed_companies
from app.storage import Store
+from plugins import readers
# ------------------------------------------------------------------ parsing
@@ -95,6 +96,12 @@ def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
+def test_no_unreadable_entry_names_a_host_with_a_module() -> None:
+ """A new module deletes its host's `_UNREADABLE` entry: its own patterns
+ replace it."""
+ assert not {host for host, _ in mod._UNREADABLE} & set(readers.HOSTS)
+
+
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
page = ('a'
'b')
From 72252a1c04fb40c2ce780aab29ed4e4b3f5d8a87 Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 15:52:38 +0200
Subject: [PATCH 4/9] Hold plugins/readers/ to its contract, helpers, the
standard library, httpx and pydantic
Co-Authored-By: Claude Fable 5.1
---
tests/test_architecture.py | 68 ++++++++++++++++++++++++++++++++++++++
1 file changed, 68 insertions(+)
diff --git a/tests/test_architecture.py b/tests/test_architecture.py
index 58d46d7..5440c6f 100644
--- a/tests/test_architecture.py
+++ b/tests/test_architecture.py
@@ -12,6 +12,11 @@ file holds `app`'s own layers.
there the door is the tool: `desk.tools.`, one level, no deeper.
3. **Plugins.** A file under `plugins/` imports `desk` and nothing else from
these trees. A plugin is app-agnostic; it never sees `app`.
+4. **Readers.** A file of `plugins/readers/` imports the standard library,
+ `httpx`, `pydantic`, `desk` through its doors and, within the package,
+ `._contract`, `._shared` and `._countries` only: a reader shares no code
+ with another reader, nor with `_runner`, which reads the readers and so
+ alone may import the package's registry, `plugins.readers`.
Tests are held to none of this.
"""
@@ -21,6 +26,7 @@ from __future__ import annotations
import ast
import importlib
import importlib.util
+import sys
from pathlib import Path
import pytest
@@ -88,6 +94,13 @@ def _package(path: Path) -> str:
APP_FILES = sorted(p for p in (ROOT / APP).rglob("*.py") if "__pycache__" not in p.parts)
PLUGIN_FILES = sorted((ROOT / "plugins").glob("*.py"))
+READER_FILES = sorted((ROOT / "plugins" / "readers").glob("*.py"))
+
+#: What a reader may import beside the standard library and `desk`.
+READER_THIRD_PARTY = frozenset({"httpx", "pydantic"})
+
+#: The package's own modules a reader may import.
+READER_SIBLINGS = frozenset({"_contract", "_shared", "_countries"})
def _outside(here: str, source: str):
@@ -120,6 +133,38 @@ def plugin_violations(source: str) -> list[str]:
return wrong
+def reader_violations(source: str, registry: bool = False) -> list[str]:
+ """Every import in a file of `plugins/readers/` outside rule 4;
+ `registry` allows `from plugins import readers`, for `_runner`."""
+ wrong = []
+ for node in ast.walk(ast.parse(source)):
+ if registry and isinstance(node, ast.ImportFrom) and node.level == 0 \
+ and node.module == "plugins" and [a.name for a in node.names] == ["readers"]:
+ continue
+ if isinstance(node, ast.Import):
+ tops = [(a.name.split(".")[0], a.name) for a in node.names]
+ elif isinstance(node, ast.ImportFrom) and node.level == 0:
+ tops = [((node.module or "").split(".")[0], node.module or "")]
+ elif isinstance(node, ast.ImportFrom):
+ names = {node.module} if node.module else {a.name for a in node.names}
+ if node.level != 1 or not names <= READER_SIBLINGS:
+ wrong.append(f"line {node.lineno}: {'.' * node.level}{node.module or ''} "
+ f"{sorted(names)} (a reader imports {', '.join(sorted(READER_SIBLINGS))})")
+ continue
+ else:
+ continue
+ for top, module in tops:
+ if top == "desk":
+ if isinstance(node, ast.ImportFrom) and (
+ reason := _door_violation(module, [a.name for a in node.names
+ if not _is_module(f"{module}.{a.name}")])):
+ wrong.append(f"line {node.lineno}: {module} ({reason})")
+ elif top not in sys.stdlib_module_names and top not in READER_THIRD_PARTY:
+ wrong.append(f"line {node.lineno}: {module} (a reader imports the standard "
+ f"library, httpx, pydantic and desk)")
+ return wrong
+
+
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
def test_a_package_imports_only_from_below_it(path: Path) -> None:
wrong = direction_violations(_package(path), path.read_text(encoding="utf-8"))
@@ -141,6 +186,12 @@ def test_a_plugin_knows_only_desk(path: Path) -> None:
assert not wrong, "\n".join(wrong)
+@pytest.mark.parametrize("path", READER_FILES, ids=lambda p: p.name)
+def test_a_reader_knows_only_its_contract_and_helpers(path: Path) -> None:
+ wrong = reader_violations(path.read_text(encoding="utf-8"), path.name == "_runner.py")
+ assert not wrong, "\n".join(wrong)
+
+
def test_the_rules_catch_what_they_are_for() -> None:
"""The checker itself, on sources that break each rule."""
assert plugin_violations("from app.storage import Store")
@@ -148,6 +199,23 @@ def test_the_rules_catch_what_they_are_for() -> None:
assert plugin_violations("from desk.sources.base import Source")
assert not plugin_violations("from desk.sources import Source, Item")
+ assert reader_violations("from plugins import ats")
+ assert reader_violations("from plugins import readers")
+ assert not reader_violations("from plugins import readers", registry=True)
+ assert reader_violations("from plugins import ats", registry=True)
+ assert reader_violations("from app.prep import words")
+ assert reader_violations("from . import _runner")
+ assert reader_violations("from ._runner import fetch")
+ assert reader_violations("from .lever import parse")
+ assert reader_violations("from .. import ats")
+ assert reader_violations("import requests")
+ assert reader_violations("from desk.sources.base import Source")
+ assert not reader_violations("import json\nfrom urllib.parse import quote\nimport httpx\n"
+ "from pydantic import ConfigDict\n"
+ "from desk.sources import strip_html\n"
+ "from . import _contract\nfrom ._shared import only\n"
+ "from ._countries import code_of")
+
assert direction_violations("storage", "from app.web import TABS")
assert direction_violations("web", "from app.main import create_app")
assert not direction_violations("main", "from app.web import pages_router")
From 36bc639384add5c69301d1927bdfee94fff19ca1 Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 15:53:47 +0200
Subject: [PATCH 5/9] Guard every read that is not a sweep, treat a read that
asks nothing as incomplete, and mark a filtered read that could not ask
remote, so no board loses its roles to a changed answer or a filter it could
not send
Co-Authored-By: Claude Fable 5.1
---
plugins/ats.py | 64 +++++++----
plugins/readers/_runner.py | 226 +++++++++++++++++++++++++------------
tests/test_readers.py | 134 +++++++++++++++++++++-
tests/test_runner.py | 24 ++--
4 files changed, 332 insertions(+), 116 deletions(-)
diff --git a/plugins/ats.py b/plugins/ats.py
index 0533775..08308a6 100644
--- a/plugins/ats.py
+++ b/plugins/ats.py
@@ -16,7 +16,9 @@ The allow-list and the classifier are ported from `openings.go` and
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is read independently; one that fails is logged,
-recorded as failing under `ats:failing:`, and skipped. A board's
+recorded as failing under `ats:failing:`, and skipped, and so is one
+whose read lists under a tenth of what it listed last time (`_runner.read`'s
+guard), until it has failed that way for the pull's `stale_days`. A board's
filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`) win, narrowed to what its reader declares
@@ -26,7 +28,8 @@ the whole board: it prunes no per-role state.
A board's size (`board_size`) is set once, when the board is added
(`app.jobs.sources.set_board`) or rechecked, and kept under
`ats:size:` with the day it was checked under
-`ats:sizechecked:` (`keep_size`); the reach reads it to leave small
+`ats:sizechecked:` (`keep_size`, which also resets the guard's count,
+so a Recheck accepts a board that shrank); the reach reads it to leave small
boards whole. A pull does not rewrite it. A board with no size yet is sized
in its batch before it is read: its reach then says `unsized` and the size
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
@@ -106,8 +109,9 @@ _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
-def _limits() -> _runner.Limits:
- return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
+def _limits(stale_days: int | None = None) -> _runner.Limits:
+ return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP,
+ stale_days=stale_days)
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
@@ -339,8 +343,7 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
- if (_runner.declared(module) and module.READS.listing == "paged"
- and module.READS.complete == "stated"):
+ if module.READS.listing == "paged" and module.READS.complete == "stated":
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
if isinstance(total, int):
return total
@@ -348,11 +351,18 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
-def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
+def keep_size(state: Any, target: str, size: int, *, today: str = "",
+ reset: bool = True) -> None:
"""Keep board `target`'s size and the day it was checked (ISO,
- `today`, else today in UTC)."""
+ `today`, else today in UTC). With `reset`, the default and what a
+ Recheck does, the 10% guard's count goes too (`ats:count:`), so
+ the next read of a board that really shrank is taken as it is instead of
+ failing as suspect; the plugin sizing a board as it reads it passes
+ False, since its count is the read it just stamped."""
state[f"ats:size:{target}"] = str(size)
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
+ if reset:
+ state.pop(f"ats:count:{target}", None)
failure_reason = _runner.failure_reason
@@ -400,14 +410,15 @@ class ATSSource(Source):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
- ask, unsized = self._ask(client, target, module)
+ ask, unsized, remote = self._ask(client, target, module)
+ stale_days = self.options.get("stale_days")
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
- limits=_limits())
+ limits=_limits(int(stale_days) if stale_days else None),
+ remote=remote)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
_runner.record_failure(self.state, failing, exc)
return
- self.state.pop(failing, None)
def compound(source_id: str) -> str:
# Matches `roles.source_id`, which `Store.mark_closed` compares.
@@ -435,11 +446,13 @@ class ATSSource(Source):
yield item_of(row, target, label)
# Stamped only once every row went out, so a board that blew up half
# way through is neither marked ok nor has a partial id list treated
- # as the whole of it.
+ # as the whole of it, and it stays failing until then: a row that
+ # fails to store (`app.jobs.pull.run_batch`) leaves the record in place.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
+ self.state.pop(failing, None)
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
- keep_size(self.state, target, len(listing.ids))
+ keep_size(self.state, target, len(listing.ids), reset=False)
if detail_sets is not None:
# A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
@@ -447,9 +460,11 @@ class ATSSource(Source):
self.state[key] = json.dumps(sorted(ids))
def _ask(self, client: httpx.Client, target: str,
- module: Any) -> tuple[readers.Ask, bool]:
+ module: Any) -> tuple[readers.Ask, bool, bool]:
"""What `target` is asked for, from the reach its batch handed it and
- its params, and whether it still needs a size. A board with no size
+ its params, whether it still needs a size, and whether that reach
+ and params want remote roles (`_runner.remote_wanted`, which the
+ runner compares with what it could ask). A board with no size
yet (`unsized` in its reach) whose reach would narrow the read is
sized first (`board_size`, kept by `keep_size`), and read whole under
`small` roles. One whose read is whole anyway is sized by that read,
@@ -457,16 +472,18 @@ class ATSSource(Source):
given = self.options.get("reach")
params, keys = self.params_for(target), self.options.get("keys")
if not isinstance(given, dict) or not given.get("unsized"):
- return _runner.ask_for(module, params, given, keys=keys), False
- reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
- ask = _runner.ask_for(module, params, reach or None, keys=keys)
+ return (_runner.ask_for(module, params, given, keys=keys), False,
+ _runner.remote_wanted(module, params, given))
+ reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None
+ ask = _runner.ask_for(module, params, reach, keys=keys)
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
- return ask, True
+ return ask, True, _runner.remote_wanted(module, params, reach)
size = board_size(target, client=client)
- keep_size(self.state, target, size)
+ keep_size(self.state, target, size, reset=False)
if size < int(given.get("small") or 0):
+ reach = None
ask = _runner.ask_for(module, params, None, keys=keys)
- return ask, False
+ return ask, False, _runner.remote_wanted(module, params, reach)
# ------------------------------------------------------ the per-role reads
@@ -567,7 +584,8 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
def item_of(row: Row, target: str, label: str) -> Item:
"""One row of board `target` as the `Item` the store takes: its id
- compound, `host:slug:id`, as `roles.source_id` has it."""
+ compound, `host:slug:id`, as `roles.source_id` has it. A row without a
+ date is dated now, as `Item` itself would date it."""
host, _, slug = target.partition(":")
description = row.get("description") or ""
return Item(
@@ -576,7 +594,7 @@ def item_of(row: Row, target: str, label: str) -> Item:
title=row["title"],
excerpt=description[:300],
body=description,
- published_at=row.get("posted"),
+ published_at=row.get("posted") or datetime.now(UTC),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py
index ebf070c..edd6915 100644
--- a/plugins/readers/_runner.py
+++ b/plugins/readers/_runner.py
@@ -5,23 +5,25 @@ A reader says how its host answers; this module reads it: the guarded fetch
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
to (`ask_for`) and its series (`series`), the listing whole or paged, one
series per `one`-filter value unioned by id (`read_listing`), completeness,
-the sweep of an oversized paged board across pulls, the 10% guard on a
-filtered or `empty="suspect"` read (`read`), the per-role detail reads
+the sweep of an oversized paged board across pulls, the 10% guard on every
+read that is not a sweep (`read`), the per-role detail reads
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
`plugins/search.py` are thin `Source` classes over it: they pass their own
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
-as they were.
-
-A module without `READS` is read with the older signatures, `feed_url(slug)`
-and `page(get, slug, cursor)`, and never gets an `Ask`.
+as they were. Every reader declares `READS`; a module without it is refused
+by the package before it gets here.
State keys, under the Source's prefix and the target (`ats:sweep:`):
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
"seen", "rows", "whole"}`, continued by the next pull
- count `{"asks", "count"}`, the roles the last whole read listed, for
- the 10% guard
+ count `{"asks", "count"}`, the roles the last read that was not a
+ sweep listed, for the 10% guard
+ failing `{"since", "reason"}`, kept by the Source (`record_failure`);
+ the guard gives way once `since` is `Limits.stale_days` old
+ remote_unasked "1" when the reach wanted remote and the read could not
+ ask for it (`Listing.remote_unasked`), else ""
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
swept "1" when the last stamped listing finished a sweep carried over
from an earlier pull (`Listing.swept`), else "". Offset paging
@@ -87,10 +89,16 @@ DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "count
#: How long a board's `facets` are kept before they are asked again.
FACETS_TTL = timedelta(days=1)
-#: A filtered or `empty="suspect"` read listing under this share of what the
-#: same read listed last time counts as a failure.
+#: A read listing under this share of what the same read listed last time
+#: counts as a failure (`read`).
SUSPECT_SHARE = 0.1
+#: The last count from which the guard holds every read. A read that is
+#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
+#: a count of 1, as it was before every read was; a whole read of a board
+#: that listed a handful of roles can drop to none by itself.
+SUSPECT_MIN = 10
+
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
@@ -105,8 +113,8 @@ class TooLarge(ValueError):
class Suspect(Exception):
- """A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE`
- of what it listed last time."""
+ """A read that listed under `SUSPECT_SHARE` of what it listed last time
+ (see `read`)."""
class MissingKey(ValueError):
@@ -115,12 +123,15 @@ class MissingKey(ValueError):
@dataclass(frozen=True, slots=True)
class Limits:
- """What one Source allows a read: pages, the pause between requests, and
- the detail reads a pull."""
+ """What one Source allows a read: pages, the pause between requests, the
+ detail reads a pull, and the days after which the 10% guard accepts the
+ answer it keeps refusing (`stale_days`, the pull's `STALE_DAYS`; None,
+ for a Source run outside a pull, never)."""
max_pages: int = MAX_PAGES
page_delay: float = PAGE_DELAY
detail_cap: int = DETAIL_CAP
+ stale_days: int | None = None
@dataclass(slots=True)
@@ -134,7 +145,9 @@ class Listing:
neither are the whole board. `swept` says a complete listing finished a
sweep carried over from an earlier pull: `ids` were read across pulls,
and roles closing meanwhile may have shifted an offset-paged host's later
- roles back past the cursor, so `ids` can miss a role that is still open."""
+ roles back past the cursor, so `ids` can miss a role that is still open.
+ `remote_unasked` says a filtered read was meant to ask for remote roles
+ as well and could not, so its `ids` say nothing of the remote roles."""
rows: list[Row]
complete: bool
@@ -144,12 +157,14 @@ class Listing:
asks: str = ""
narrowed: bool = False
swept: bool = False
+ remote_unasked: bool = False
# --------------------------------------------------------------- modules
def declared(module: ModuleType) -> bool:
- """Whether `module` declares `READS`, and so takes `ask`."""
+ """Whether `module` declares `READS`. Every reader does, so the runner no
+ longer asks; `app.jobs.reach` still does."""
return isinstance(getattr(module, "READS", None), Reads)
@@ -334,6 +349,33 @@ def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
return count
+def _wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None,
+ reach: Mapping[str, Any] | None) -> tuple[dict[str, Any], bool]:
+ """The dimensions `reach` and then the params set, params winning, before
+ they are narrowed to what the reader declares, and whether the read is
+ wide. A reach of more than `MAX_SERIES` countries on a `one` country
+ filter is dropped whole, which makes the read wide; a `remote` of False
+ asks nothing."""
+ given = reach or {}
+ reached = _dimensions(module, given)
+ wide = bool(given.get("wide"))
+ if module.READS.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
+ reached = {}
+ wide = True
+ wanted = reached | _dimensions(module, dict(subscription_params or {}))
+ if wanted.get("remote") is False:
+ del wanted["remote"]
+ return wanted, wide
+
+
+def remote_wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
+ reach: Mapping[str, Any] | None = None) -> bool:
+ """Whether the reach or the params want remote roles, whether or not the
+ reader can ask for them. `read_listing` compares it with what the series
+ asked (`Listing.remote_unasked`)."""
+ return _wanted(module, subscription_params, reach)[0].get("remote") is True
+
+
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
reach: Mapping[str, Any] | None = None, *,
keys: Mapping[str, Any] | None = None) -> Ask:
@@ -346,7 +388,8 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
dropped. A `remote` of False asks nothing. A search's params go whole into
`Ask.params`; a reader with `READS.key` gets the resolved key under
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
- A module without `READS` gets `Ask()`.
+ A remote the reader cannot filter on is dropped here; `remote_wanted`
+ still says it was wanted.
Fetch terms (`reach["terms"]`) go only to a board reader that declares
`query="one"`, and only on a wide read: `reach["wide"]` (a company set
@@ -358,19 +401,10 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
whole, since remote without its countries would read only the remote
roles; over it with terms alone, the terms go too and the board is read
unfiltered."""
- if not declared(module):
- return Ask()
reads: Reads = module.READS
params = dict(subscription_params or {})
given = reach or {}
- reached = _dimensions(module, given)
- wide = bool(given.get("wide"))
- if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
- reached = {}
- wide = True
- wanted = reached | _dimensions(module, params)
- if wanted.get("remote") is False:
- del wanted["remote"]
+ wanted, wide = _wanted(module, params, given)
declared_now = reads.filters.declared()
wanted = {name: value for name, value in wanted.items() if name in declared_now}
terms = _terms(given.get("terms"))
@@ -400,7 +434,7 @@ def narrowed(module: ModuleType, ask: Ask) -> bool:
query. Such a read is never the whole board, nor the board within its
places, so it closes nothing. A search's query is what
the search is, and does not count."""
- board = not declared(module) or module.READS.scope == "board"
+ board = module.READS.scope == "board"
return bool(ask.terms) or (board and bool(ask.query))
@@ -417,10 +451,7 @@ def series(module: ModuleType, ask: Ask,
remote part beside places is then left out. Each of those is then asked
once per fetch term (`ask.terms`), with the term as `query`: the host
intersects a keyword with its facets, as the survey found on every
- host, so the union by id is "one of the terms, in these places". `[ask]`
- for a module without `READS`."""
- if not declared(module):
- return [ask]
+ host, so the union by id is "one of the terms, in these places"."""
if ask.remote and (ask.countries or ask.places):
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
@@ -450,8 +481,8 @@ def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]
known = known_facets.get(dimension) or {}
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
if not mapped:
- log.info("readers: %s knows none of %s %s", module.NAME, dimension,
- ", ".join(values))
+ log.warning("readers: %s knows none of %s %s; nothing asked", module.NAME,
+ dimension, ", ".join(values))
return []
if dimension == "country":
countries = mapped
@@ -497,7 +528,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
for index, ask in enumerate(asks):
if index:
time.sleep(limits.page_delay)
- url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug)
+ url = module.feed_url(slug, ask)
try:
body = get(url)
except TooLarge:
@@ -514,7 +545,8 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
can_sweep: bool = False,
- facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing:
+ facets: Mapping[str, Mapping[str, str]] | None = None,
+ remote: bool = False) -> Listing:
"""One target's listing, read and parsed. Raises on any failure of the
first request, and on a reader that does not understand an answer.
@@ -532,22 +564,35 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
its series and cursor; the sweep is complete when it reaches the end,
and `ids` then hold every id it listed, and the listing is `swept`. A
read at the reader's `ceiling` is not swept: the host answers nothing
- past it."""
+ past it.
+
+ A filtered read that asks no series, because an `ids` dimension names
+ no value the host's facets know, asked nothing, which is no evidence of
+ anything: it is incomplete and closes nothing. `filtered` is a board's
+ only: a search's places are what the search is, as its query is.
+ `remote` says the reach or the params wanted remote roles; when no series
+ asked for them (a reader without a remote filter, or one whose facets
+ state none beside places) the listing is `remote_unasked`."""
limits = limits or Limits()
ask = ask or Ask()
+ reads: Reads = module.READS
asks = series(module, ask, facets)
- is_filtered = filtered(ask)
+ is_filtered = filtered(ask) and reads.scope == "board"
by_words = narrowed(module, ask)
+ unasked = (is_filtered and (remote or bool(ask.remote))
+ and not any(a.remote or a.remote_ids for a in asks))
key = fingerprint(asks)
if not asks:
- return Listing([], True, [], is_filtered, None, key, by_words)
+ log.warning("readers: %s:%s: the filter maps to nothing the host knows; "
+ "nothing asked, not closing", module.NAME, slug)
+ return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
if not hasattr(module, "page"):
- rows = _whole(module, slug, get, asks, limits)
- return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words)
+ whole_rows = _whole(module, slug, get, asks, limits)
+ return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
+ key, by_words, remote_unasked=unasked)
- reads: Reads | None = module.READS if declared(module) else None
- never = reads is not None and reads.complete == "never"
- ceiling = reads.ceiling if reads is not None else None
+ never = reads.complete == "never"
+ ceiling = reads.ceiling
start, cursor, seen, earlier, whole = 0, None, set(), [], True
resumed = bool(sweep and sweep.get("asks") == key)
if sweep and resumed:
@@ -568,10 +613,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
if pages:
time.sleep(limits.page_delay)
try:
- if reads is not None:
- data, results, total, cursor = module.page(get, slug, cursor, asks[index])
- else:
- data, results, total, cursor = module.page(get, slug, cursor)
+ data, results, total, cursor = module.page(get, slug, cursor, asks[index])
except httpx.HTTPStatusError as exc:
if not pages or exc.response.status_code != 429:
raise
@@ -603,15 +645,17 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
listed = list(dict.fromkeys(earlier + list(rows)))
if not stop:
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
- resumed and whole)
+ resumed and whole, unasked)
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
module.NAME, slug, pages, stop)
at_ceiling = ceiling is not None and len(seen) >= ceiling
if not can_sweep or never or at_ceiling or not _storable(cursor):
- return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words)
+ return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words,
+ remote_unasked=unasked)
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
"rows": listed, "whole": whole}
- return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words)
+ return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words,
+ remote_unasked=unasked)
def _storable(cursor: Any) -> bool:
@@ -637,7 +681,7 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
(a country, a place, or remote),
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
- if not declared(module) or not hasattr(module, "facets"):
+ if not hasattr(module, "facets"):
return {}
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
@@ -659,46 +703,79 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
slug: str, get: Get, ask: Ask | None = None, *,
- limits: Limits | None = None, can_sweep: bool = True) -> Listing:
+ limits: Limits | None = None, can_sweep: bool = True, remote: bool = False) -> Listing:
"""`read_listing` for one target with its state under `:...:`:
- the facets it needs, the sweep it continues, and the 10% guard. A read
- that is filtered, narrowed by terms, or whose reader declares
- `empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same
- read listed last time (the same `fingerprint`, so a term read is only
- held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
- the rest once the rows are stored."""
+ the facets it needs, the sweep it continues, and the 10% guard. Nothing
+ is written here but the facets; `stamp` keeps the rest once the rows are
+ stored.
+
+ The guard holds every read that is not a sweep: one that listed under
+ `SUSPECT_SHARE` of what the same read listed last time (the same
+ `fingerprint`, so a term read is only held to a term read with the same
+ terms and places) raises `Suspect`, and the Source records the target
+ as failing, so its roles stay open. A host whose answer changed shape
+ parses to no rows with a 200, and closing a board on that would lose it
+ whole. It holds from a last count of `SUSPECT_MIN`, or of 1 for a read
+ that is filtered, narrowed or of an `empty="suspect"` reader. A board
+ that really shrank is accepted once the target has been failing for
+ `limits.stale_days` (its `:failing:` record), and its
+ stamp then resets the count; a Recheck resets it at once
+ (`plugins.ats.keep_size`)."""
+ limits = limits or Limits()
ask = ask or Ask()
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
- sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets)
- suspect = declared(module) and module.READS.empty == "suspect"
+ sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
+ remote=remote)
+ held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
last = _load(state, f"{prefix}:count:{target}")
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
- if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count
+ if (listing.sweep is None and count >= (1 if held else SUSPECT_MIN)
and len(listing.ids) < SUSPECT_SHARE * count):
- raise Suspect(f"listed {len(listing.ids)} roles, under "
- f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
+ why = (f"listed {len(listing.ids)} roles, under "
+ f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
+ if _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
+ log.warning("readers: %s:%s: %s, accepted after %d days of failing",
+ module.NAME, slug, why, limits.stale_days)
+ return listing
+ raise Suspect(why)
return listing
+def _failing_for(state: Mapping[str, str], key: str, days: int | None) -> bool:
+ """Whether the failure kept under `key` began at least `days` ago; False
+ for `days` None or no record."""
+ since = str(_load(state, key).get("since") or "")
+ if days is None or not since:
+ return False
+ try:
+ began = datetime.fromisoformat(since)
+ except ValueError:
+ return False
+ return datetime.now(UTC) - began >= timedelta(days=days)
+
+
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
listing: Listing, ids: list[str]) -> None:
"""Record a read whose rows are stored: `ok` (the run's marker),
`complete`, `ids` (the compound ids, what the pull closes against),
`filtered`, `narrowed` (a read by terms, which the pull never closes
- against), whether it finished a sweep from earlier pulls (`swept`), the
+ against), whether it finished a sweep from earlier pulls (`swept`),
+ whether a filtered read left remote unasked (`remote_unasked`), the
sweep to continue or none, and the count for the 10% guard once a read
- is whole."""
+ is not a sweep. A read that asked nothing keeps the count it had."""
state[f"{prefix}:ok:{target}"] = run
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
+ state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
if listing.sweep is not None:
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
- else:
- state.pop(f"{prefix}:sweep:{target}", None)
+ return
+ state.pop(f"{prefix}:sweep:{target}", None)
+ if listing.asks != fingerprint([]):
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
"count": len(listing.ids)})
@@ -766,7 +843,7 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
name, slug, row["source_id"], ", ".join(ignored))
- taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment]
+ taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
if taken.get("location") and "places" not in taken:
taken["places"] = [{"text": taken["location"], "remote": None}]
filled[key] = taken
@@ -776,12 +853,12 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
for row in rows:
key = compound(row)
if key in filled:
- out.append(row | {"description": ""} | filled[key]) # type: ignore[operator]
+ out.append(row | {"description": ""} | filled[key])
elif key in detailed:
- out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator]
+ out.append(row | {"description": "", "location": "", "places": [],
"remote": None})
else:
- out.append(row | {"description": ""}) # type: ignore[operator]
+ out.append(row | {"description": ""})
if listed is not None:
done &= listed
failing &= listed
@@ -791,8 +868,9 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
# -------------------------------------------------------------------- raw
def places_of(row: Mapping[str, Any]) -> list[Place]:
- """A row's places: its `places`, or, for a row without (an undeclared
- reader's), its `location` as the one place, split nothing."""
+ """A row's places: its `places`, or, for a row without (a raw row stored
+ before readers returned places), its `location` as the one place, split
+ nothing."""
places = row.get("places")
if not isinstance(places, list):
location = str(row.get("location") or "").strip()
diff --git a/tests/test_readers.py b/tests/test_readers.py
index 7791581..49251f6 100644
--- a/tests/test_readers.py
+++ b/tests/test_readers.py
@@ -26,7 +26,7 @@ if str(ROOT) not in sys.path:
from plugins import ats, readers # noqa: E402
from plugins.readers import _runner # noqa: E402
from plugins.readers import lever # noqa: E402
-from plugins.readers._contract import Ask # noqa: E402
+from plugins.readers._contract import Ask, Filters, Reads # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -253,7 +253,9 @@ def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
- module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
+ module.READS = Reads(scope="board", listing="whole", complete="whole", places="one",
+ text="detail" if detail is not None else "listing", robots="test")
+ module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
@@ -590,3 +592,131 @@ def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
with _client(handler) as client, pytest.raises(ats._Refused):
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
+
+
+# ------------------------------------------- the guard, asks, remote unasked
+
+def _listing_host(**reads: Any) -> Any:
+ """A declared whole-listing board `fake` whose feed lists `module.ids`,
+ whatever is asked; `reads` override its declaration."""
+ module: Any = types.ModuleType("fake")
+ module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
+ module.slug_ok = lambda slug: slug.isalnum()
+ module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
+ module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
+ module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
+ "complete": "whole", "places": "one", "robots": "test"} | reads))
+ module.ids = []
+ module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
+ module.parse = lambda body, slug: [
+ {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
+ "places": [{"text": "Berlin", "remote": None}]} for i in json.loads(body)]
+ return module
+
+
+def _listed(module: Any) -> Callable[..., bytes]:
+ return lambda url, **kwargs: json.dumps(module.ids).encode()
+
+
+def _read_and_stamp(state: dict[str, str], module: Any,
+ limits: _runner.Limits | None = None) -> _runner.Listing:
+ listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
+ limits=limits)
+ _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
+ return listing
+
+
+def test_an_unfiltered_whole_read_of_nothing_after_forty_is_suspect() -> None:
+ """A Greenhouse-style board, trusted, unfiltered: a changed answer that
+ parses to no rows is refused rather than read as a board that closed."""
+ module = _listing_host()
+ state: dict[str, str] = {}
+ module.ids = [str(n) for n in range(40)]
+ _read_and_stamp(state, module)
+ module.ids = []
+ with pytest.raises(_runner.Suspect, match="listed 0 roles, under 10% of the 40"):
+ _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
+
+
+def test_a_whole_read_under_suspect_min_is_not_held() -> None:
+ module = _listing_host()
+ state: dict[str, str] = {}
+ module.ids = [str(n) for n in range(5)]
+ _read_and_stamp(state, module)
+ module.ids = []
+ assert _read_and_stamp(state, module).ids == []
+ assert json.loads(state["ats:count:fake:acme"])["count"] == 0
+
+
+def test_the_same_suspect_answer_is_accepted_after_stale_days() -> None:
+ module = _listing_host()
+ state: dict[str, str] = {}
+ module.ids = [str(n) for n in range(40)]
+ _read_and_stamp(state, module)
+ module.ids = ["1"]
+ failing = "ats:failing:fake:acme"
+ recent = (datetime.now(UTC) - timedelta(days=20)).isoformat(timespec="seconds")
+ state[failing] = json.dumps({"since": recent, "reason": "Suspect: listed 1 roles"})
+ limits = _runner.Limits(page_delay=0, stale_days=21)
+ with pytest.raises(_runner.Suspect):
+ _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits)
+
+ old = (datetime.now(UTC) - timedelta(days=22)).isoformat(timespec="seconds")
+ state[failing] = json.dumps({"since": old, "reason": "Suspect: listed 1 roles"})
+ # Outside a pull (no `stale_days`) the guard never gives way.
+ with pytest.raises(_runner.Suspect):
+ _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
+ assert _read_and_stamp(state, module, limits).ids == ["1"]
+ assert json.loads(state["ats:count:fake:acme"])["count"] == 1
+
+
+def test_keep_size_resets_the_guards_count_unless_told_not_to() -> None:
+ state = {"ats:count:fake:acme": json.dumps({"asks": "[]", "count": 40})}
+ ats.keep_size(state, "fake:acme", 40, reset=False)
+ assert "ats:count:fake:acme" in state
+ ats.keep_size(state, "fake:acme", 3)
+ assert "ats:count:fake:acme" not in state
+ assert state["ats:size:fake:acme"] == "3"
+
+
+def test_a_filter_that_maps_to_no_host_id_asks_nothing_and_is_incomplete(
+ monkeypatch: pytest.MonkeyPatch) -> None:
+ module = _listing_host(filters=Filters(country="ids"))
+ module.facets = lambda get, slug: {"country": {}}
+ module.ids = ["1", "2"]
+ state: dict[str, str] = {}
+ listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
+ Ask(countries=("DK",)))
+ assert (listing.rows, listing.ids, listing.complete, listing.filtered) == (
+ [], [], False, True)
+
+ # Through the plugin: stamped, but not complete, so the settle closes nothing.
+ monkeypatch.setitem(readers.HOSTS, "fake", module)
+ monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b'["1"]')
+ source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
+ "params": {"countries": ["DK"]}}], "run": "r"})
+ assert list(source.fetch()) == []
+ assert (source.state["ats:ok:fake:acme"], source.state["ats:complete:fake:acme"]) == (
+ "r", "")
+
+
+def test_a_filtered_read_that_cannot_ask_remote_says_so() -> None:
+ reach = {"countries": ["DK"], "remote": True}
+ blind = _listing_host(filters=Filters(country="many"))
+ ask = _runner.ask_for(blind, None, reach)
+ assert (ask.countries, ask.remote) == (("DK",), None)
+ assert _runner.remote_wanted(blind, None, reach)
+ listing = _runner.read_listing(blind, "acme", _listed(blind), ask,
+ remote=_runner.remote_wanted(blind, None, reach))
+ assert listing.filtered and listing.remote_unasked
+ state: dict[str, str] = {}
+ _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
+ assert state["ats:remote_unasked:fake:acme"] == "1"
+
+ # A host that filters on remote asks it as a series of its own.
+ sees = _listing_host(filters=Filters(country="many", remote="many"))
+ ask = _runner.ask_for(sees, None, reach)
+ listing = _runner.read_listing(sees, "acme", _listed(sees), ask, remote=True)
+ assert listing.filtered and not listing.remote_unasked
+ # A params' remote of False asks nothing and wants nothing.
+ assert not _runner.remote_wanted(blind, {"remote": False}, reach)
diff --git a/tests/test_runner.py b/tests/test_runner.py
index 69fd80a..124cf4f 100644
--- a/tests/test_runner.py
+++ b/tests/test_runner.py
@@ -103,8 +103,6 @@ def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
both = _whole(filters=Filters(country="many", remote="many"))
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
assert (ask.countries, ask.remote) == (("DE",), True)
- # A module without READS reads the whole board.
- assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
@@ -139,11 +137,12 @@ def test_ids_filters_are_mapped_through_the_facets() -> None:
ask = Ask(countries=("DK", "XX"))
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
assert mapped.countries == ("h-1",)
- # None known: the host has nothing in reach, and the read is empty.
+ # None known: nothing is asked, which is no evidence, so the read is
+ # incomplete and closes nothing.
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
facets={"country": {}})
- assert (listing.rows, listing.complete, listing.filtered) == ([], True, True)
+ assert (listing.rows, listing.complete, listing.filtered) == ([], False, True)
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
@@ -189,17 +188,6 @@ def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
_runner.read_listing(module, "acme", web)
-def test_a_module_without_reads_gets_the_older_signatures() -> None:
- module: Any = types.ModuleType("old")
- module.NAME = "old"
- module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
- module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
- listing = _runner.read_listing(module, "acme",
- Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
- Ask(countries=("DK",)))
- assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
-
-
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
pages = [["1", "2"], ["3"]]
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
@@ -333,9 +321,11 @@ def test_a_suspect_read_far_under_the_last_count_fails() -> None:
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
_runner.read(state, "ats", "t", module, "acme", web)
- # A reader whose empty answer is trusted reads it as it is.
+ # A reader whose empty answer is trusted is held too, from a count of
+ # `SUSPECT_MIN`.
module.READS = module.READS.model_copy(update={"empty": "trusted"})
- assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"]
+ with pytest.raises(_runner.Suspect):
+ _runner.read(state, "ats", "t", module, "acme", web)
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
From 24a165818c7f4d72c54299a97ebdcd8fd68c2303 Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 15:53:47 +0200
Subject: [PATCH 6/9] Read searches through the runner, so a search has the 10%
guard, the shared stamp and a search:failing record when it does not answer
Co-Authored-By: Claude Fable 5.1
---
plugins/search.py | 99 ++++++++++++++++++++++++++++-------------------
1 file changed, 59 insertions(+), 40 deletions(-)
diff --git a/plugins/search.py b/plugins/search.py
index 01a163f..50baf9d 100644
--- a/plugins/search.py
+++ b/plugins/search.py
@@ -17,14 +17,19 @@ one run is not closed here. A search that read everything that matches was
complete (`complete="stated"`, as many distinct results as the stated total),
and the pull closes what it stopped listing; the rest close once no search
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
+
+In a pull a search is read by `_runner.read` under the prefix `search`, as a
+board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
+and a search that fails or is suspect is kept as failing under
+`search:failing:` (`_runner.record_failure`) until it answers, which
+the settle reads as it reads a failing board's.
"""
from __future__ import annotations
-import json
import logging
import sys
-from collections.abc import Iterator, Mapping
+from collections.abc import Iterator, Mapping, MutableMapping
from datetime import UTC, datetime
from pathlib import Path
from types import ModuleType
@@ -65,8 +70,9 @@ def _pages(params: Mapping[str, Any]) -> int:
return DEFAULT_PAGES
-def _limits(params: Mapping[str, Any]) -> _runner.Limits:
- return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
+def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits:
+ return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY,
+ stale_days=stale_days)
def provider_of(target: str) -> str:
@@ -76,10 +82,13 @@ def provider_of(target: str) -> str:
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
- keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
+ keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None,
+ stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]:
"""One search read through the runner: its provider and its listing.
Raises ValueError for an unknown provider or a missing key, before any
- request."""
+ request. With `state` (a pull) it is `_runner.read` under `search`, the
+ 10% guard included and never swept; without (a search tried before it is
+ saved) a bare `read_listing`, since there is nothing to compare with."""
provider = provider_of(target)
if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of "
@@ -88,7 +97,11 @@ def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
slug = target.partition(":")[2]
ask = _runner.ask_for(module, params, keys=keys)
get = _runner.getter(client, module.allowed(slug), provider)
- return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
+ limits = _limits(params, stale_days)
+ if state is None:
+ return module, _runner.read_listing(module, slug, get, ask, limits=limits)
+ return module, _runner.read(state, "search", target, module, slug, get, ask,
+ limits=limits, can_sweep=False)
def run_search(target: str, params: dict[str, Any],
@@ -130,37 +143,43 @@ class SearchSource(Source):
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
- provider = provider_of(target)
- if not provider:
- log.warning("search: %r is not provider:name; skipped", target)
- continue
- try:
- _module, listing = _read(client, target, self.params_for(target),
- self.options)
- except Exception as exc:
- log.warning("search: %s failed (%s)", target, exc)
- continue
+ yield from self._search(client, target, label, run_marker)
- seen_ids = []
- for row in listing.rows:
- source_id = f"{provider}:{row['source_id']}"
- seen_ids.append(source_id)
- description = row.get("description") or ""
- yield Item(
- source_id=source_id,
- location=row["url"],
- title=row["title"],
- excerpt=description[:300],
- body=description,
- published_at=row.get("posted"),
- subscription=label,
- kind="role",
- raw=_runner.raw_of(row) | {
- "company": row.get("company") or "",
- "provider": provider,
- "search": target,
- },
- )
- self.state[f"search:ok:{target}"] = run_marker
- self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
- self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
+ def _search(self, client: httpx.Client, target: str, label: str,
+ run_marker: str) -> Iterator[Item]:
+ """One search: read, its rows out as items, then stamped. A search
+ that fails, a suspect answer included, is recorded under
+ `search:failing:` and stamps nothing; the record goes once a
+ read is stamped, after every row went out, as for a board."""
+ failing = f"search:failing:{target}"
+ provider = provider_of(target)
+ stale_days = self.options.get("stale_days")
+ try:
+ _module, listing = _read(client, target, self.params_for(target), self.options,
+ self.state, int(stale_days) if stale_days else None)
+ except Exception as exc:
+ log.warning("search: %s failed (%s)", target, exc)
+ _runner.record_failure(self.state, failing, exc)
+ return
+
+ now = datetime.now(UTC)
+ for row in listing.rows:
+ description = row.get("description") or ""
+ yield Item(
+ source_id=f"{provider}:{row['source_id']}",
+ location=row["url"],
+ title=row["title"],
+ excerpt=description[:300],
+ body=description,
+ published_at=row.get("posted") or now,
+ subscription=label,
+ kind="role",
+ raw=_runner.raw_of(row) | {
+ "company": row.get("company") or "",
+ "provider": provider,
+ "search": target,
+ },
+ )
+ _runner.stamp(self.state, "search", target, run_marker, listing,
+ [f"{provider}:{i}" for i in listing.ids])
+ self.state.pop(failing, None)
From 9c8a6107877fad5fc80cab4f6416d3d075e2b3c2 Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 15:53:47 +0200
Subject: [PATCH 7/9] Guard each target in a batch, close remote-unasked and
narrowed roles by age, settle failing searches, and raise one notice per
settle naming what needs a look
Co-Authored-By: Claude Fable 5.1
---
app/jobs/cascade.py | 12 +-
app/jobs/pull.py | 389 +++++++++++++++++++++++++++++++------------
tests/test_pull.py | 237 +++++++++++++++++++++++++-
tests/test_search.py | 53 ++++++
4 files changed, 576 insertions(+), 115 deletions(-)
diff --git a/app/jobs/cascade.py b/app/jobs/cascade.py
index c430545..3b389e1 100644
--- a/app/jobs/cascade.py
+++ b/app/jobs/cascade.py
@@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
Under `NULL` there is no queue to take the chain, so this raises rather
than report a pull that never happened; `app job pull` claims the run and
- starts the chain on a queue of its own instead of calling this.
+ starts the chain on a queue of its own instead of calling this. Anything
+ that raises once the run is claimed, building the chain or handing it
+ over, releases the claim first, or the next pull would wait
+ `pull.ABANDONED` for a run that never started.
"""
run = pull.claim_run(store)
if not run:
since = pull.running(store)
context.notice(already_running(since))
return {"skipped": pull.ALREADY_RUNNING, "running": since}
- if not context.task(pull_chain(run)):
+ try:
+ started = context.task(pull_chain(run))
+ except Exception:
+ pull.release_run(store, run)
+ raise
+ if not started:
pull.release_run(store, run)
raise RuntimeError("no queue to start the pull cascade on")
return {"started": " > ".join(CASCADES["pull"]), "run": run}
diff --git a/app/jobs/pull.py b/app/jobs/pull.py
index 09368d2..cec1eea 100644
--- a/app/jobs/pull.py
+++ b/app/jobs/pull.py
@@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see
`plugins/readers/_runner.py`).
A board read with a filter (`ats:filtered:`, from the agents' reach,
-`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
-person acted on: those with a vote or an application (`_close_unlisted`). A
-filtered read, or one of a reader that declares `empty="suspect"`, that lists
-under a tenth of what the same read listed last time is recorded by the
-runner as a failure, not as a list to close against.
+`app.jobs.reach`, or its subscription's `params`) lists only the roles in
+reach, so its close keeps open the roles a person acted on: those with a vote
+or an application (`_close_unlisted`). When the reach wanted remote roles and
+the reader could not ask for them (`ats:remote_unasked:`), the read
+says nothing of the remote roles either, so they stay open too and close only
+once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
+that lists under a tenth of what the same read listed last time is recorded
+by the runner as a failure, not as a list to close against (`_runner.read`).
A board swept across pulls (`ats:swept:`, a complete read that finished
a sweep an earlier pull began) closes a role only when two finished sweeps in a
@@ -33,14 +36,21 @@ not swept clears them.
A read narrowed by the agents' fetch terms (`ats:narrowed:`) lists
the roles matching a word, which says nothing of the roles it did not list,
-so it closes nothing; only an unfiltered or a place-filtered complete read
-closes.
+so it closes by list nothing: a role of such a board closes once the board
+has not listed it for `STALE_DAYS`, the rule searches have.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
-roles only once no search has seen them for `STALE_DAYS`. A board
-that has failed on every fetch for `STALE_DAYS` (`ats:failing:`)
-closes all of its roles, and its next answer reopens what it lists.
+roles only once no search has seen them for `STALE_DAYS`. A board or a
+search that has failed on every fetch for `STALE_DAYS` (`ats:failing:`,
+`search:failing:`) closes all of its roles, and its next answer
+reopens what it lists. A target that raises in its batch after its read, a
+row that does not store, is recorded as failing the same way and the batch
+goes on with the next one (`run_batch`).
+
+The settle says what needs a look in one notice (`settled`): the targets
+that did not answer or were read incomplete, and the boards that closed more
+than `CLOSE_NOTICE` roles or more than half of theirs.
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
set only while the previous run is settled, in one statement), so the
@@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta
from functools import partial
from typing import Any
-from pipeline import NULL, RunContext, Tally, Task, job
+from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
from app.config import Config
from app.jobs.reach import reaches
@@ -91,6 +101,19 @@ STALE_DAYS = 21
#: `ats:failing:` -> `{"since", "reason"}`, removed on its next answer.
FAILING = "ats:failing:"
+#: The same for a search, kept by the `search` plugin.
+SEARCH_FAILING = "search:failing:"
+
+#: A board that closes more roles than this in one settle is named in the
+#: settle's notice, and so is one that closes more than half of its open
+#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
+#: one is an ordinary day.
+CLOSE_NOTICE = 50
+CLOSE_SHARE_MIN = 10
+
+#: How many targets a notice names in its text; its detail names them all.
+NOTICE_NAMES = 10
+
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
@@ -142,27 +165,38 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
`close_boards` compares against. `run` is a marker the caller claimed
already (the `pull` job, `app job pull`); without one the finder claims
its own, and queues nothing when another run is in flight. A run with
- nothing to read is settled here, since no settle will see a batch of it."""
- seed_searches(config, store)
- subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
- targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
- if not any(found for _, found in targets):
- if run:
- release_run(store, run)
- return []
- run = run or claim_run(store)
- if not run:
- log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
- running(store))
- return []
- # The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
- reach_of = reaches(store, subs["ats"])
- batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
- for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
- for batch in batches:
- if batch["source"] == "ats":
- batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
- return batches
+ nothing to read is settled here, since no settle will see a batch of it,
+ and so is a run whose finder raises: no batch and no settle would follow,
+ and the next pull would wait `ABANDONED` for a claim nobody holds."""
+ held = run
+ try:
+ seed_searches(config, store)
+ subs = {source: store.subscriptions(source=source, fetchable=True)
+ for source in SOURCES}
+ targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
+ if not any(found for _, found in targets):
+ if run:
+ release_run(store, run)
+ return []
+ run = run or claim_run(store)
+ if not run:
+ log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
+ running(store))
+ return []
+ held = run
+ # The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
+ reach_of = reaches(store, subs["ats"])
+ batches: list[dict[str, Any]] = [
+ {"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
+ for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
+ for batch in batches:
+ if batch["source"] == "ats":
+ batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
+ return batches
+ except Exception:
+ if held:
+ release_run(store, held)
+ raise
@job("pull_batch", "Reading boards", resource="http")
@@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs. `reach` is `target ->` the
reach the run computed for a board (`app.jobs.reach`), handed to the
- plugin as its `reach` option.
+ plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
+ which the runner's 10% guard accepts the answer it kept refusing.
+
+ Each target is guarded on its own. A read that fails is the plugin's to
+ record; anything raised after it, a row `upsert_role` refuses or a bug
+ past the read, is recorded here under `:failing:` with
+ its reason, and the batch goes on with the next target. Otherwise one
+ bad row would cost the targets after it in the batch their read on every
+ pull, since batches are stable, and leave no trace on any of them. A
+ Kill (`Cancelled`) still ends the batch.
"""
from plugins.ats import ATSSource
+ from plugins.readers import _runner
from plugins.search import SearchSource
# Fetchable now, not when the pull was queued: a board switched off on the
@@ -200,24 +244,39 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
if sub is None:
continue
context.check()
- options = dict(config.sources.get(source) or {}) | {"run": run}
- plugin: ATSSource | SearchSource
- if source == "search":
- options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
- plugin = SearchSource(options)
- else:
- # A board's params are its filter; a reader's key is kept with
- # the searches' (`sources.search.`).
- options["boards"] = [{"url": target, "label": sub.company or sub.label,
- "params": sub.params}]
- options["keys"] = dict(config.sources.get("search") or {})
- options["reach"] = (reach or {}).get(target)
- plugin = ATSSource(options)
- plugin.state = store.state
- for item in plugin.fetch():
- item.source = source
- store.upsert_role(item, sub)
- stored += 1
+ options = dict(config.sources.get(source) or {}) | {"run": run,
+ "stale_days": STALE_DAYS}
+ items: Any = None
+ try:
+ plugin: ATSSource | SearchSource
+ if source == "search":
+ options["searches"] = [{"url": target, "label": sub.label,
+ "params": sub.params}]
+ plugin = SearchSource(options)
+ else:
+ # A board's params are its filter; a reader's key is kept with
+ # the searches' (`sources.search.`).
+ options["boards"] = [{"url": target, "label": sub.company or sub.label,
+ "params": sub.params}]
+ options["keys"] = dict(config.sources.get("search") or {})
+ options["reach"] = (reach or {}).get(target)
+ plugin = ATSSource(options)
+ plugin.state = store.state
+ items = plugin.fetch()
+ for item in items:
+ item.source = source
+ store.upsert_role(item, sub)
+ stored += 1
+ except Cancelled:
+ raise
+ except Exception as exc:
+ log.warning("pull: %s failed after its read (%s); on to the next target",
+ target, exc, exc_info=True)
+ _runner.record_failure(store.state, f"{source}:failing:{target}", exc)
+ finally:
+ # The plugin's generator holds its HTTP client until it is closed.
+ if items is not None and hasattr(items, "close"):
+ items.close()
return {"roles": stored, "run": run}
@@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str:
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
- stopped listing a role from, and tally what happened.
+ stopped listing a role from, tally what happened, and say what needs a
+ look.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
@@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`ats:ok:` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only
- when that list was complete, and otherwise by `STALE_DAYS`. A board that
- has failed on every fetch since `STALE_DAYS` ago closes all its roles.
+ when that list was complete, and otherwise by `STALE_DAYS`. A board or a
+ search that has failed on every fetch since `STALE_DAYS` ago closes all
+ its roles (`failing_boards`). How a read that answered closes is
+ `_close_read`'s.
The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have
replaced. A settle with no batch behind it (a refused pull, nothing to
- read) closes nothing by list or by age. Last, the run is marked settled,
- which lets the next pull start.
+ read) closes nothing by list or by age and says nothing. Last, the run is
+ marked settled, which lets the next pull start.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
- switched-off one is closed again here.
+ switched-off one is closed again here; those closes are asked for and not
+ named in the notice.
+
+ The counts go into the result (`unanswered` counts boards and searches
+ whose stamp is not this run's), and one notice (`settled`) names every
+ target unanswered, every board read incomplete other than one mid-sweep,
+ and every board that closed more than `CLOSE_NOTICE` roles or more than
+ half the open roles it had, when there is any. The result alone was
+ seen by nobody: a settle closed 12,588 roles with nothing on screen.
The `prep` stage that follows maps what this run stored and folds the
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
and the roles this run added join theirs.
"""
- answered = searched = failed = incomplete = closed = 0
+ answered = searched = incomplete = closed = 0
+ unanswered: list[str] = []
+ partial: list[str] = []
+ many: list[tuple[str, int, int]] = []
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store)
run_marker = _run_of(tally, context)
+ open_before = _open_counts(store) if run_marker else {}
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
@@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
- failed += 1
- since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
- if since and since < stale:
- closed += store.close_subscription(sub.id)
- continue
- if source == "search":
- searched += 1
+ unanswered.append(sub.target)
+ since = failing.get(sub.target, ("", ""))[0]
+ shut = store.close_subscription(sub.id) if since and since < stale else 0
else:
- answered += 1
- if not store.state.get(f"{source}:complete:{sub.target}", ""):
- if source == "ats":
- incomplete += 1
- continue
- if store.state.get(f"{source}:narrowed:{sub.target}", ""):
- continue
- seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
- is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
- if store.state.get(f"{source}:swept:{sub.target}", ""):
- closed += _close_swept(store, source, sub, seen_ids, is_filtered)
- continue
- store.state.pop(f"{source}:missing:{sub.target}", None)
- if is_filtered:
- closed += _close_unlisted(store, source, seen_ids, sub.id)
- else:
- closed += store.mark_closed(source, seen_ids, sub.id)
+ if source == "search":
+ searched += 1
+ else:
+ answered += 1
+ read = _close_read(store, source, sub, stale)
+ if read is None:
+ if source == "ats":
+ incomplete += 1
+ if not store.state.get(f"ats:sweep:{sub.target}"):
+ partial.append(sub.target)
+ continue
+ shut = read
+ closed += shut
+ before = open_before.get(sub.id, 0)
+ if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
+ many.append((sub.target, shut, before))
if searched:
closed += store.close_unseen("search", stale)
if run_marker:
+ if unanswered or partial or many:
+ context.notice(settled(unanswered, partial, many, failing,
+ boards=answered, searches=searched, closed=closed))
release_run(store, run_marker)
roles = sum(int(result.get("roles", 0)) for result in tally.results)
- return {"boards": answered, "searches": searched, "failed": failed,
+ return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
"incomplete": incomplete, "roles": roles, "closed": closed}
-def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
+def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
+ """Close what one target's read of this run says has closed, and return
+ how many; None when the read was incomplete, which closes nothing.
+
+ A read narrowed by terms closes only the roles it has not listed for
+ `STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
+ two sweeps in a row missed (`_close_swept`); any other complete read
+ clears that record and closes what it did not list, all of it unfiltered
+ (`Store.mark_closed`), filtered sparing the roles acted on
+ (`_close_unlisted`). A filtered read that could not ask for remote roles
+ (`remote_unasked`) spares the remote ones as well, and closes those by
+ age instead."""
+ state, target = store.state, sub.target
+ if not state.get(f"{source}:complete:{target}", ""):
+ return None
+ seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
+ if state.get(f"{source}:narrowed:{target}", ""):
+ return _close_aged(store, source, sub.id, seen_ids, stale)
+ is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
+ spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
+ aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
+ if spare_remote else 0)
+ if state.get(f"{source}:swept:{target}", ""):
+ return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
+ spare_remote=spare_remote)
+ state.pop(f"{source}:missing:{target}", None)
+ if is_filtered:
+ return aged + _close_unlisted(store, source, seen_ids, sub.id,
+ spare_remote=spare_remote)
+ return store.mark_closed(source, seen_ids, sub.id)
+
+
+def _open_counts(store: Store) -> dict[int, int]:
+ """`subscription id -> open roles`, before the settle closes any, for the
+ share a board closed."""
+ with store.transaction() as conn:
+ return {int(row[0]): int(row[1]) for row in conn.execute(
+ """SELECT subscription_id, count(*) FROM roles
+ WHERE closed_at IS NULL AND subscription_id IS NOT NULL
+ GROUP BY subscription_id""")}
+
+
+def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
+ failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
+ closed: int) -> Notice:
+ """The notice of a settle that has something to look at: the targets
+ that did not answer (with the reason their failing record gives), the
+ boards read incomplete, and the boards that closed many roles, each
+ as `closed of open`, with the totals. The text names the first
+ `NOTICE_NAMES` of each, the detail all of them, one a line."""
+ def named(target: str) -> str:
+ reason = failing.get(target, ("", ""))[1]
+ return f"{target} ({reason})" if reason else target
+
+ groups = [("Not answering", [named(t) for t in unanswered]),
+ ("Read incomplete", list(incomplete)),
+ ("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
+ text = [f"The pull read {boards} boards and {searches} searches and closed "
+ f"{closed:,} roles."]
+ detail: list[str] = []
+ for title, names in groups:
+ if not names:
+ continue
+ shown = ", ".join(names[:NOTICE_NAMES])
+ more = len(names) - NOTICE_NAMES
+ text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
+ else "."))
+ detail += [f"{title}: {name}" for name in names]
+ return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
+ text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
+
+
+#: The roles a partial read never closes by its list: one with a vote (any,
+#: neutral included) or an application (which holds the notes).
+_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
+ AND uid NOT IN (SELECT role_uid FROM applications
+ WHERE role_uid IS NOT NULL)"""
+
+#: The remote roles a filtered read that could not ask for them leaves alone.
+_NOT_REMOTE = " AND remote != 'remote'"
+
+
+def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
+ *, spare_remote: bool = False) -> int:
"""`Store.mark_closed` for a filtered read: close every open role of the
- board it did not list, except a role someone acted on, one with a vote
- (any, neutral included) or an application (which holds the notes). A
+ board it did not list, except a role someone acted on (`_ACTED_ON`). A
filtered read cannot tell a role gone from a role outside its filter, and
a role acted on stays until the board is read without the filter or
- fails for `STALE_DAYS`."""
+ fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
+ well, since the read could not ask for them."""
with store.transaction() as conn:
return conn.execute(
- """UPDATE roles SET closed_at = ?
- WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
- AND source_id NOT IN (SELECT value FROM json_each(?))
- AND uid NOT IN (SELECT uid FROM role_votes)
- AND uid NOT IN (SELECT role_uid FROM applications
- WHERE role_uid IS NOT NULL)""",
+ f"""UPDATE roles SET closed_at = ?
+ WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
+ AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
+ + (_NOT_REMOTE if spare_remote else ""),
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
+def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
+ stale: str, *, remote_only: bool = False) -> int:
+ """Close the open roles of one board that its read did not list and that
+ no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
+ writes on every listing): the rule for what a read cannot speak for. A
+ read narrowed by terms closes all such roles, as a search does; with
+ `remote_only`, a filtered read that could not ask for remote closes only
+ its remote roles so, sparing those acted on as its list does."""
+ where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
+ with store.transaction() as conn:
+ return conn.execute(
+ f"""UPDATE roles SET closed_at = ?
+ WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
+ AND last_seen < ?
+ AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
+ (store.now(), source, subscription_id, stale,
+ json.dumps(list(seen_ids)))).rowcount
+
+
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
- filtered: bool) -> int:
+ filtered: bool, *, spare_remote: bool = False) -> int:
"""The close of a board whose complete read finished a sweep across
pulls: its open roles missing from `seen_ids` that also are in
`:missing:`, the ones the previous finished sweep missed.
@@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
missed may only have shifted back past the cursor when roles closed
between pulls, so one sweep is not evidence that it is gone. A filtered
read keeps its exceptions as `_close_unlisted` does: a role acted on
- stays open, and stays in `missing`."""
+ stays open, and stays in `missing`, and so does a remote role with
+ `spare_remote`."""
key = f"{source}:missing:{sub.target}"
try:
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
except (TypeError, ValueError):
earlier = set()
- acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
- AND uid NOT IN (SELECT role_uid FROM applications
- WHERE role_uid IS NOT NULL)""" if filtered else ""
+ acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
with store.transaction() as conn:
missed = [row[0] for row in conn.execute(
"""SELECT source_id FROM roles
@@ -365,16 +536,20 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
- """`target -> (since, reason)` for every `ats` board whose last fetch
- failed: when the failures began, and what the latest one said."""
+ """`target -> (since, reason)` for every `ats` board and every search
+ whose last fetch failed: when the failures began, and what the latest
+ one said. No board host is also a search provider, so a board's target
+ (`host:slug`) and a search's (`provider:name`) never collide in one
+ dict."""
out: dict[str, tuple[str, str]] = {}
- for key, value in store.state_items(FAILING).items():
- try:
- record = json.loads(value)
- except ValueError:
- continue
- out[key[len(FAILING):]] = (str(record.get("since") or ""),
- str(record.get("reason") or ""))
+ for prefix in (FAILING, SEARCH_FAILING):
+ for key, value in store.state_items(prefix).items():
+ try:
+ record = json.loads(value)
+ except ValueError:
+ continue
+ out[key[len(prefix):]] = (str(record.get("since") or ""),
+ str(record.get("reason") or ""))
return out
diff --git a/tests/test_pull.py b/tests/test_pull.py
index 6a94718..1ee5e8f 100644
--- a/tests/test_pull.py
+++ b/tests/test_pull.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import json
import sys
import threading
+import types
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -21,8 +22,8 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext, Tally # noqa: E402
from app.models import Application # noqa: E402
-from plugins import readers # noqa: E402
-from plugins.readers import _runner # noqa: E402
+from plugins import ats, readers # noqa: E402
+from plugins.readers import Filters, Reads, _runner # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app import prep # noqa: E402
from app.jobs import cascade, pull # noqa: E402
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
- assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
+ assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0,
"roles": 1, "closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -163,7 +164,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
- assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
+ assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0,
"roles": 0, "closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
@@ -256,7 +257,7 @@ def test_a_failing_board_keeps_its_roles_before_stale_days(
_answer(monkeypatch, ValueError("gone"))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1"}
- assert (result["failed"], result["closed"]) == (1, 0)
+ assert (result["unanswered"], result["closed"]) == (1, 0)
since, reason = pull.failing_boards(acme)["workable:acme"]
assert since and reason == "ValueError: gone"
@@ -486,7 +487,7 @@ def test_run_a_settle_does_not_close_on_run_b_ids(
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
- assert (result["failed"], result["closed"]) == (1, 0)
+ assert (result["unanswered"], result["closed"]) == (1, 0)
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
assert pull.running(acme) == b["run"]
@@ -507,3 +508,227 @@ def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> No
context = RunContext(waited_for=(failed,))
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
assert result["boards"] == 1 and pull.running(acme) == ""
+
+
+# -------------------------------------------------------- losses, guarded
+
+def _days_ago(days: float) -> str:
+ return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds")
+
+
+def _set(store: Store, column: str, value: str, *source_ids: str) -> None:
+ with store.transaction() as conn:
+ conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?",
+ [(value, i) for i in source_ids])
+
+
+@pytest.fixture
+def big(store: Store) -> Store:
+ store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
+ board="greenhouse")
+ return store
+
+
+def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails(
+ big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ _answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
+ _pull(big)
+ _answer(monkeypatch, ([], True))
+ result = _pull(big)
+ assert len(_open(big)) == 40
+ assert (result["unanswered"], result["closed"]) == (1, 0)
+ assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1]
+
+
+def test_a_whole_read_of_nothing_after_five_closes_the_board(
+ big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ _answer(monkeypatch, ([_row(str(n)) for n in range(5)], True))
+ _pull(big)
+ _answer(monkeypatch, ([], True))
+ assert _pull(big)["closed"] == 5
+ assert _open(big) == set()
+
+
+def test_a_suspect_answer_is_accepted_after_stale_days(
+ big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ _answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
+ _pull(big)
+ _answer(monkeypatch, ([_row("1")], True))
+ _pull(big)
+ assert len(_open(big)) == 40
+ record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"])
+ big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps(
+ record | {"since": _days_ago(pull.STALE_DAYS + 1)})
+ result = _pull(big)
+ assert _open(big) == {"greenhouse:big:1"}
+ assert (result["unanswered"], result["closed"]) == (0, 39)
+ assert pull.failing_boards(big) == {}
+ assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1
+
+
+def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]:
+ """A whole-listing host `fake` that filters by country and, per `remote`,
+ on remote; its feed lists the ids in the returned list."""
+ listed: list[str] = []
+ module: object = types.ModuleType("fake")
+ module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined]
+ module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined]
+ module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined]
+ module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
+ module.READS = Reads( # type: ignore[attr-defined]
+ scope="board", listing="whole", text="listing", complete="whole", places="one",
+ robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type]
+ module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
+ module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined]
+ monkeypatch.setitem(readers.HOSTS, "fake", module)
+ monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps(
+ listed).encode())
+ return listed
+
+
+def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale(
+ store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ listed = _remote_host(monkeypatch, remote="none")
+ store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake")
+ reach = {"fake:acme": {"countries": ["DK"], "remote": True}}
+
+ def pull_once() -> dict[str, object]:
+ run = pull.claim_run(store)
+ result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type]
+ reach=reach)
+ return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
+
+ listed[:] = ["1", "2", "3", "4"]
+ pull_once()
+ _set(store, "remote", "remote", "fake:acme:3", "fake:acme:4")
+ listed[:] = ["1"]
+ assert pull_once()["closed"] == 1
+ assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"}
+ assert store.state["ats:remote_unasked:fake:acme"] == "1"
+
+ _set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4")
+ assert pull_once()["closed"] == 1
+ assert _open(store) == {"fake:acme:1", "fake:acme:3"}
+
+
+def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days(
+ acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ _answer(monkeypatch, ([_row(i) for i in "123"], True))
+ _pull(acme)
+
+ def fake(*args: object, **kwargs: object) -> _runner.Listing:
+ return _runner.Listing([_row("1")], True, ["1"], narrowed=True)
+
+ monkeypatch.setattr(_runner, "read_listing", fake)
+ assert _pull(acme)["closed"] == 0
+ _set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3")
+ assert _pull(acme)["closed"] == 1
+ assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
+
+
+def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on(
+ store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")]
+ for target in targets:
+ slug = target.partition(":")[2]
+ store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse")
+
+ def fake_fetch(self: ATSSource) -> object:
+ [(target, _)] = self.targets()
+ yield _item("greenhouse", target.partition(":")[2], "1")
+ self.state[f"ats:ok:{target}"] = self.options["run"]
+
+ real = Store.upsert_role
+
+ def upsert(self: Store, item: Item, sub: object = None) -> bool:
+ if item.source_id.startswith("greenhouse:b:"):
+ raise ValueError("a row the store refuses")
+ return real(self, item, sub) # type: ignore[arg-type]
+
+ monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
+ monkeypatch.setattr(Store, "upsert_role", upsert)
+ result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type]
+
+ assert result["roles"] == 2
+ assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"}
+ assert "ats:ok:greenhouse:b" not in store.state
+ since, reason = pull.failing_boards(store)["greenhouse:b"]
+ assert since and reason == "ValueError: a row the store refuses"
+ assert set(pull.failing_boards(store)) == {"greenhouse:b"}
+
+
+def test_a_board_whose_rows_do_not_store_keeps_its_failing_record(
+ acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ since = _days_ago(2)
+ acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
+ {"since": since, "reason": "404 Not Found"})
+ _answer(monkeypatch, ([_row("1")], True))
+
+ def refuse(self: Store, item: Item, sub: object = None) -> bool:
+ raise ValueError("no")
+
+ monkeypatch.setattr(Store, "upsert_role", refuse)
+ _pull(acme)
+ assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no")
+ assert "ats:ok:workable:acme" not in acme.state
+
+
+# ------------------------------------------------------- the settle speaks
+
+def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many(
+ store: Store) -> None:
+ sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
+ board="greenhouse")
+ store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone",
+ board="greenhouse")
+ for n in range(80):
+ store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub)
+ run = pull.claim_run(store)
+ store.state["ats:ok:greenhouse:big"] = run
+ store.state["ats:complete:greenhouse:big"] = "1"
+ store.state["ats:ids:greenhouse:big"] = json.dumps(
+ [f"greenhouse:big:{n}" for n in range(20)])
+ store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps(
+ {"since": _days_ago(1), "reason": "404 Not Found"})
+
+ said: list[object] = []
+ result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type]
+ context=RunContext(on_notice=said.append))
+ assert (result["unanswered"], result["closed"]) == (1, 60)
+ [notice] = said
+ assert notice.code == "pull.settled" # type: ignore[attr-defined]
+ assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined]
+ assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined]
+ assert "closed 60 roles" in notice.text # type: ignore[attr-defined]
+
+
+def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ _answer(monkeypatch, ([_row(i) for i in "12"], True))
+ said: list[object] = []
+ results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
+ for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type]
+ pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type]
+ context=RunContext(on_notice=said.append))
+ assert said == []
+
+
+def test_a_finder_that_raises_releases_the_pull_claim(
+ acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ def broken(store: Store, subs: object) -> dict[str, object]:
+ raise RuntimeError("reach broke")
+
+ monkeypatch.setattr(pull, "reaches", broken)
+ # The worker runs the chain's finder once the job is done; here at once.
+ context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme))
+ with pytest.raises(RuntimeError, match="reach broke"):
+ cascade.run_pull(FakeConfig(), acme, context=context)
+ assert pull.running(acme) == ""
+
+ # And the job itself lets go when the chain cannot be built.
+ def no_chain(run: str) -> object:
+ raise RuntimeError("no chain")
+
+ monkeypatch.setattr(cascade, "pull_chain", no_chain)
+ with pytest.raises(RuntimeError, match="no chain"):
+ cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True))
+ assert pull.running(acme) == ""
diff --git a/tests/test_search.py b/tests/test_search.py
index dc34a6a..0cf9d9e 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -371,3 +371,56 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
assert store.roles() == []
# And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
+
+
+# ------------------------------------------------- failures and the guard
+
+def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The provider `fake` of `_fake_provider`, answering with an error."""
+ _fake_provider(monkeypatch, [], complete=True)
+
+ def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
+ raise ValueError("hits renamed")
+
+ readers.PROVIDERS["fake"].page = page
+
+
+def test_a_search_whose_provider_raises_is_recorded_failing(
+ store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
+ params={"q": "backend"})
+ _fake_provider(monkeypatch, ["h1"], complete=True)
+ _pull(store)
+ _broken_provider(monkeypatch)
+ result = _pull(store)
+ assert result["unanswered"] == 1
+ since, reason = pull.failing_boards(store)["fake:cph"]
+ assert since and reason == "ValueError: hits renamed"
+ assert {r.source_id for r in store.roles()} == {"fake:h1"}
+
+ # Failing for `STALE_DAYS` closes its roles, as a board's.
+ old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
+ timespec="seconds")
+ store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason})
+ _pull(store)
+ assert store.roles() == []
+
+ # Its next answer removes the record and reopens what it lists.
+ _fake_provider(monkeypatch, ["h1"], complete=True)
+ _pull(store)
+ assert pull.failing_boards(store) == {}
+ assert {r.source_id for r in store.roles()} == {"fake:h1"}
+
+
+def test_a_search_that_lists_nothing_after_forty_is_suspect(
+ store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
+ params={"q": "backend"})
+ forty = [f"h{n}" for n in range(40)]
+ _fake_provider(monkeypatch, forty, complete=True)
+ _pull(store)
+ _fake_provider(monkeypatch, [], complete=True)
+ result = _pull(store)
+ assert (result["unanswered"], result["closed"]) == (1, 0)
+ assert len(store.roles()) == 40
+ assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]
From 2cc83b1d84bcb1c5107d0c2811189879bdefd403 Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 15:54:13 +0200
Subject: [PATCH 8/9] Correct the contract docstring: detail is read by enrich,
the example is Workday's, the touch points of a new reader
Adds the search providers' PARAMS, clean_params and company, and a test
that the example READS is workday.py's.
Co-Authored-By: Claude Fable 5.1
---
plugins/readers/__init__.py | 150 +++++++++++++++++++++------------
tests/test_readers_contract.py | 10 +++
2 files changed, 105 insertions(+), 55 deletions(-)
diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py
index 5e873fe..6134d17 100644
--- a/plugins/readers/__init__.py
+++ b/plugins/readers/__init__.py
@@ -3,8 +3,8 @@
A reader registers by existing: every module in this package whose name does
not start with `_` is loaded when the package is imported, checked against
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
-`HOSTS` holds those of `scope="board"` (and every module without `READS`),
-`PROVIDERS` those of `scope="search"`. A module that breaks the contract
+`HOSTS` holds those of `scope="board"`, `PROVIDERS` those of
+`scope="search"`. A module that breaks the contract, or declares no `READS`,
fails the import with an ImportError naming the module and every problem,
rather than a pull that fails board by board. `plugins/ats.py` builds its
allow-list, feeds, pages, the Sources page's host select and its
@@ -14,21 +14,26 @@ since `desk.sources.registry` loads `plugins/*.py` by path and the tests
patch names on the `ats` module. The types are in `._contract`.
A module is named after its source (`plugins/readers/workday.py`) and
-imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It
-holds:
+imports `desk`, `._contract`, `._shared` and `._countries` only (besides the
+standard library, `httpx` and `pydantic`), never `plugins.ats` or another
+reader; `tests/test_architecture.py` holds that. A board's module holds, as
+Workday's does:
NAME = "workday" # the host of a target `workday:`; equals the module's name
LABEL = "Workday" # what the Sources page's select shows
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
RANK = 100 # optional int; lower comes first in the select and the probes
- KEEP_CASE = False # optional; True keeps a discovered slug's case
- ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
+ KEEP_CASE = True # optional; True keeps a discovered slug's case
- READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000,
- text="detail", complete="stated",
- filters=Filters(country="ids"), places="many",
- fields=Fields(posted="listed"),
- robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)")
+ READS = Reads(
+ scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
+ complete="stated", places="many",
+ filters=Filters(country="ids", remote="ids", query="one"),
+ fields=Fields(posted="trusted", remote="stated", employment="stated"),
+ robots=("read: robots.txt allows // and disallows /refreshFacet/ and has no rule "
+ "on /wday/cxs///, the listing's POST .../jobs and each role's "
+ "JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"),
+ )
def slug_ok(slug: str) -> bool # strict, see below
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
@@ -49,24 +54,35 @@ and, optionally:
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
# read once when `feed_url` is over the cap
- def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt)
def probe_url(slug: str) -> str # discovery: 2xx only for a board
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
-A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its
-slug is the provider's own (a search reads no board), and it is not
-discovered. A module without `READS` is read as a board with the older
-signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by
-`undeclared()`; the conformance test marks it until it declares.
+**Search providers.** A module of `scope="search"` holds `NAME`, `LABEL`,
+`allowed`, `READS`, the functions its listing needs, and:
+
+ PARAMS = {"q": "query", "area": "place"} # saved param -> the filter dimension it fills
+ def clean_params(params: dict[str, Any]) -> dict[str, Any]
+
+`app/jobs/sources.search_params` keeps the params of `PARAMS` whose
+dimension `READS.filters` declares and stores what `clean_params` returns;
+`clean_params` raises ValueError, with a reason a person can read, for a
+value the provider would not accept. The runner hands the saved params to
+`page` in `Ask.params`. A search lists many companies, so its rows carry
+`company`, the employer's name, which `plugins/search.py` stores. Its slug
+is the provider's own (a search reads no board), and it has no patterns,
+probes or careers page: it is never discovered.
**Declaration.** `READS` states what the reader does; the runner does it.
`problems()` checks it against the functions: `paged` needs `page` and
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
-`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key`
-names a config entry, `robots` is not empty, and each function takes the
-arguments above. The traits:
+`facets`, `scope="search"` needs `complete="never"` or `"stated"` and
+`PARAMS` naming filter dimensions, a `key` names a config entry, `robots`
+is not empty and carries a date, and each function takes the arguments
+above. `tests/test_readers_contract.py` checks it against the reader's
+fixtures: each declared field shows on some row, each field declared `none`
+on none. The traits:
- `scope`: `board`, one company's whole list; `search`, many companies,
never a whole list, so nothing a search stops listing is closed by it.
@@ -81,11 +97,13 @@ arguments above. The traits:
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
`one` (one value a request), `many` (a list in one request), `ids` (host
ids looked up with `facets`).
-- `places`: `one`, `many`, or `copies` when the host publishes one posting
- per place with its own id.
+- `places`: `one`, `many`, `copies` when the host publishes one posting
+ per place with its own id, or `none` when the host states no place.
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
posting date; `listed`, a date the host moves; `none`), and `remote`,
- `country`, `seniority`, `employment` (`stated` or `none`).
+ `country`, `seniority`, `employment` (`stated` or `none`). A field may
+ come from the listing or from `detail`: BambooHR, Gem and Rippling date a
+ role only on its own page.
- `key`: `Key(name, fields, env)`, the config entry `sources.search.`
and its environment fallbacks. The runner resolves it and passes it in
`Ask.params["key"]`.
@@ -93,7 +111,7 @@ arguments above. The traits:
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
far under the last count counts as failed.
- `title`: `provisional` when the listing may lack the title and `detail`
- fills it.
+ fills it. Nothing reads it yet: `_runner.DETAIL_FIELDS` has no `title`.
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
wants from one series of requests. It sets only dimensions the reader
@@ -124,8 +142,7 @@ against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
paths are case-sensitive (Workday's site). A pattern without a group names
the host with no slug. A host that `discover.py` names in its own
-`_UNREADABLE` table is taken from its module instead as soon as the module
-exists.
+`_UNREADABLE` table loses its entry there when its module is added.
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
name; `_shared.only(host)` is the usual one. Every request goes through
@@ -165,9 +182,8 @@ see `desk.sources.strip_html`), `remote` (True, False, a word or None) and
`salary_text`; where the host states pay, `salary_min`, `salary_max`
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
-`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a
-new field needs no change there. Undeclared modules do not state `places`
-yet.
+`employment`; `teaser`; and, on a search row, `company`. The runner keeps
+every stated key in `raw`, so a new field needs no change there.
**Places.** A role has a list of places: `places` is `[Place(text=...,
remote=...)]`, one entry per place the host states, in the host's order, in
@@ -176,12 +192,14 @@ place and never joins places into one `text`. `remote` is True for a remote
place, False for an office, None when unsaid. A host that publishes one
posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role
-that keeps each copy's link.
+that keeps each copy's link. A host that states no place declares
+`places="none"` and hands over `places == []`.
-**Helpers.** `._shared` holds what more than one host needs. A module maps
-no word to the app's vocabulary: it hands on the host's own words under
-`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them
-through the host's table in `app/prep/hosts/.py`.
+**Helpers.** `._shared` holds what more than one host needs, `._countries`
+the country names hosts write. A module maps no word to the app's
+vocabulary: it hands on the host's own words under `seniority`,
+`employment`, `remote` and `country`, and `app.prep` maps them through the
+host's table in `app/prep/hosts/.py`.
- `place(text, remote=None)` makes a `Place`: remote when the text has the
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
@@ -197,38 +215,60 @@ is shown and embedded until `description` arrives. A reader that sees one
keeps it.
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
-dict of the fields it found. The shared code calls it for roles not yet read
-(`ats:detailed:`), at most `_runner.DETAIL_CAP` (200) a board a
-pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that
-failed (`ats:detailfailed:`). Only `_runner.DETAIL_FIELDS` are taken:
+dict of the fields it found. The pull reads listings only; the `enrich`
+stage (`app/jobs/enrich.py`) calls `detail` after scoring, so the requests
+go where the ranking is: the open roles without text that an agent ranks
+among its best `enrich.top` at `enrich.min_score` or above, best first, at
+most `enrich.per_board` from one board and `enrich.per_run` in all, one
+batch per board through `ats.read_board_details` and
+`_runner.read_details`, `_runner.PAGE_DELAY` apart, the roles never tried
+before those whose read failed. Only `_runner.DETAIL_FIELDS` are taken:
`description`, `location`, `places`, `teaser`, `remote`, `country`,
-`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the
-listing's empty value; any other key is left out with a debug log line.
+`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys
+whose columns survive a later pull that sends the listing's empty value;
+any other key is left out with a debug log line, `seniority` among them.
`title`, `url`, `department` and `salary_text` cannot be filled from
-`detail`: the listing rewrites them every pull. A role not read in detail
-on a pull is sent with an empty description, so the one read earlier stays.
-A 404 or a 410 counts as read; any other failure is logged and retried on a
-later pull; a 429 ends the detail reads of that board for the pull.
-A role read in detail on an earlier pull is sent without the
-listing's place and remote, so what its own page said about them stays.
-Completeness is decided by the listing alone.
+`detail`: the listing rewrites them every pull. A 404 or a 410 counts as read; any
+other failure is logged and retried on a later run; a read that gave no text
+is retried after `enrich.retry_days`; a 429 ends the reads of that board for
+the run. A role read in detail is sent by later pulls without the listing's
+place and remote, so what its own page said about them stays. Completeness
+is decided by the listing alone.
**robots.txt.** Decided when the reader is built, not checked at run time.
-Whoever builds a reader reads the host's robots.txt for the paths the reader
-requests and reports what it disallows, and the owner decides whether the
-host is read anyway. `robots` records that decision with its date, e.g.
-`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is
-disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time;
-`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are
-read by nothing and a new reader defines neither.
+Whoever builds a reader reads the robots.txt of every host the reader
+requests, for the paths it requests (the API's, not the career page a
+person opens), and the owner decides whether the host is read anyway.
+`robots` records what the file says of those paths, the date it was read
+and the decision, e.g. `"read: robots.txt has no rule on /wday/cxs/
+(checked 2026-09-30)"` or `"read though /api/ is disallowed: owner,
+2026-09-30"`; a record without a date fails the contract. Nothing reads
+robots.txt at run time.
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
the slug is a board; `probe_ok(body)` narrows that when a host answers an
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
name is probed as, `(guess,)` without it. A host without `probe_url` is never
probed, only found by a link.
-"""
+**Adding a reader.** A new host or provider touches:
+
+- its module here;
+- `app/prep/hosts/.py` when it states seniority or employment in its
+ own words (the contract test requires the table; a `REMOTE` table when
+ its remote words are not the app's);
+- its fixtures under `tests/fixtures/ats/` (`tests/fixtures/search/` for a
+ provider), named as `tests/test_readers_contract.py` describes, and
+ `tests/test_ats_.py` and `tests/test_prep_hosts_.py`;
+- its entry in `discover._UNREADABLE`, deleted, where discovery named the
+ host before;
+- a section in `plugins/README.md`, and a row in `docs/filter-survey.md`
+ when it filters at the source.
+
+Nothing else: `HOSTS` and `PROVIDERS`, the Sources page's select,
+discovery's patterns and probes, and the reach a board is read with are
+derived from the module.
+"""
from __future__ import annotations
import importlib
diff --git a/tests/test_readers_contract.py b/tests/test_readers_contract.py
index 859b060..4d739e6 100644
--- a/tests/test_readers_contract.py
+++ b/tests/test_readers_contract.py
@@ -32,6 +32,7 @@ already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels).
from __future__ import annotations
import sys
+import textwrap
import types
from collections.abc import Callable
from pathlib import Path
@@ -535,6 +536,15 @@ def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
assert place_problems(rows, "many") == []
+def test_the_contracts_example_is_workdays_declaration() -> None:
+ doc = readers.__doc__ or ""
+ start = doc.index(" READS = Reads(")
+ source = textwrap.dedent(doc[start:doc.index("\n )\n", start) + 6])
+ namespace: dict[str, Any] = {"Reads": Reads, "Filters": Filters, "Fields": Fields}
+ exec(source, namespace)
+ assert namespace["READS"] == readers.READERS["workday"].READS
+
+
def test_the_contract_types_are_what_the_package_exports() -> None:
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
From a221625eff96eb62e5174d25107ec7582b592ace Mon Sep 17 00:00:00 2001
From: Wolf
Date: Wed, 30 Sep 2026 16:01:30 +0200
Subject: [PATCH 9/9] Give the test readers a READS, a dated robots record and
the search params the contract now requires
Co-Authored-By: Claude Fable 5.1
---
tests/test_readers.py | 5 ++++-
tests/test_search.py | 4 +++-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/tests/test_readers.py b/tests/test_readers.py
index 49251f6..8a0b89c 100644
--- a/tests/test_readers.py
+++ b/tests/test_readers.py
@@ -64,8 +64,11 @@ PATTERNS = (r"good\\.example/([a-z]+)",)
def slug_ok(slug): return True
def allowed(slug): return lambda h: h == "good.example"
def careers_url(slug): return "https://good.example/" + slug
-def feed_url(slug): return "https://good.example/" + slug + ".json"
+def feed_url(slug, ask): return "https://good.example/" + slug + ".json"
def parse(body, slug): return []
+from plugins.readers import Reads
+READS = Reads(scope="board", listing="whole", text="listing", complete="whole", places="one",
+ robots="read (checked 2026-09-30)")
"""
diff --git a/tests/test_search.py b/tests/test_search.py
index 0cf9d9e..d9c13c0 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -264,8 +264,10 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL = "fake", "Fake"
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
- places="one", fields=Fields(country="stated"), robots="test")
+ places="one", fields=Fields(country="stated"), robots="test (checked 2026-09-30)")
module.allowed = lambda slug: lambda host: host == "search.fake.example"
+ module.PARAMS = {"q": "query"}
+ module.clean_params = lambda params: dict(params)
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
assert ask.params["q"]