Merge branch 'wp-b-runner'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:29:15 +02:00
commit 650ae3f13a
14 files changed
+1608 -1492

No files matched your search

+37 -3
View File
@@ -13,7 +13,16 @@ on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
still leaves the boards it had already finished stamped, so one bad board in a
batch of ten does not cost the other nine their closure. A board that answered
but was not read to its stated total (a paged host that hit a 429 or the page
cap) stores what it read and closes nothing.
cap) stores what it read and closes nothing; a paged board over the page cap
is swept across pulls and closes once the sweep reached its end (see
`plugins/readers/_runner.py`).
A board read with a filter (`ats:filtered:<target>`, from its subscription's
`params`) lists only the roles in reach, so its close keeps open the roles a
person acted on: those with a vote or an application (`_close_unlisted`). A
filtered read, or one of a reader that declares `empty="suspect"`, that lists
under a tenth of what the same read listed last time is recorded by the
runner as a failure, not as a list to close against.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
@@ -175,7 +184,11 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
plugin = SearchSource(options)
else:
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
# A board's params are its filter; a reader's key is kept with
# the searches' (`sources.search.<name>`).
options["boards"] = [{"url": target, "label": sub.company or sub.label,
"params": sub.params}]
options["keys"] = dict(config.sources.get("search") or {})
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
@@ -252,7 +265,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
incomplete += 1
continue
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
closed += store.mark_closed(source, seen_ids, sub.id)
if store.state.get(f"{source}:filtered:{sub.target}", ""):
closed += _close_unlisted(store, source, seen_ids, sub.id)
else:
closed += store.mark_closed(source, seen_ids, sub.id)
if searched:
closed += store.close_unseen("search", stale)
if run_marker:
@@ -263,6 +279,24 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
"""`Store.mark_closed` for a filtered read: close every open role of the
board it did not list, except a role someone acted on, one with a vote
(any, neutral included) or an application (which holds the notes). A
filtered read cannot tell a role gone from a role outside its filter, and
a role acted on stays until the board is read without the filter or
fails for `STALE_DAYS`."""
with store.transaction() as conn:
return conn.execute(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?))
AND uid NOT IN (SELECT uid FROM role_votes)
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)""",
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
"""`target -> (since, reason)` for every `ats` board whose last fetch
failed: when the failures began, and what the latest one said."""
+4 -4
View File
@@ -28,7 +28,7 @@ functions are the classifier's vocabulary and the hosts are what it parses,
and this reaches `plugins/` the same way `app.cli` does, since a plugin is
app-agnostic and never reaches back into `app`. The
Jobindex areas, Adzuna countries and Platsbanken regions on the Sources page
come from `plugins.search` for the same reason.
come from the search readers in `plugins/readers/` for the same reason.
"""
from __future__ import annotations
@@ -49,9 +49,9 @@ if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
from plugins.ats import FUNCTIONS, HOSTS # noqa: E402
from plugins.search import ( # noqa: E402
ADZUNA_COUNTRIES, JOBINDEX_AREAS, PLATSBANKEN_REGIONS,
)
from plugins.readers.adzuna import COUNTRIES as ADZUNA_COUNTRIES # noqa: E402
from plugins.readers.jobindex import AREAS as JOBINDEX_AREAS # noqa: E402
from plugins.readers.platsbanken import REGIONS as PLATSBANKEN_REGIONS # noqa: E402
TEMPLATE_DIR = Path(__file__).resolve().parent.parent / "templates"
STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
+131 -589
View File
@@ -2,50 +2,44 @@
`plugins/readers/`, one module per host.
Every one of these hosts serves a company's own list of roles as public,
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
drops off the board is *closed*, which is the cheapest staleness check there
is. What a host says and how its feed reads is its module's; this file holds
what every host shares: the fetch and its allow-list, paging and
completeness, the per-role detail reads, robots.txt, the classifier, and the
plugin itself.
unauthenticated JSON, XML or RSS, and a role that drops off the board is
*closed*, which is the cheapest staleness check there is. What a host says
and how its feed reads is its module's, what every reader shares is
`plugins/readers/_runner.py`'s; this file is the `ats` Source over the runner,
the classifier, and the names the app and the tests reach for on `ats`.
The allow-list and the classifier are ported from `openings.go` and
`cmd/seedgen/main.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
`classify`'s nine functions and their keywords. The body cap is not a
straight port — see `MAX_BYTES` below.
`classify`'s nine functions and their keywords.
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is fetched independently; one that fails or times
out is logged, recorded as failing under `ats:failing:<target>` (see
`ATSSource.fetch`), and skipped, and the rest continue.
its slug is. Every board is read independently; one that fails is logged,
recorded as failing under `ats:failing:<target>`, and skipped. A board's
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
filter, narrowed to what its reader declares (`_runner.ask_for`).
A host that pages states a total or does not. A board with a total counts as
complete only when it read as many distinct roles as the total says, one
without when it read up to an empty page (see `_paged`); a host that answers
with the whole board at once is complete whenever it parses.
The limits below are read when a board is read, so a test that replaces one
on this module reaches the runner.
"""
from __future__ import annotations
import ipaddress
import json
import logging
import re
import sys
import time
from collections.abc import Callable, Iterator, MutableMapping
from datetime import UTC, datetime, timedelta
from typing import Any
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from pathlib import Path
from urllib.parse import unquote
from typing import Any
import httpx
from desk.sources import (
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, # noqa: F401
)
try:
@@ -58,6 +52,7 @@ except ModuleNotFoundError:
# file. So the root goes on the path then, and only then.
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import readers
from plugins.readers import _runner
from plugins.readers._shared import Get, Row
# The names the parsers had here before they moved to their host modules,
@@ -70,156 +65,57 @@ from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
log = logging.getLogger(__name__)
#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every
#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's
#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers —
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
#: at its host's `fallback_url` where it has one (Greenhouse, without
#: `?content=true`, see `_fetch_rows`), so at least the titles land.
MAX_BYTES = 96 << 20
#: The runner's limits, kept here as the `ats` Source's own: see `_runner`.
MAX_BYTES = _runner.MAX_BYTES
TIMEOUTS = _runner.TIMEOUTS
DEADLINE = _runner.DEADLINE
MAX_PAGES = _runner.MAX_PAGES
PAGE_DELAY = _runner.PAGE_DELAY
DETAIL_CAP = _runner.DETAIL_CAP
DETAIL_FIELDS = _runner.DETAIL_FIELDS
#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai`
#: failed a pull of 2026-09-28 with a read timeout. Measured live on
#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse
#: boards with the most open roles (databricks, stripe, anthropic, feverup,
#: datadog), with and without `pay_transparency=true`, cached and with a
#: cache-busting parameter (84 requests; CloudFront answered every one
#: "Miss", so all reached the origin): the body arrives in under 1 s once it
#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the
#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic,
#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise.
#: The flag made no difference that three runs could tell apart from that
#: noise. So a read may wait 30 s, about twice the worst first byte seen,
#: and a whole request, redirects and body included, 60 s (see `_get`),
#: about four times the slowest seen, which still leaves a 96 MB body room
#: on a link far slower than the one measured.
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
DEADLINE = 60.0
#: The runner's errors, under the names they had here.
_Refused = _runner.Refused
_TooLarge = _runner.TooLarge
_local_host = _runner.local_host
#: host -> the name the host gives itself: the hosts this plugin reads, in
#: the order a select offers them. What the Sources page's host select is
#: rendered from, so it cannot offer a host `is_known_target` refuses.
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
#: host -> parser, for the hosts whose module parses a whole body.
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
def _limits() -> _runner.Limits:
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
"""The hosts a request or a redirect may land on for one board: what its
host module's `allowed` says, which is one fixed API host or the
company's own site. Anything else is refused, at the first hop and at
every redirect: a feed is external content, and a board row must never
be able to point this fetch at an internal address."""
host module's `allowed` says. Anything else is refused, at the first hop
and at every redirect."""
module = readers.HOSTS.get(host)
return module.allowed(slug) if module is not None else lambda h: False
class _Refused(Exception):
"""A board answered somewhere its allow-list does not cover."""
class _TooLarge(ValueError):
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
knows the old shape still catches it; `_fetch_rows` catches it by type to
retry at the host's `fallback_url`."""
class _Disallowed(Exception):
"""The site's robots.txt disallows the page a board is read from (see
`_check_robots`)."""
class _RobotsUnreachable(Exception):
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
connection error or a redirect off the site. RFC 9309 says to assume
the site disallows everything then, so the board is not read this pull."""
#: Host name suffixes that name a machine on a private network, never a
#: company's careers site.
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
def _local_host(name: str) -> bool:
"""Whether a host name is an address or a local name rather than a public
DNS name: an IP literal (v4 or v6), anything whose last label is a number
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the
resolver), `localhost`, or a private suffix. A Teamtailor slug can be any
host name, so the allow-list alone would let one point the fetch here."""
name = name.strip("[]").rstrip(".").lower()
try:
ipaddress.ip_address(name)
return True
except ValueError:
pass
last = name.rsplit(".", 1)[-1]
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
"""Fetch one URL for a board: https only on the default port, never an
address or a local name (`_local_host`), the host's own allow-list, at
most three redirect hops each checked the same way, and the body cut off
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
top of the User-Agent and Accept every request carries, and replace
either where they name it. With `www_twin`, a redirect may also land on
the first URL's host with `www.` added or taken off, which only the
robots.txt read asks for (see `_check_robots`).
The body is streamed and counted as it arrives, so an oversized one is
refused at `MAX_BYTES` rather than after it has been read whole, and
the whole request, redirects included, must end within `DEADLINE`: the
per-read `TIMEOUTS` alone let a board that trickles bytes run on for
good. The deadline is checked between chunks, so it can be overrun by
at most one read timeout."""
allow = _allowed_host(host, slug)
if www_twin:
bare = httpx.URL(url).host.removeprefix("www.")
board_allow = allow
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
deadline = time.monotonic() + DEADLINE
current = url
for _hop in range(4):
parsed = httpx.URL(current)
if (parsed.scheme != "https" or parsed.port not in (None, 443)
or _local_host(parsed.host) or not allow(parsed.host)):
raise _Refused(f"{current} is not an allowed host for {host}")
with client.stream(
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
json=(payload or {}) if method == "POST" else None,
) as response:
if response.is_redirect:
location = response.headers.get("location") or ""
if not location:
raise ValueError("redirect with no location")
current = str(httpx.URL(current).join(location))
continue
response.raise_for_status()
chunks: list[bytes] = []
size = 0
for chunk in response.iter_bytes():
size += len(chunk)
if size > MAX_BYTES:
raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)")
if time.monotonic() > deadline:
raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s",
request=response.request)
chunks.append(chunk)
return b"".join(chunks)
raise ValueError("too many redirects")
headers: dict[str, str] | None = None) -> bytes:
"""`_runner.fetch` for one board, with this module's `MAX_BYTES` and
`DEADLINE`."""
return _runner.fetch(client, url, _allowed_host(host, slug), method=method,
payload=payload, headers=headers, max_bytes=MAX_BYTES,
deadline=DEADLINE, name=host)
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
"""The `get` a host module is handed: `_get` bound to one board. `_get`
is looked up when a request is made, not when this is built, so a test
that replaces `ats._get` reaches every host; `payload` and `headers` are
passed on only when given, which keeps a replacement that predates them
working."""
is looked up when a request is made, so a test that replaces `ats._get`
reaches every host; `payload` and `headers` are passed on only when
given, which keeps a replacement that predates them working."""
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
extra: dict[str, Any] = {}
@@ -231,12 +127,6 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get:
return get
#: host -> parser, for the hosts whose module parses a whole body. Every one
#: takes the raw response body and the slug.
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
# --------------------------------------------------------- the classifier
#: The fixed set of functions, and the keyword rules that map a title
@@ -381,362 +271,40 @@ def careers_url(target: str) -> str:
return module.careers_url(slug) if module is not None else ""
#: Pages read from one paged board at most, and the pause between two of
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
#: keeps Workable, which answers a burst with 429, from seeing one. The same
#: pause separates two detail reads.
MAX_PAGES = 100
PAGE_DELAY = 0.5
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""Every page of a board whose host pages, `PAGE_DELAY` apart, until the
host has no next page, the stated total is reached, or `MAX_PAGES`.
Returns the rows and whether the board is complete: as many distinct
results read as the host's stated total, or, for a host that states none
(Teamtailor), the read ended on an empty page (a page with nothing new in
it ends the read short of that, incomplete, so an offset the host ignores
cannot loop). Counted on the raw results, so a row the parser drops does
not make a board incomplete, and on distinct ids, so a page that shifted
under an offset and repeated a role does not make it complete.
A 429 after the first page stops the read with what it has, incomplete;
one on the first page raises like any other failure."""
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
rows: dict[str, Row] = {}
seen: set[str] = set()
total: Any = None
cursor: Any = None
for page in range(MAX_PAGES):
if page:
time.sleep(PAGE_DELAY)
try:
data, results, total, cursor = module.page(get, slug, cursor)
except httpx.HTTPStatusError as exc:
if not page or exc.response.status_code != 429:
raise
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
target, page + 1, len(seen), total)
return list(rows.values()), False
before = len(seen)
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
for row in module.rows(data, slug):
rows.setdefault(row["source_id"], row)
if (cursor is None or (isinstance(total, int) and len(seen) >= total)
or (total is None and len(seen) == before)):
break
complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None
if not complete:
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen),
"an unstated total" if total is None else total)
return list(rows.values()), complete
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""One board's listing, fetched and parsed: its rows and whether they are
the whole board. Raises on any failure — refused host, timeout, a feed
shape the host's parser does not understand. A host that does not page
answers with the whole board, so it is complete whenever it parses; a
paged one is complete when `_paged` says so.
A body that exceeds `MAX_BYTES` gets one retry at the host's
`fallback_url` where it has one (Greenhouse, without `?content=true`):
the description is lost, but the titles land rather than the whole board
being skipped."""
def _fetch_rows(client: httpx.Client, target: str,
ask: readers.Ask | None = None) -> tuple[list[Row], bool]:
"""One board's listing, read and parsed, unfiltered unless `ask` says:
its rows and whether they are the whole board. Raises on any failure.
No sweep and no state: what `fetch_board` and the host tests read."""
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
if hasattr(module, "page"):
return _paged(client, target)
get = _getter(client, host, slug)
try:
body = get(module.feed_url(slug))
except _TooLarge:
if not hasattr(module, "fallback_url"):
raise
log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed",
target, MAX_BYTES)
body = get(module.fallback_url(slug))
return module.parse(body, slug), True
# -------------------------------------------------------------- robots.txt
#: How long a robots.txt answer holds for one board before it is read again.
ROBOTS_TTL = timedelta(days=1)
#: This app's product token, the name a robots.txt group addresses it by:
#: the `desk` of `desk/<version> (personal aggregator)`.
ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower()
#: One rule of a group: `(allow, path pattern)`.
_Rule = tuple[bool, str]
def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]:
"""The groups of a robots.txt, per RFC 9309: one or more `user-agent`
lines (their tokens lowercased) and the `allow` and `disallow` rules that
follow them, up to the next `user-agent` line after a rule. Comments,
other keys and rules before any `user-agent` line are left out, and so is
an empty rule, which allows everything and so says nothing."""
groups: list[tuple[set[str], list[_Rule]]] = []
in_agents = False
for line in text.splitlines():
key, sep, value = line.split("#", 1)[0].partition(":")
key, value = key.strip().lower(), value.strip()
if not sep:
continue
if key == "user-agent":
if not in_agents:
groups.append((set(), []))
in_agents = True
groups[-1][0].add(value.lower())
elif key in ("allow", "disallow") and groups:
in_agents = False
if value:
groups[-1][1].append((key == "allow", value))
return groups
def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]:
"""The rules for `token`: every group that names it (matched whole and
ignoring case, as RFC 9309 says, not as a substring), merged; else every
`*` group, merged; else none, which allows everything."""
for wanted in (token, "*"):
rules = [rule for agents, group in groups if wanted in agents for rule in group]
if rules or any(wanted in agents for agents, _group in groups):
return rules
return []
def _robots_pattern(path: str) -> re.Pattern[str]:
"""A rule's path as a regex anchored at the start: `*` is any run of
characters, and a `$` at the end anchors the end."""
anchored = path.endswith("$")
body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*"))
return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL)
def _robots_verdict(rules: list[_Rule], path: str) -> bool:
"""Whether `path` is allowed by `rules`: the longest matching rule wins,
an allow wins a tie, and no match allows. Both sides are compared
percent-decoded, so `%7E` and `~` are one character."""
path = unquote(path)
best: tuple[int, bool] | None = None
for allow, pattern in rules:
if _robots_pattern(unquote(pattern)).match(path):
weight = (len(pattern), allow)
best = weight if best is None or weight > best else best
return best is None or best[1]
def _robots_allows(body: bytes, page: str) -> bool:
"""Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules
for every agent (`*`), and by those it has for this app's own token
(`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this
app by name is honoured as well. Paths are case-sensitive; a byte order
mark is dropped."""
groups = _robots_groups(body.decode("utf-8-sig", "replace"))
url = httpx.URL(page)
path = (url.raw_path or b"/").decode("ascii", "replace")
return all(_robots_verdict(_robots_rules(groups, token), path)
for token in ("*", ROBOTS_TOKEN))
def _check_robots(client: httpx.Client, target: str,
state: MutableMapping[str, str] | None = None) -> None:
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
robots.txt that disallows that page, and `_RobotsUnreachable` when it
cannot be read. Read through `_get`, so from the board's own allowed
hosts only, except that a redirect between the host and its `www.` twin
is followed (at most three hops, https only); any other redirect off the
site is unreachable. With `state`, the verdict and when it was reached
are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
which allows, and is kept like any other verdict. A 5xx, a 429, a
timeout or a connection error means the file is unreachable, which the
RFC says to treat as disallowing everything: the board is not read on
this pull, fails with "robots.txt could not be read", and nothing is
kept, so the next pull asks again. A 429 counts as unreachable although
it is a 4xx, since it says "not now" rather than "there is none"."""
host, _, slug = target.partition(":")
module = readers.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
return
page = module.robots_page(slug)
key = f"ats:robots:{target}"
now = datetime.now(UTC)
if state is not None:
try:
kept = json.loads(state.get(key) or "{}")
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
except (ValueError, KeyError, TypeError):
kept, fresh = {}, False
if fresh:
if not kept.get("allowed", True):
raise _Disallowed(f"robots.txt disallows {page}")
return
url = f"https://{httpx.URL(page).host}/robots.txt"
try:
body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
except httpx.HTTPStatusError as exc:
code = exc.response.status_code
if not 400 <= code < 500 or code == 429:
raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
allowed = True
except Exception as exc:
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
else:
# A host that serves a path whatever its case is matched without
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
if getattr(module, "ROBOTS_ANY_CASE", False):
allowed = _robots_allows(body.lower(), page.lower())
else:
allowed = _robots_allows(body, page)
if state is not None:
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
"allowed": allowed})
if not allowed:
raise _Disallowed(f"robots.txt disallows {page}")
# ---------------------------------------------------------------- details
#: Roles read in detail from one board in one pull at most. The rest of a
#: large new board is read over the next pulls, 200 at a time.
DETAIL_CAP = 200
#: The row keys a host's `detail` may fill: those whose column survives a
#: later pull that sends the listing's empty value for them, so what a role's
#: own page said is not wiped the pull after it was read. `description`,
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
#: earliest date a role ever had. A filled column is still overwritten by
#: another filled value, so `_read_details` withholds the listing's place and
#: remote for a role already read. `title` and `url` are not here, since the
#: listing always states them and would overwrite them; nor are `department`
#: (no column of its own, only `raw` and the classifier's guess) and
#: `salary_text`, both rewritten from the listing on every pull.
DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min",
"salary_max", "posted")
listing = _runner.read_listing(readers.HOSTS[host], slug, _getter(client, host, slug), ask,
limits=_limits())
return listing.rows, listing.complete
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
detailed: set[str], failed: set[str]
) -> tuple[list[Row], set[str], set[str]]:
"""The rows of a board whose host has `detail`, with the roles not yet
read in detail filled from their own pages: at most `DETAIL_CAP` of them,
`PAGE_DELAY` apart, first the roles never tried and then those whose
detail failed before (`failed`), so a few roles that always fail cannot
take the whole cap pull after pull. Returns the rows and the compound ids
read (for `ats:detailed:<target>`) and failed (for
`ats:detailfailed:<target>`).
Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key
is left out with a debug line. Every row not read in detail in this call
goes out with an empty description: a role read on an earlier pull keeps
the one it has, since `upsert_role` never overwrites a filled column with
an empty value, and a teaser from the listing must not replace it. A role
read on an earlier pull goes out without the listing's place and remote
as well, for the same reason: the listing states them every pull
("2 Locations"), and would replace what the role's own page said.
A detail that answers 404 or 410 counts as read: the role is gone or has
no page of its own, and asking again would not change that. One that
raises otherwise is logged and the role kept in `failed`; a 429 stops the
reads for this board until the next pull, and does not mark the role as
failing. None of this touches completeness, which is the listing's alone.
The ids of roles no longer listed are dropped from both sets, but only
after a complete listing, so one cut short does not make the next pull
read its missing roles again."""
"""`_runner.read_details` for one board, with `rows` as the whole listing
when `complete`."""
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
done, failing = set(detailed), set(failed) - set(detailed)
def compound(row: Row) -> str:
return f"{host}:{slug}:{row['source_id']}"
unread = [row for row in rows if compound(row) not in done]
queue = ([row for row in unread if compound(row) not in failing]
+ [row for row in unread if compound(row) in failing])[:DETAIL_CAP]
filled: dict[str, Row] = {}
for row in queue:
key = compound(row)
time.sleep(PAGE_DELAY)
try:
answer = module.detail(get, slug, row)
except httpx.HTTPStatusError as exc:
code = exc.response.status_code
if code == 429:
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
target)
break
if code in (404, 410):
done.add(key)
failing.discard(key)
else:
log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code)
failing.add(key)
continue
except Exception as exc:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
failing.add(key)
continue
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS",
target, row["source_id"], ", ".join(ignored))
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
done.add(key)
failing.discard(key)
out = []
for row in rows:
key = compound(row)
if key in filled:
out.append(row | {"description": ""} | filled[key])
elif key in detailed:
out.append(row | {"description": "", "location": "", "remote": None})
else:
out.append(row | {"description": ""})
if complete:
listed = {compound(row) for row in rows}
done &= listed
failing &= listed
return out, done, failing
listed = {f"{host}:{slug}:{row['source_id']}" for row in rows} if complete else None
return _runner.read_details(readers.HOSTS[host], host, slug, _getter(client, host, slug),
rows, detailed, failed, listed=listed, limits=_limits())
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
"does this board answer, what does it list, and is that all of it". For
`app companies add|check`, which ask that question about one board at a
time outside any pull. The listing only, with no detail reads, and
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
`_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
that cannot be checked."""
"""Fetch and parse one board directly: no `Source`, no state, just "does
this board answer, what does it list, and is that all of it". For `app
companies add|check`. The listing only, unfiltered, no detail reads.
Raises the same way `_fetch_rows` does."""
with httpx.Client() as client:
_check_robots(client, target)
return _fetch_rows(client, target)
def failure_reason(exc: BaseException) -> str:
"""One short line for why a board did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, _Disallowed):
return "robots.txt disallows it"
if isinstance(exc, _RobotsUnreachable):
return "robots.txt could not be read"
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
return text[:200]
failure_reason = _runner.failure_reason
# -------------------------------------------------------------- the plugin
@@ -745,9 +313,8 @@ class ATSSource(Source):
name = "ats"
description = f"Company job boards: {', '.join(HOSTS.values())}."
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
#: company}` — `url` carries the `host:slug` target and `label` the
#: company name, which is how `Source.targets()`'s generic dict reading
#: hands both back without a plugin-specific override.
#: company, "params": {...}}`: `url` carries the `host:slug` target,
#: `label` the company name and `params` the board's filter.
subscription_field = "boards"
spec = PluginSpec(
@@ -763,99 +330,74 @@ class ATSSource(Source):
)},
)
def _id_set(self, key: str) -> set[str]:
"""A JSON list of ids kept in state, as a set; empty when there is
none or it does not parse."""
try:
return {str(i) for i in json.loads(self.state.get(key) or "[]")}
except (ValueError, TypeError):
return set()
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
# Handed in by the batch that runs this plugin, as the marker of
# the pull run that queued it (see `app/jobs/pull.py`), and never
# read from state, where a later run may have replaced it. A
# plugin run on its own — a test, a dry run — has no run to stamp
# itself against, so it makes one up.
# the pull run that queued it (see `app/jobs/pull.py`). A plugin
# run on its own (a test, a dry run) makes one up.
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try:
_check_robots(client, target, self.state)
rows, complete = _fetch_rows(client, target)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
# The first failure in a row sets `since`, later ones only
# the reason; `app.jobs.pull` closes the board's roles
# once `since` is `STALE_DAYS` old.
try:
since = json.loads(self.state.get(failing, "") or "{}").get("since")
except ValueError:
since = None
self.state[failing] = json.dumps({
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
"reason": failure_reason(exc)})
continue
self.state.pop(failing, None)
yield from self._board(client, target, label, run_marker)
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(readers.HOSTS.get(host), "detail"):
detailed, failed = (self._id_set(key) for key in detail_keys)
rows, detailed, failed = _read_details(client, target, rows, complete,
detailed, failed)
detail_sets = (detailed, failed)
def _board(self, client: httpx.Client, target: str, label: str,
run_marker: str) -> Iterator[Item]:
host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try:
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
ask = _runner.ask_for(module, self.params_for(target),
keys=self.options.get("keys"))
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
limits=_limits())
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
_runner.record_failure(self.state, failing, exc)
return
self.state.pop(failing, None)
seen_ids = []
for row in rows:
# The compound id, matching `roles.source_id` — what
# `Store.mark_closed` compares against, not the board's own
# bare job id.
compound_id = f"{host}:{slug}:{row['source_id']}"
seen_ids.append(compound_id)
yield Item(
source_id=compound_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
subscription=label,
kind="role",
raw={
"company": label,
"place": row["location"],
"department": row["department"],
"function": classify(row["department"], row["title"]),
"remote": row["remote"],
# Every parser sets salary_text, empty or not (see
# "salary_text" in `readers/_shared.py`); only
# the hosts that state structured pay also set the
# normalised figures, so the rest leave them at
# row.get's default of no figure.
"salary_text": row.get("salary_text", ""),
"salary_min": row.get("salary_min"),
"salary_max": row.get("salary_max"),
"currency": row.get("currency", ""),
"board": host,
"slug": slug,
},
)
# Stamped only once the board has parsed without raising, so a
# board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it. `complete`
# says whether the ids are the whole board, which the pull
# needs before it closes what they leave out. The ids read in
# detail are kept here too, after every row went out, so a
# pull stopped half way reads the rest again rather than
# losing their descriptions.
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
if detail_sets is not None:
for key, ids in zip(detail_keys, detail_sets, strict=True):
self.state[key] = json.dumps(sorted(ids))
def compound(source_id: str) -> str:
# Matches `roles.source_id`, which `Store.mark_closed` compares.
return f"{host}:{slug}:{source_id}"
rows = listing.rows
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(module, "detail"):
detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys)
listed = {compound(i) for i in listing.ids} if listing.complete else None
rows, detailed, failed = _runner.read_details(
module, host, slug, get, rows, detailed, failed, listed=listed,
limits=_limits())
detail_sets = (detailed, failed)
for row in rows:
description = row.get("description") or ""
yield Item(
source_id=compound(row["source_id"]),
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": label,
"function": classify(row.get("department") or "", row["title"]),
"board": host,
"slug": slug,
},
)
# Stamped only once every row went out, so a board that blew up half
# way through is neither marked ok nor has a partial id list treated
# as the whole of it, and a pull stopped half way reads the rest of
# its details again.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
if detail_sets is not None:
for key, ids in zip(detail_keys, detail_sets, strict=True):
self.state[key] = json.dumps(sorted(ids))
+14 -14
View File
@@ -129,7 +129,8 @@ exists.
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
name; `_shared.only(host)` is the usual one. Every request goes through
`get`, which is `ats._get` bound to the board:
`get`, which is `_runner.fetch` bound to the target (`_runner.getter`; the
`ats` Source binds it through `ats._get`):
get(url, *, method="GET", payload=None, headers=None) -> bytes
@@ -138,7 +139,7 @@ default User-Agent and `Accept: application/json, text/xml`, and replace
either where they name it. `get` is https only on port 443, never to an IP
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
most three redirects each checked against `allowed(slug)`, cuts the body at
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
`_runner.MAX_BYTES` and ends the whole request within `_runner.DEADLINE`. It returns
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
failure, and another exception (`ValueError` among them) for a refused host,
@@ -151,8 +152,9 @@ id, to decide completeness), the total the host states or None, and the
cursor of the next page or None after the last. The first call gets cursor
None. A board with a total is complete when as many distinct ids were read;
one without, when the read ended on an empty page. A 429 after the first page
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
apart. A paged module may also have `parse(body, slug)` for one page's body.
ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
`_runner.PAGE_DELAY` apart, and a board cut there is swept across pulls
(`_runner.read_listing`). A paged module may also have `parse(body, slug)` for one page's body.
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
own id for the role, unique on the board), `title`, `url` (http or https,
@@ -183,11 +185,11 @@ keeps it.
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
dict of the fields it found. The shared code calls it for roles not yet read
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
and `posted`, the keys whose columns survive a later pull that sends the
(`ats:detailed:<target>`), at most `_runner.DETAIL_CAP` (200) a board a
pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that
failed (`ats:detailfailed:<target>`). Only `_runner.DETAIL_FIELDS` are taken:
`description`, `location`, `places`, `teaser`, `remote`, `country`,
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the
listing's empty value; any other key is left out with a debug log line.
`title`, `url`, `department` and `salary_text` cannot be filled from
`detail`: the listing rewrites them every pull. A role not read in detail
@@ -203,11 +205,9 @@ Whoever builds a reader reads the host's robots.txt for the paths the reader
requests and reports what it disallows, and the owner decides whether the
host is read anyway. `robots` records that decision with its date, e.g.
`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is
disallowed: owner, 2026-09-30"`. The runtime check in `ats.py`, which
`robots_page` and `ROBOTS_ANY_CASE` feed, predates this rule and goes with
the runner; a new reader defines neither. Until then it reads
`https://<host>/robots.txt` at most once a day per board and fails the board
when the page is disallowed for `*` or the app's token.
disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time;
`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are
read by nothing and a new reader defines neither.
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
the slug is a board; `probe_ok(body)` narrows that when a host answers an
+712
View File
@@ -0,0 +1,712 @@
"""The runner: what the app does for every reader, board or search, from what
the reader declares in `READS`.
A reader says how its host answers; this module reads it: the guarded fetch
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
to (`ask_for`) and its series (`series`), the listing whole or paged, one
series per `one`-filter value unioned by id (`read_listing`), completeness,
the sweep of an oversized paged board across pulls, the 10% guard on a
filtered or `empty="suspect"` read (`read`), the per-role detail reads
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
`plugins/search.py` are thin `Source` classes over it: they pass their own
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
as they were.
A module without `READS` is read with the older signatures, `feed_url(slug)`
and `page(get, slug, cursor)`, and never gets an `Ask`.
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
"seen", "rows", "whole"}`, continued by the next pull
count `{"asks", "count"}`, the roles the last whole read listed, for
the 10% guard
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
"""
from __future__ import annotations
import ipaddress
import itertools
import json
import logging
import os
import re
import time
from collections.abc import Callable, Iterable, Mapping, MutableMapping
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from types import ModuleType
from typing import Any
import httpx
from desk.sources import USER_AGENT
from ._contract import Ask, Key, Place, Reads
from ._shared import Get, Row
log = logging.getLogger(__name__)
#: openings.go's 8 MB body cap is not kept: every URL fetched here is a feed
#: on a fixed allow-list, and the real run of 2026-09-23 hit 8 MB on the
#: biggest employers (Anduril at 41 MB, Anthropic, OpenAI, Databricks between
#: 9 and 19 MB). 96 MB fits them; a board still over it gets one retry at its
#: reader's `fallback_url` where it has one.
MAX_BYTES = 96 << 20
#: The first byte of a Greenhouse board waited up to 16.6 s in 84 requests
#: measured on 2026-09-29, and the body then arrived in under 1 s. So a read
#: may wait 30 s and a whole request, redirects and body included, 60 s.
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
DEADLINE = 60.0
#: Pages read from one target in one pull at most, and the pause between two
#: requests (pages and detail reads), which keeps Workable from answering 429.
MAX_PAGES = 100
PAGE_DELAY = 0.5
#: Roles read in detail from one board in one pull at most.
DETAIL_CAP = 200
#: The row keys a reader's `detail` may fill: those whose column survives a
#: later pull that sends the listing's empty value, so what a role's own page
#: said is not wiped the pull after it was read. `title`, `url`, `department`
#: and `salary_text` are left out, since the listing rewrites them every pull.
#: Workday, SuccessFactors and Rippling state `country` and `employment` on a
#: role's own page only.
DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "country",
"employment", "currency", "salary_min", "salary_max", "posted")
#: How long a board's `facets` are kept before they are asked again.
FACETS_TTL = timedelta(days=1)
#: A filtered or `empty="suspect"` read listing under this share of what the
#: same read listed last time counts as a failure.
SUSPECT_SHARE = 0.1
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
class Refused(Exception):
"""A request or a redirect aimed at a host its allow-list does not cover."""
class TooLarge(ValueError):
"""A response over the byte cap. A `ValueError`, as it was before it had a
type of its own."""
class Suspect(Exception):
"""A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE`
of what it listed last time."""
class MissingKey(ValueError):
"""A reader's `READS.key` has no value in the config or the environment."""
@dataclass(frozen=True, slots=True)
class Limits:
"""What one Source allows a read: pages, the pause between requests, and
the detail reads a pull."""
max_pages: int = MAX_PAGES
page_delay: float = PAGE_DELAY
detail_cap: int = DETAIL_CAP
@dataclass(slots=True)
class Listing:
"""One read of a target's listing. `rows` are this pull's rows, `ids` the
`source_id`s the read listed: this pull's, or the whole sweep's once it
reached the end. `complete` says `ids` are the whole list. `sweep` is the
state to keep for the next pull, None when no sweep is under way.
`filtered` says the read asked the host for a part of the list."""
rows: list[Row]
complete: bool
ids: list[str] = field(default_factory=list)
filtered: bool = False
sweep: dict[str, Any] | None = None
asks: str = ""
# --------------------------------------------------------------- modules
def declared(module: ModuleType) -> bool:
"""Whether `module` declares `READS`, and so takes `ask`."""
return isinstance(getattr(module, "READS", None), Reads)
def module_of(name: str) -> ModuleType | None:
"""The reader named `name`, a board host first, else a search provider."""
from plugins import readers
return readers.HOSTS.get(name) or readers.PROVIDERS.get(name)
# ----------------------------------------------------------------- fetch
#: Host name suffixes that name a machine on a private network.
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
def local_host(name: str) -> bool:
"""Whether a host name is an address or a local name rather than a public
DNS name: an IP literal, anything whose last label is a number
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1), `localhost`
or a private suffix."""
name = name.strip("[]").rstrip(".").lower()
try:
ipaddress.ip_address(name)
return True
except ValueError:
pass
last = name.rsplit(".", 1)[-1]
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
def fetch(client: httpx.Client, url: str, allow: Callable[[str], bool], *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None, max_bytes: int | None = None,
deadline: float | None = None, name: str = "") -> bytes:
"""Fetch one URL for one target: https on port 443 only, never an address
or a local name, only hosts `allow` accepts, at most three redirects each
checked the same way, the body cut at `max_bytes` (`MAX_BYTES`) and the
whole request ended within `deadline` (`DEADLINE`) seconds. A POST sends
`payload` as JSON, `{}` when None; `headers` are added to the
User-Agent and Accept and replace either where they name it.
The body is streamed and counted as it arrives, and the deadline checked
between chunks, so a trickling answer cannot run on. Raises `Refused`,
`TooLarge`, `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError`
on a timeout or a connection failure, and ValueError on a broken
redirect."""
cap = MAX_BYTES if max_bytes is None else max_bytes
limit = DEADLINE if deadline is None else deadline
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
ends = time.monotonic() + limit
current = url
for _hop in range(4):
parsed = httpx.URL(current)
if (parsed.scheme != "https" or parsed.port not in (None, 443)
or local_host(parsed.host) or not allow(parsed.host)):
raise Refused(f"{current} is not an allowed host for {name or 'this reader'}")
with client.stream(
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
json=(payload or {}) if method == "POST" else None,
) as response:
if response.is_redirect:
location = response.headers.get("location") or ""
if not location:
raise ValueError("redirect with no location")
current = str(httpx.URL(current).join(location))
continue
response.raise_for_status()
chunks: list[bytes] = []
size = 0
for chunk in response.iter_bytes():
size += len(chunk)
if size > cap:
raise TooLarge(f"response too large (over {cap} bytes)")
if time.monotonic() > ends:
raise httpx.ReadTimeout(f"no whole answer within {limit:.0f} s",
request=response.request)
chunks.append(chunk)
return b"".join(chunks)
raise ValueError("too many redirects")
def getter(client: httpx.Client, allow: Callable[[str], bool], name: str = "") -> Get:
"""The `get` a reader is handed: `fetch` bound to one target's allow-list."""
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
return fetch(client, url, allow, method=method, payload=payload, headers=headers,
name=name)
return get
# ------------------------------------------------------------------- ask
#: A filter dimension of `Filters` -> the field of `Ask` it fills.
_ASK_FIELD = {"country": "countries", "place": "places", "remote": "remote", "query": "query"}
#: Keys a subscription's params or a reach may use, -> the dimension.
_PARAM_DIMENSION = {"countries": "country", "country": "country", "places": "place",
"place": "place", "remote": "remote", "query": "query", "q": "query"}
def _values(value: Any) -> tuple[str, ...]:
"""A param's value as a tuple of non-empty strings."""
items = value if isinstance(value, list | tuple | set) else [value]
return tuple(str(v).strip() for v in items if v not in (None, "", 0) and str(v).strip())
def _dimensions(module: ModuleType, given: Mapping[str, Any]) -> dict[str, Any]:
"""`dimension -> value` from a params or reach mapping: the generic keys
(`countries`, `places`, `remote`, `q`) and the reader's own `PARAMS`
(saved param -> dimension, e.g. Jobindex's `area` -> `place`)."""
names = _PARAM_DIMENSION | dict(getattr(module, "PARAMS", {}) or {})
out: dict[str, Any] = {}
for key, value in given.items():
dimension = names.get(key)
if dimension == "remote":
if isinstance(value, bool):
out["remote"] = value
elif dimension == "query":
if str(value or "").strip():
out["query"] = str(value).strip()
elif dimension in ("country", "place"):
values = _values(value)
if dimension == "country":
values = tuple(v.upper() for v in values)
if values:
out[dimension] = values
return out
def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
"""A reader's key: each of `key.fields` from the config entry
`sources.search.<key.name>` (`keys` is `sources.search`), else from the
environment variable at the same position in `key.env`. Raises
`MissingKey` naming both places when a field has neither."""
entry = (keys or {}).get(key.name) or {}
out: dict[str, str] = {}
for index, name in enumerate(key.fields):
env = key.env[index] if index < len(key.env) else ""
value = str((entry.get(name) if isinstance(entry, Mapping) else "") or "") \
or (os.environ.get(env, "") if env else "")
if not value:
where = f"sources.search.{key.name}: {', '.join(key.fields)}"
if key.env:
where += f", or {' and '.join(key.env)}"
raise MissingKey(f"{key.name} needs a key: {where}")
out[name] = value
return out
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
reach: Mapping[str, Any] | None = None, *,
keys: Mapping[str, Any] | None = None) -> Ask:
"""The `Ask` one target resolves to. `reach` (from the agents, package F)
and then the subscription's own `params` (a board's override, e.g.
`{"countries": ["DK"], "remote": true}`, or a search's saved params) set
the dimensions, params winning per dimension, narrowed to the ones the
reader declares in `READS.filters`. A search's params go whole into
`Ask.params`; a reader with `READS.key` gets the resolved key under
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
A module without `READS` gets `Ask()`."""
if not declared(module):
return Ask()
reads: Reads = module.READS
params = dict(subscription_params or {})
wanted = _dimensions(module, reach or {}) | _dimensions(module, params)
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()
if name in reads.filters.declared()}
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
if reads.key is not None:
extra["key"] = resolve_key(reads.key, keys)
return Ask(**fields, params=extra)
def filtered(ask: Ask) -> bool:
"""Whether `ask` reads a part of a board: a country, a place or remote."""
return bool(ask.countries or ask.places or ask.remote is not None)
def series(module: ModuleType, ask: Ask,
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
"""The series of requests `ask` takes on this reader: one per value of a
dimension declared `one` (the product when two are), one with every
value for `many`, and host ids from `facets` for `ids` (values `facets`
does not know are dropped; an empty list when none is left, since the
host then has nothing in reach). `[ask]` for a module without `READS`."""
if not declared(module):
return [ask]
reach = module.READS.filters
countries, places = ask.countries, ask.places
for dimension in ("country", "place"):
values = countries if dimension == "country" else places
if getattr(reach, dimension) != "ids" or not values:
continue
known = (facets or {}).get(dimension) or {}
mapped = tuple(known[v] for v in values if v in known)
if not mapped:
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
", ".join(values))
return []
if dimension == "country":
countries = mapped
else:
places = mapped
by_country = [(c,) for c in countries] if reach.country == "one" and countries \
else [countries]
by_place = [(p,) for p in places] if reach.place == "one" and places else [places]
return [ask.model_copy(update={"countries": c, "places": p})
for c, p in itertools.product(by_country, by_place)]
def fingerprint(asks: Iterable[Ask]) -> str:
"""What a read asked, without its params (which may hold a key): the same
string for the same series, so a sweep or a count is only compared with
a read that asked the same."""
return json.dumps([ask.model_dump(exclude={"params"}, mode="json") for ask in asks],
sort_keys=True)
# --------------------------------------------------------------- listing
def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
limits: Limits) -> dict[str, Row]:
"""A whole listing, one request a series, unioned by `source_id`. A body
over the cap gets one retry at the reader's `fallback_url`."""
rows: dict[str, Row] = {}
for index, ask in enumerate(asks):
if index:
time.sleep(limits.page_delay)
url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug)
try:
body = get(url)
except TooLarge:
if not hasattr(module, "fallback_url"):
raise
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
module.NAME, slug)
body = get(module.fallback_url(slug))
for row in module.parse(body, slug):
rows.setdefault(row["source_id"], row)
return rows
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
can_sweep: bool = False,
facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing:
"""One target's listing, read and parsed. Raises on any failure of the
first request, and on a reader that does not understand an answer.
A whole listing is complete whenever it parses. A paged one reads each
series' pages, `limits.page_delay` apart, until the host has no next
page, the stated total is reached, a page brings nothing new (a host
that ignores the cursor, incomplete) or `limits.max_pages` pages in all.
A series is complete when it read as many distinct result ids as its
stated total, or, without a total, when it ended on the host's last
page; never for `complete="never"`. The read is complete when every
series is. A 429 after the first page ends the read, incomplete.
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
the `sweep` to keep, and the next pull, handed it back, continues from
its series and cursor; the sweep is complete when it reaches the end,
and `ids` then hold every id it listed. A read at the reader's `ceiling`
is not swept: the host answers nothing past it."""
limits = limits or Limits()
ask = ask or Ask()
asks = series(module, ask, facets)
is_filtered = filtered(ask)
key = fingerprint(asks)
if not asks:
return Listing([], True, [], is_filtered, None, key)
if not hasattr(module, "page"):
rows = _whole(module, slug, get, asks, limits)
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key)
reads: Reads | None = module.READS if declared(module) else None
never = reads is not None and reads.complete == "never"
ceiling = reads.ceiling if reads is not None else None
start, cursor, seen, earlier, whole = 0, None, set(), [], True
if sweep and sweep.get("asks") == key:
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
whole = bool(sweep.get("whole", True))
rows: dict[str, Row] = {}
pages = 0
stop: str = ""
index = start
for index in range(start, len(asks)):
total: Any = None
ended = stalled = False
while True:
if pages >= limits.max_pages:
stop = "cap"
break
if pages:
time.sleep(limits.page_delay)
try:
if reads is not None:
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
else:
data, results, total, cursor = module.page(get, slug, cursor)
except httpx.HTTPStatusError as exc:
if not pages or exc.response.status_code != 429:
raise
log.warning("readers: %s:%s: 429 on page %d; kept %d, not complete",
module.NAME, slug, pages + 1, len(rows))
stop = "429"
break
pages += 1
before = len(seen)
seen |= _result_ids(results)
for row in module.rows(data, slug):
rows.setdefault(row["source_id"], row)
if cursor is None or (isinstance(total, int) and len(seen) >= total):
ended = True
break
if total is None and len(seen) == before:
stalled = True
break
if stop:
break
done = (not never and not stalled
and (len(seen) >= total if isinstance(total, int) else ended))
if not done:
log.warning("readers: %s:%s: read %d of %s; not complete", module.NAME, slug,
len(seen), "an unstated total" if total is None else total)
whole = whole and done
cursor, seen = None, set()
listed = list(dict.fromkeys(earlier + list(rows)))
if not stop:
return Listing(list(rows.values()), whole, listed, is_filtered, None, key)
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
module.NAME, slug, pages, stop)
at_ceiling = ceiling is not None and len(seen) >= ceiling
if not can_sweep or never or at_ceiling or not _storable(cursor):
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key)
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
"rows": listed, "whole": whole}
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key)
def _storable(cursor: Any) -> bool:
"""Whether a cursor survives a trip through JSON in state."""
try:
return json.loads(json.dumps(cursor)) == cursor
except (TypeError, ValueError):
return False
# ------------------------------------------------------------ with state
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
try:
value = json.loads(state.get(key) or "{}")
except (TypeError, ValueError):
return {}
return value if isinstance(value, dict) else {}
def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]:
"""The reader's `facets` when `ask` sets a dimension it declares `ids`,
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
if not declared(module) or not hasattr(module, "facets"):
return {}
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name in ("country", "place")):
return {}
now = datetime.now(UTC)
if state is not None and key:
kept = _load(state, key)
try:
if now - datetime.fromisoformat(kept["checked"]) < FACETS_TTL:
return dict(kept.get("facets") or {})
except (KeyError, TypeError, ValueError):
pass
found = dict(module.facets(get, slug) or {})
if state is not None and key:
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
"facets": found})
return found
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
slug: str, get: Get, ask: Ask | None = None, *,
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
the facets it needs, the sweep it continues, and the 10% guard. A read
that is filtered or whose reader declares `empty="suspect"`, and that
listed under `SUSPECT_SHARE` of what the same read listed last time,
raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
the rest once the rows are stored."""
ask = ask or Ask()
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets)
suspect = declared(module) and module.READS.empty == "suspect"
last = _load(state, f"{prefix}:count:{target}")
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
if ((listing.filtered or suspect) and listing.sweep is None and count
and len(listing.ids) < SUSPECT_SHARE * count):
raise Suspect(f"listed {len(listing.ids)} roles, under "
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
return listing
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
listing: Listing, ids: list[str]) -> None:
"""Record a read whose rows are stored: `ok` (the run's marker),
`complete`, `ids` (the compound ids, what the pull closes against),
`filtered`, the sweep to continue or none, and the count for the 10%
guard once a read is whole."""
state[f"{prefix}:ok:{target}"] = run
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
if listing.sweep is not None:
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
else:
state.pop(f"{prefix}:sweep:{target}", None)
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
"count": len(listing.ids)})
# ---------------------------------------------------------------- details
def id_set(state: Mapping[str, str], key: str) -> set[str]:
"""A JSON list of ids kept in state, as a set; empty when there is none
or it does not parse."""
try:
return {str(i) for i in json.loads(state.get(key) or "[]")}
except (TypeError, ValueError):
return set()
def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[Row],
detailed: set[str], failed: set[str], *, listed: set[str] | None = None,
limits: Limits | None = None) -> tuple[list[Row], set[str], set[str]]:
"""The rows with the roles not yet read in detail filled from their own
pages, at most `limits.detail_cap`, `limits.page_delay` apart: first the
roles never tried, then those in `failed`. Ids are compound,
`<name>:<slug>:<source_id>`. Returns the rows and the ids read and
failed, to keep as the caller's `detailed` and `failed` sets.
Only `DETAIL_FIELDS` are taken from what `detail` returns. A `location`
without `places` becomes the one place. Every row not read now goes out
with an empty description, so one read earlier stays, and a row read on
an earlier pull without the listing's place and remote, so what its page
said stays. A 404 or 410 counts as read, a 429 stops the reads for this
pull, anything else keeps the role in `failed`. `listed` is every id of a
complete listing: ids outside it leave both sets."""
limits = limits or Limits()
done, failing = set(detailed), set(failed) - set(detailed)
def compound(row: Row) -> str:
return f"{name}:{slug}:{row['source_id']}"
unread = [row for row in rows if compound(row) not in done]
queue = ([row for row in unread if compound(row) not in failing]
+ [row for row in unread if compound(row) in failing])[:limits.detail_cap]
filled: dict[str, Row] = {}
for row in queue:
key = compound(row)
time.sleep(limits.page_delay)
try:
answer = module.detail(get, slug, row)
except httpx.HTTPStatusError as exc:
code = exc.response.status_code
if code == 429:
log.warning("readers: %s:%s: 429 on a detail read; the rest wait", name, slug)
break
if code in (404, 410):
done.add(key)
failing.discard(key)
else:
log.warning("readers: %s:%s: detail of %s answered %s", name, slug,
row["source_id"], code)
failing.add(key)
continue
except Exception as exc:
log.warning("readers: %s:%s: detail of %s failed (%s)", name, slug,
row["source_id"], exc)
failing.add(key)
continue
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
name, slug, row["source_id"], ", ".join(ignored))
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment]
if taken.get("location") and "places" not in taken:
taken["places"] = [{"text": taken["location"], "remote": None}]
filled[key] = taken
done.add(key)
failing.discard(key)
out: list[Row] = []
for row in rows:
key = compound(row)
if key in filled:
out.append(row | {"description": ""} | filled[key]) # type: ignore[operator]
elif key in detailed:
out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator]
"remote": None})
else:
out.append(row | {"description": ""}) # type: ignore[operator]
if listed is not None:
done &= listed
failing &= listed
return out, done, failing
# -------------------------------------------------------------------- raw
def places_of(row: Mapping[str, Any]) -> list[Place]:
"""A row's places: its `places`, or, for a row without (an undeclared
reader's), its `location` as the one place, split nothing."""
places = row.get("places")
if not isinstance(places, list):
location = str(row.get("location") or "").strip()
return [{"text": location, "remote": None}] if location else []
out: list[Place] = []
for place in places:
if isinstance(place, Mapping) and str(place.get("text") or "").strip():
out.append({"text": str(place["text"]).strip(), "remote": place.get("remote")})
return out
def raw_of(row: Mapping[str, Any]) -> dict[str, Any]:
"""The `raw` of an `Item` for one row: every key the row states except
those the `Item` carries itself (`ITEM_KEYS`), with `places` always a
list, `place` the display line of the places joined with "; ", and the
keys storage reads always present (`department`, `remote`,
`salary_text`, `salary_min`, `salary_max`, `currency`, `teaser`)."""
places = places_of(row)
raw = {k: v for k, v in row.items() if k not in ITEM_KEYS}
return raw | {
"places": places,
"place": "; ".join(place["text"] for place in places),
"department": row.get("department") or "",
"remote": row.get("remote"),
"salary_text": row.get("salary_text") or "",
"salary_min": row.get("salary_min"),
"salary_max": row.get("salary_max"),
"currency": row.get("currency") or "",
"teaser": row.get("teaser") or "",
}
# --------------------------------------------------------------- failures
def failure_reason(exc: BaseException) -> str:
"""One short line for why a target did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
return text[:200]
def record_failure(state: MutableMapping[str, str], key: str, exc: BaseException) -> None:
"""Keep a failure under `key` as `{"since", "reason"}`: the first failure
in a row sets `since`, later ones only the reason. The pull closes a
target's roles once `since` is old enough; its next answer removes the
key."""
since = _load(state, key).get("since")
state[key] = json.dumps({"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
"reason": failure_reason(exc)})
+64 -381
View File
@@ -5,421 +5,109 @@ roles at companies nobody has listed yet, by asking a cross-company board a
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
A subscription is one saved search; its `params` are the question.
Three boards, checked live on 2026-09-23 and, for Platsbanken, 2026-09-29:
jobindex Denmark's largest job board. Every search result page has an RSS
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
own syntax: words are OR, `"a b"` a phrase, `+word` required.
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region
Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus.
20 results a page.
adzuna An aggregator with a documented API for most of Europe, the UK
and the US, but not Denmark (adzuna.dk does not exist). Needs a
free key from developer.adzuna.com, as `sources.search.adzuna:
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
against Adzuna's documented response and not yet run live.
platsbanken Sweden's public employment service, through the JobTech search
API at jobsearch.api.jobtechdev.se, no key needed. `region` is
the id of a län in the JobTech taxonomy (Skåne län is
`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page,
and an `offset` above 2000 is refused. It states `total`. It has
no remote flag worth reading (`workplace_model` is mostly
empty), so `remote` comes from the headline.
Considered and left out: Jobnet (its search endpoint answers 401 outside the
browser session of its own site) and Jooble (a key needs a manual
application, and its API host answered a Cloudflare challenge).
The providers are the readers of `scope="search"` in `plugins/readers/`
(`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other
reader: the search's params go in `Ask.params`, a declared key is resolved
from `sources.search.<name>` or the environment and refused when missing,
and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most
`MAX_PAGES`).
A search result list is never a board's complete list, so a role missing from
one run is not closed here. A search that read everything that matches was
complete, and the pull closes what it stopped listing; the rest close once no
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna and
Platsbanken state how many results match, so their searches are complete when
they read that many; Jobindex states no count, so a Jobindex search is
complete when a page within the page limit came back short.
complete (`complete="stated"`, as many distinct results as the stated total),
and the pull closes what it stopped listing; the rest close once no search
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
"""
from __future__ import annotations
import html
import json
import logging
import os
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable, Iterator
import sys
from collections.abc import Iterator, Mapping
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from pathlib import Path
from types import ModuleType
from typing import Any
import httpx
from desk.sources import (
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
)
try:
from plugins import readers
except ModuleNotFoundError:
# Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`.
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import readers
from plugins.readers import _runner
from plugins.readers._shared import Row
log = logging.getLogger(__name__)
TIMEOUT = 15.0
#: Pages fetched per search unless its params say otherwise, and the most a
#: Pages read per search unless its params say otherwise, and the most a
#: search may ask for. A search that needs more than ten pages is too broad to
#: read anyway.
DEFAULT_PAGES = 5
MAX_PAGES = 10
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
#: What every provider turns a result into.
Row = dict[str, Any]
#: The pause between two pages of one search.
PAGE_DELAY = _runner.PAGE_DELAY
def _web_url(url: str) -> bool:
parsed = httpx.URL((url or "").strip())
return parsed.scheme in ("http", "https") and bool(parsed.host)
def _pages(params: dict[str, Any]) -> int:
def _pages(params: Mapping[str, Any]) -> int:
try:
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
except (TypeError, ValueError):
return DEFAULT_PAGES
def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes:
response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True,
headers={"User-Agent": USER_AGENT})
response.raise_for_status()
return response.content
# ---------------------------------------------------------------- jobindex
JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss"
JOBINDEX_PAGE = 20
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
#: against the titles its feeds answer with. 0 is all of Denmark.
JOBINDEX_AREAS: dict[int, str] = {
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
8: "Aarhus", 7: "Aalborg",
}
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
#: What a Danish ad's town line says about where the work is done:
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def _remote(text: str) -> str | None:
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
def _text(fragment: str) -> str:
"""HTML to plain text. Jobindex breaks titles after a slash with a
zero-width space, which would otherwise end up in `title_key`."""
return html.unescape(strip_html(fragment)).replace("\u200b", "").strip()
def parse_jobindex(body: bytes) -> list[Row]:
return _jobindex_page(body)[0]
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
"""One page of a Jobindex RSS search, and how many items it had before
any was dropped. An item is `<title>` "Role, Company",
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
("robot job") has no logo, so its company is what follows the title's
last comma, or '' when there is none. A body with no `<channel>` is not
a search result and raises, since an empty list would read as a search
that ran out and close everything it listed before."""
root = ET.fromstring(body)
channel = root.find("channel")
if channel is None:
raise ValueError("jobindex answered without an RSS channel")
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
search_area = area_match.group(1).strip() if area_match else ""
rows = []
items = channel.findall("item")
for item in items:
link = (item.findtext("link") or "").strip()
full_title = (item.findtext("title") or "").strip()
if not full_title or not _web_url(link):
continue
description = item.findtext("description") or ""
h4 = _H4.search(description)
title = _text(h4.group(1)) if h4 else full_title
alt = _ALT.search(description)
company = html.unescape(alt.group(1)).strip() if alt else ""
if not company and full_title.startswith(title + ", "):
company = full_title[len(title) + 2:].strip()
area = _AREA.search(description)
posted = None
if pub := item.findtext("pubDate"):
try:
posted = parsedate_to_datetime(pub)
except (TypeError, ValueError):
posted = None
place = _text(area.group(1)) if area else search_area
rows.append({
"id": link.rstrip("/").rsplit("/", 1)[-1],
"title": title or full_title,
"company": company,
"place": place,
"country": "DK",
"remote": _remote(f"{place} {title}"),
"url": link,
"posted": posted,
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
"salary_min": None, "salary_max": None, "currency": "",
})
return rows, len(items)
def search_jobindex(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, newest first, until a short page or the page
limit. Returns the rows and whether the results ran out: a page with
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
a dropped item does not pass for the end. Jobindex states no total, so a
search that read the page limit full is never complete."""
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
if params.get("area"):
query["geoareaid"] = params["area"]
rows: list[Row] = []
for page in range(1, _pages(params) + 1):
got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
rows += got
if items < JOBINDEX_PAGE:
return rows, True
return rows, False
# ------------------------------------------------------------------ adzuna
ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
ADZUNA_PAGE = 50
#: Adzuna's countries in Europe, and the currency its salaries are in.
ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
def parse_adzuna(body: bytes, country: str) -> list[Row]:
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
and snippets carry `<strong>` highlights. A salary Adzuna estimated
(`salary_is_predicted`) is not the posting's, so it is dropped."""
return _adzuna_rows(json.loads(body), country)
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
rows = []
for job in data.get("results") or []:
title = _text(job.get("title") or "")
url = (job.get("redirect_url") or "").strip()
if not title or not _web_url(url):
continue
predicted = str(job.get("salary_is_predicted") or "0") == "1"
rows.append({
"id": str(job.get("id") or ""),
"title": title,
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
"place": ((job.get("location") or {}).get("display_name") or "").strip(),
"country": country.upper(),
"remote": _remote(title),
"url": url,
"posted": _parse_iso(job.get("created")),
"description": _text(job.get("description") or ""),
"salary_min": None if predicted else job.get("salary_min"),
"salary_max": None if predicted else job.get("salary_max"),
"currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""),
})
return rows
def _parse_iso(value: Any) -> datetime | None:
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
except ValueError:
return None
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
def search_adzuna(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, until a short page, the stated `count` or
the page limit. Returns the rows and whether they are all `count` says
match, or, on an answer with no `count`, whether a page came back short."""
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
if not (app_id and app_key):
raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, "
"or ADZUNA_APP_ID and ADZUNA_APP_KEY")
country = str(params.get("country") or "").lower()
if country not in ADZUNA_COUNTRIES:
raise ValueError(f"adzuna country {country!r} is not one of "
f"{', '.join(sorted(ADZUNA_COUNTRIES))}")
query: dict[str, Any] = {"app_id": app_id, "app_key": app_key,
"results_per_page": ADZUNA_PAGE, "sort_by": "date",
"what": str(params.get("q") or "")}
if params.get("where"):
query["where"] = params["where"]
rows: list[Row] = []
seen: set[str] = set()
count: Any = None
ran_out = False
for page in range(1, _pages(params) + 1):
url = ADZUNA_SEARCH.format(country=country, page=page)
data = json.loads(_get(client, url, query))
results = data.get("results") or []
count = data.get("count", count)
seen |= {str(job.get("id") or "") for job in results}
rows += _adzuna_rows(data, country)
if len(results) < ADZUNA_PAGE:
ran_out = True
break
if isinstance(count, int) and len(seen) >= count:
break
# Adzuna states how many results match; the search is complete when it
# read that many distinct ones. Without a count, a short page is the end.
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
PLATSBANKEN_SEARCH = "https://jobsearch.api.jobtechdev.se/search"
PLATSBANKEN_PAGE = 100
#: The API refuses an `offset` above 2000, which no page limit reaches.
PLATSBANKEN_MAX_OFFSET = 2000
#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region`
#: concepts (which also list regions outside Sweden). "" is all of Sweden and
#: is not a key here.
PLATSBANKEN_REGIONS: dict[str, str] = {
"DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län",
"K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län",
"wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län",
"MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län",
"tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län",
"CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län",
"s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län",
"EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län",
"NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län",
"zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län",
"oLT3_Q9p_3nn": "Östergötlands län",
}
def parse_platsbanken(body: bytes) -> list[Row]:
"""One page of JobTech's `/search`: `{"total": {"value"}, "hits": [...]}`.
A hit's `webpage_url` is Platsbanken's own page for the ad. `remote` is
read from the headline and from `workplace_model`, which the ads mostly
leave empty."""
return _platsbanken_rows(json.loads(body))
def _platsbanken_rows(data: dict[str, Any]) -> list[Row]:
rows = []
for hit in data.get("hits") or []:
title = _text(hit.get("headline") or "")
url = (hit.get("webpage_url") or "").strip()
if not title or not _web_url(url):
continue
address = hit.get("workplace_address") or {}
model = (hit.get("workplace_model") or {}).get("label") or ""
rows.append({
"id": str(hit.get("id") or ""),
"title": title,
"company": ((hit.get("employer") or {}).get("name") or "").strip(),
"place": (address.get("municipality") or address.get("region") or "").strip(),
"country": "SE",
"remote": _remote(f"{title} {model}"),
"url": url,
"posted": _parse_iso(hit.get("publication_date")),
"description": _text((hit.get("description") or {}).get("text") or ""),
"salary_min": None, "salary_max": None, "currency": "",
})
return rows
def search_platsbanken(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, newest first, until a short page, the stated
total or the page limit. Returns the rows and whether they are all the
`total` says match, as for Adzuna. `region` is a key of
`PLATSBANKEN_REGIONS`, or empty for all of Sweden."""
region = str(params.get("region") or "")
if region and region not in PLATSBANKEN_REGIONS:
raise ValueError(f"platsbanken region {region!r} is not one of the ids in "
"PLATSBANKEN_REGIONS")
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "pubdate-desc",
"limit": PLATSBANKEN_PAGE}
if region:
query["region"] = region
rows: list[Row] = []
seen: set[str] = set()
total: Any = None
ran_out = False
for page in range(_pages(params)):
offset = page * PLATSBANKEN_PAGE
if offset > PLATSBANKEN_MAX_OFFSET:
break
data = json.loads(_get(client, PLATSBANKEN_SEARCH, query | {"offset": offset}))
hits = data.get("hits") or []
total = (data.get("total") or {}).get("value", total)
seen |= {str(hit.get("id") or "") for hit in hits}
rows += _platsbanken_rows(data)
if len(hits) < PLATSBANKEN_PAGE:
ran_out = True
break
if isinstance(total, int) and len(seen) >= total:
break
return rows, (len(seen) == total) if isinstance(total, int) else ran_out
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
tuple[list[Row], bool]]] = {
"jobindex": search_jobindex,
"adzuna": search_adzuna,
"platsbanken": search_platsbanken,
}
def _limits(params: Mapping[str, Any]) -> _runner.Limits:
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
def provider_of(target: str) -> str:
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
provider, sep, name = target.partition(":")
return provider if sep and name.strip() and provider in _SEARCHES else ""
return provider if sep and name.strip() and provider in readers.PROVIDERS else ""
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
"""One search read through the runner: its provider and its listing.
Raises ValueError for an unknown provider or a missing key, before any
request."""
provider = provider_of(target)
if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of "
f"{', '.join(readers.PROVIDERS)}")
module = readers.PROVIDERS[provider]
slug = target.partition(":")[2]
ask = _runner.ask_for(module, params, keys=keys)
get = _runner.getter(client, module.allowed(slug), provider)
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
def run_search(target: str, params: dict[str, Any],
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
"""One search, outside any pull: for `app searches add`, which runs a
search once before saving it."""
provider = provider_of(target)
if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of "
f"{', '.join(PROVIDERS)}")
search once before saving it. `keys` is the config's `sources.search`.
Each row also carries `place`, the display line of its places."""
with httpx.Client() as client:
return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {})
_module, listing = _read(client, target, params, keys)
rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows]
return rows, listing.complete
# -------------------------------------------------------------- the plugin
class SearchSource(Source):
name = "search"
description = "Saved searches on cross-company job boards: Jobindex, Adzuna, Platsbanken."
description = "Saved searches on cross-company job boards: " + ", ".join(
module.LABEL for module in readers.PROVIDERS.values()) + "."
#: Entries are `{"url": target, "label": name, "params": {...}}`.
subscription_field = "searches"
@@ -447,37 +135,32 @@ class SearchSource(Source):
log.warning("search: %r is not provider:name; skipped", target)
continue
try:
rows, complete = _SEARCHES[provider](
client, self.params_for(target), self.options.get(provider) or {})
_module, listing = _read(client, target, self.params_for(target),
self.options)
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
continue
seen_ids = []
for row in rows:
source_id = f"{provider}:{row['id']}"
for row in listing.rows:
source_id = f"{provider}:{row['source_id']}"
seen_ids.append(source_id)
description = row.get("description") or ""
yield Item(
source_id=source_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
subscription=label,
kind="role",
raw={
"company": row["company"],
"place": row["place"],
"country": row["country"],
"remote": row["remote"],
"salary_min": row["salary_min"],
"salary_max": row["salary_max"],
"currency": row["currency"],
raw=_runner.raw_of(row) | {
"company": row.get("company") or "",
"provider": provider,
"search": target,
},
)
self.state[f"search:ok:{target}"] = run_marker
self.state[f"search:complete:{target}"] = "1" if complete else ""
self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
-11
View File
@@ -1,11 +0,0 @@
User-agent: *
Disallow: /applybutton/
Disallow: /talentcommunity/
Disallow: /mobile/talentcommunity/
Disallow: /emailsubscribe/
Disallow: /email/image/
Disallow: /services/
Disallow: /preapply/
Disallow: /error
Disallow: /unsubscribe/
Disallow: /reset/
+6 -5
View File
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats # noqa: E402
from plugins.readers import smartrecruiters # noqa: E402
from plugins.readers import _runner, smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -487,7 +487,7 @@ def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
monkeypatch: pytest.MonkeyPatch) -> None:
clock = iter(range(0, 1000, 10))
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
monkeypatch.setattr(_runner.time, "monotonic", lambda: float(next(clock)))
monkeypatch.setattr(ats, "DEADLINE", 25.0)
body = (b"x" for _ in range(100))
with _client(lambda request: httpx.Response(200, content=body)) as client, \
@@ -752,12 +752,13 @@ def test_teamtailor_is_read_through_the_streamed_get_and_a_redirect_elsewhere_is
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
def fake(client: object, target: str) -> object:
def fake(*args: object, **kwargs: object) -> _runner.Listing:
if isinstance(answer, Exception):
raise answer
return answer
rows, complete = answer # type: ignore[misc]
return _runner.Listing(rows, complete, [row["source_id"] for row in rows])
monkeypatch.setattr(ats, "_fetch_rows", fake)
monkeypatch.setattr(_runner, "read_listing", fake)
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
"run": "run-1"})
return source
-13
View File
@@ -155,19 +155,6 @@ def test_an_empty_board_is_complete_and_a_401_is_a_failure(web: dict[str, Any])
ats._fetch_rows(client, "hibob:liquidia")
def test_the_robots_txt_that_is_the_apps_own_page_allows(web: dict[str, Any]) -> None:
"""The host answers /robots.txt with its single-page app's HTML, a 200
with no rules."""
web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \
b'<!DOCTYPE html><html><head><title>Careers</title></head><body></body></html>'
with httpx.Client() as client:
ats._check_robots(client, "hibob:dixa")
web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \
b"User-agent: *\nDisallow: /\n"
with pytest.raises(ats._Disallowed):
ats._check_robots(client, "hibob:dixa")
# ------------------------------------------------------- slugs and hosts
def test_the_module_is_registered_as_a_paged_host() -> None:
-19
View File
@@ -133,25 +133,6 @@ def test_a_429_after_the_first_page_ends_the_read_incomplete(web: dict[str, Any]
assert len(rows) == 5 and not complete
def test_robots_are_read_from_the_api_host_whose_404_allows(web: dict[str, Any]) -> None:
request = httpx.Request("GET", "https://api.hr-manager.net/robots.txt")
web["answers"]["https://api.hr-manager.net/robots.txt"] = httpx.HTTPStatusError(
"404", request=request, response=httpx.Response(404, request=request))
state: dict[str, str] = {}
with httpx.Client() as client:
ats._check_robots(client, "hrmanager:stsgroup_tr", state)
assert '"allowed": true' in state["ats:robots:hrmanager:stsgroup_tr"]
assert web["asked"] == ["https://api.hr-manager.net/robots.txt"]
def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide(
web: dict[str, Any]) -> None:
web["answers"]["https://api.hr-manager.net/robots.txt"] = \
b"User-agent: *\nDisallow: /JobPortal.svc/\n"
with httpx.Client() as client, pytest.raises(ats._Disallowed):
ats._check_robots(client, "hrmanager:stsgroup_tr")
# ------------------------------------------------------- slugs and hosts
def test_the_module_is_registered_as_a_paged_host() -> None:
+49 -6
View File
@@ -20,7 +20,9 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext, Tally # noqa: E402
from plugins import ats # noqa: E402
from app.models import Application # noqa: E402
from plugins import readers # noqa: E402
from plugins.readers import _runner # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.jobs import cascade, pull # noqa: E402
from app.storage import Store # noqa: E402
@@ -202,14 +204,16 @@ def _row(job_id: str) -> dict[str, object]:
"description": "", "remote": None, "salary_text": ""}
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
def fake(client: object, target: str) -> object:
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None:
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
with `filtered`, a read the board's filter narrowed."""
def fake(*args: object, **kwargs: object) -> _runner.Listing:
if isinstance(answer, Exception):
raise answer
return answer
rows, complete = answer # type: ignore[misc]
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered)
monkeypatch.setattr(ats, "_fetch_rows", fake)
monkeypatch.setattr(_runner, "read_listing", fake)
def _pull(store: Store) -> dict[str, object]:
@@ -275,6 +279,45 @@ def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
assert pull.failing_boards(acme) == {}
def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
_pull(acme)
uid = {r.source_id: r.uid for r in acme.roles()}
agent = acme.save_agent("Hunt")
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
_answer(monkeypatch, ([_row("1")], True), filtered=True)
result = _pull(acme)
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
assert result["closed"] == 1
assert acme.state["ats:filtered:workable:acme"] == "1"
# Read without the filter again, the board closes as before.
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
assert _open(acme) == {"workable:acme:1"}
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
board="workable", params={"countries": ["dk"], "remote": True})
asked: list[object] = []
def fake(module: object, slug: str, get: object, ask: object, **kwargs: object
) -> _runner.Listing:
asked.append(ask)
return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type]
monkeypatch.setattr(_runner, "read_listing", fake)
_pull(store)
[ask] = asked
declared = readers.HOSTS["workable"].READS.filters.declared()
assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined]
# ------------------------------------------------------ one pull at a time
def _ago(**delta: float) -> str:
+11 -193
View File
@@ -1,6 +1,6 @@
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
every host: the registry and its contract check, the fetch a host is handed,
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
the per-role detail reads, and Lever's EU instance. No network —
a fake host stands in where a behaviour belongs to no real host yet."""
from __future__ import annotations
@@ -11,7 +11,7 @@ import subprocess
import sys
import types
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -23,6 +23,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats, readers # noqa: E402
from plugins.readers import _runner # noqa: E402
from plugins.readers import lever # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -242,11 +243,9 @@ def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatc
# ------------------------------------------------------- a fake host
def _fake_host(*, detail: Callable[..., Any] | None = None,
robots: bool = False) -> types.ModuleType:
def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
"""A host `fake` whose feed is a JSON list of ids, each role with a
teaser for a description, and, with `detail`, a per-role page; with
`robots`, its listing is the page robots.txt must allow."""
teaser for a description, and, with `detail`, a per-role page."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
module.slug_ok = lambda slug: slug.isalnum()
@@ -260,8 +259,6 @@ def _fake_host(*, detail: Callable[..., Any] | None = None,
for i in json.loads(body)]
if detail is not None:
module.detail = detail
if robots:
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
return module
@@ -411,8 +408,8 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
ids stay, and the next pull does not read them in detail again."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]'})
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False))
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
@@ -511,7 +508,7 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
"title": "Another title"}
caplog.set_level("DEBUG", logger=ats.log.name)
caplog.set_level("DEBUG", logger=_runner.log.name)
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
assert (item.body, item.title, item.published_at) == (
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
@@ -519,8 +516,9 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
assert "department, salary_text, title left out" in caplog.text
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
"salary_min", "salary_max", "posted"}
assert set(ats.DETAIL_FIELDS) == {"description", "location", "places", "teaser", "remote",
"country", "employment", "currency", "salary_min",
"salary_max", "posted"}
def _paged_host() -> types.ModuleType:
@@ -583,186 +581,6 @@ def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
assert "ats:detailed:fake:acme" not in source.state
# ------------------------------------------------------------ robots.txt
ROBOTS = "https://jobs.fake.example/robots.txt"
def test_robots_that_allow_the_page_let_the_board_be_read(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
@pytest.mark.parametrize("robots", [
b"User-agent: *\nDisallow: /acme/\n",
# A rule for this app by name counts as well as the one for everyone.
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
.encode(),
])
def test_robots_that_disallow_the_page_fail_the_board(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert items == []
assert web["asked"] == [ROBOTS] # the listing is never fetched
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
assert "ats:ok:fake:acme" not in state
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
host = _fake_host(robots=True)
items, _state = _run(monkeypatch, host)
assert len(items) == 1 # paths are case-sensitive by default
host.ROBOTS_ANY_CASE = True
web["asked"].clear()
items, state = _run(monkeypatch, host)
assert items == []
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
@pytest.mark.parametrize(("robots", "page", "allowed"), [
# The longest matching rule wins, whatever the order of the lines.
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
# An allow wins a tie.
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
# `*` inside a path, and the query counts as part of it.
("User-agent: *\nDisallow: /*company\n",
"https://career5.successfactors.eu/career?company=x", False),
# `$` anchors the end.
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
# An empty Disallow allows.
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
# A byte order mark and CRLF line ends.
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
# Paths are case-sensitive.
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
# A group for this app by name counts, a group for someone else does
# not, and a token matches whole, not as a substring.
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
"https://x.example/jobs", False),
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
# Two agents share one group, and a comment is no rule.
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
"https://x.example/jobs/1", False),
# A group for this app that allows does not undo the `*` group's refusal.
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
"https://x.example/jobs", False),
# Rules before any user-agent line belong to no group.
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
])
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
assert ats._robots_allows(robots.encode(), page) is allowed
def test_a_robots_verdict_is_kept_for_a_day(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
module = _fake_host(robots=True)
_items, state = _run(monkeypatch, module)
assert web["asked"] == [ROBOTS]
# Within the day the kept verdict holds, even though the site changed.
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert (items, web["asked"]) == ([], [])
# A day on, robots.txt is read again.
kept = json.loads(state["ats:robots:fake:acme"])
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
state["ats:robots:fake:acme"] = json.dumps(kept)
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
assert len(items) == 1 and "ats:failing:fake:acme" not in state
@pytest.mark.parametrize("code", [404, 410, 401, 403])
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert len(items) == 1
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
@pytest.mark.parametrize("failure", [
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
])
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
items, state = _run(monkeypatch, _fake_host(robots=True))
assert items == [] and web["asked"] == [ROBOTS]
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
"robots.txt could not be read"
assert "ats:robots:fake:acme" not in state
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
where a robots.txt that disallows everything is served."""
def handler(request: httpx.Request) -> httpx.Response:
if str(request.url) == ROBOTS:
return httpx.Response(301, headers={"location": location})
if request.url.path == "/robots.txt":
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
return httpx.Response(200, text='["1"]')
return handler
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Disallowed):
ats._check_robots(client, "fake:acme", {})
@pytest.mark.parametrize("location", [
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
"http://www.jobs.fake.example/robots.txt",
])
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
ats._check_robots(client, "fake:acme", {})
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
pytest.raises(ats._Refused):
ats._get(client, ROBOTS, "fake", "acme")
def test_fetch_board_honours_robots_txt(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
"https://jobs.fake.example/acme/list": b'["1"]'})
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
with pytest.raises(ats._Disallowed) as caught:
ats.fetch_board("fake:acme")
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
# ----------------------------------------------------------------- Lever EU
def test_a_lever_slug_may_name_the_eu_instance() -> None:
+420
View File
@@ -0,0 +1,420 @@
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
subscription resolves to, series and their union, whole and paged listings,
completeness from the declaration, the sweep across pulls, the 10% guard,
the facets kept in state, the detail reads and the `raw` an item carries. No
network: `get` is a dict of URL -> body or exception."""
from __future__ import annotations
import json
import sys
import types
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, Filters, Key, Reads, _runner # noqa: E402
NO_DELAY = _runner.Limits(page_delay=0)
def _status(code: int) -> httpx.HTTPStatusError:
request = httpx.Request("GET", "https://jobs.fake.example/")
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
class Web:
"""A `get` answering from a dict of URL -> body or exception."""
def __init__(self, answers: dict[str, Any] | None = None) -> None:
self.answers = answers or {}
self.asked: list[str] = []
def __call__(self, url: str, **kwargs: Any) -> bytes:
self.asked.append(url)
answer = self.answers.get(url, _status(404))
if isinstance(answer, BaseException):
raise answer
return answer
def _row(i: str, **extra: Any) -> dict[str, Any]:
return {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
"places": [{"text": "Berlin", "remote": None}]} | extra
def _whole(**reads: Any) -> Any:
"""A declared whole-listing board: the feed is a JSON list of ids, one
feed per `ask.countries` value."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
"complete": "whole", "places": "many", "robots": "test"} | reads))
module.feed_url = lambda slug, ask: (f"https://jobs.fake.example/{slug}/list"
+ "".join(f"?c={c}" for c in ask.countries))
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
return module
def _paged(pages: list[list[str]], total: int | None = None, **reads: Any) -> Any:
"""A declared paged board over `pages`, the cursor being the page index."""
module = _whole(**({"listing": "paged", "complete": "stated" if total is not None
else "end"} | reads))
del module.feed_url, module.parse
module.asked = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
index = int(cursor or 0)
module.asked.append((index, ask.countries))
body = get(f"https://jobs.fake.example/{slug}/page/{index}")
ids = json.loads(body) if body else pages[index]
more = index + 1 < len(pages) and bool(ids)
return ids, [{"id": i} for i in ids], total, index + 1 if more else None
module.page = page
module.rows = lambda ids, slug: [_row(i) for i in ids]
return module
def _pages_web(pages: list[list[str]]) -> Web:
return Web({f"https://jobs.fake.example/acme/page/{n}": json.dumps(ids).encode()
for n, ids in enumerate(pages)})
# -------------------------------------------------------------------- ask
def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
board = _whole(filters=Filters(country="one"))
ask = _runner.ask_for(board, {"countries": ["dk", "se"], "remote": True, "places": ["X"]})
assert (ask.countries, ask.remote, ask.places, ask.params) == (("DK", "SE"), None, (), {})
# The params override the reach per dimension; the reach fills the rest.
both = _whole(filters=Filters(country="many", remote="many"))
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
assert (ask.countries, ask.remote) == (("DE",), True)
# A module without READS reads the whole board.
assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
monkeypatch: pytest.MonkeyPatch) -> None:
module = _whole(scope="search", listing="paged", complete="stated", text="excerpt",
filters=Filters(query="one", place="one"),
key=Key(name="fake", fields=("id", "secret"), env=("FAKE_ID", "FAKE_SECRET")))
module.PARAMS = {"q": "query", "area": "place"}
monkeypatch.delenv("FAKE_ID", raising=False)
monkeypatch.setenv("FAKE_SECRET", "s")
ask = _runner.ask_for(module, {"q": "python", "area": 1, "pages": 2},
keys={"fake": {"id": "i"}})
assert (ask.query, ask.places) == ("python", ("1",))
assert ask.params == {"q": "python", "area": 1, "pages": 2,
"key": {"id": "i", "secret": "s"}}
with pytest.raises(ValueError, match=r"sources\.search\.fake: id, secret, or FAKE_ID"):
_runner.ask_for(module, {"q": "x"}, keys={})
def test_series_are_one_a_value_for_one_and_one_for_many() -> None:
ask = Ask(countries=("DK", "SE"), places=("Aarhus",))
one = _whole(filters=Filters(country="one", place="many"))
assert [(a.countries, a.places) for a in _runner.series(one, ask)] == [
(("DK",), ("Aarhus",)), (("SE",), ("Aarhus",))]
many = _whole(filters=Filters(country="many"))
assert _runner.series(many, ask) == [ask]
def test_ids_filters_are_mapped_through_the_facets() -> None:
module = _whole(filters=Filters(country="ids"))
module.facets = lambda get, slug: {}
ask = Ask(countries=("DK", "XX"))
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
assert mapped.countries == ("h-1",)
# None known: the host has nothing in reach, and the read is empty.
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
facets={"country": {}})
assert (listing.rows, listing.complete, listing.filtered) == ([], True, True)
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
module = _whole(filters=Filters(country="ids"))
calls: list[str] = []
module.facets = lambda get, slug: calls.append(slug) or {"country": {"DK": "7"}}
state: dict[str, str] = {}
assert _runner.facets_for(module, "acme", Web(), Ask(), state, "k") == {}
for _ in range(2):
found = _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k")
assert found == {"country": {"DK": "7"}}
assert calls == ["acme"]
kept = json.loads(state["k"])
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
state["k"] = json.dumps(kept)
_runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k")
assert calls == ["acme", "acme"]
# ---------------------------------------------------------------- listing
def test_a_whole_listing_reads_one_feed_a_series_and_unions_by_id() -> None:
module = _whole(filters=Filters(country="one"))
web = Web({"https://jobs.fake.example/acme/list?c=DK": b'["1", "2"]',
"https://jobs.fake.example/acme/list?c=SE": b'["2", "3"]'})
listing = _runner.read_listing(module, "acme", web, Ask(countries=("DK", "SE")),
limits=NO_DELAY)
assert [r["source_id"] for r in listing.rows] == ["1", "2", "3"]
assert (listing.complete, listing.filtered, listing.ids) == (True, True, ["1", "2", "3"])
def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
"""Greenhouse's shape: the feed with the text, and one without it."""
module = _whole()
module.fallback_url = lambda slug: f"https://jobs.fake.example/{slug}/small"
web = Web({"https://jobs.fake.example/acme/list": _runner.TooLarge("over"),
"https://jobs.fake.example/acme/small": b'["1"]'})
listing = _runner.read_listing(module, "acme", web)
assert ([r["source_id"] for r in listing.rows], listing.complete) == (["1"], True)
# A reader without a fallback fails.
del module.fallback_url
with pytest.raises(_runner.TooLarge):
_runner.read_listing(module, "acme", web)
def test_a_module_without_reads_gets_the_older_signatures() -> None:
module: Any = types.ModuleType("old")
module.NAME = "old"
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
listing = _runner.read_listing(module, "acme",
Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
Ask(countries=("DK",)))
assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
pages = [["1", "2"], ["3"]]
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2", "3"], True)
short = _runner.read_listing(_paged(pages, total=4), "acme", _pages_web(pages),
limits=NO_DELAY)
assert not short.complete
def test_the_page_cap_leaves_a_board_incomplete() -> None:
pages = [["1"], ["2"], ["3"]]
listing = _runner.read_listing(_paged(pages), "acme", _pages_web(pages),
limits=_runner.Limits(max_pages=2, page_delay=0))
assert (listing.ids, listing.complete, listing.sweep) == (["1", "2"], False, None)
def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
"""A host that ignores the cursor answers page one again; without a total
the read would never end, so a page with nothing new stops it."""
pages = [["1", "2"], ["1", "2"], ["3"]]
module = _paged(pages)
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], False)
assert [index for index, _ in module.asked] == [0, 1]
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
pages = [["1"], ["2"]]
web = _pages_web(pages)
web.answers["https://jobs.fake.example/acme/page/1"] = _status(429)
listing = _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1"], False)
web.answers["https://jobs.fake.example/acme/page/0"] = _status(429)
with pytest.raises(httpx.HTTPStatusError):
_runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY)
def test_a_search_is_never_complete_when_it_says_so() -> None:
pages = [["1"]]
module = _paged(pages, scope="search", complete="never", text="excerpt")
assert not _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY).complete
def test_paged_series_each_read_to_the_end_and_union() -> None:
pages = [["1", "2"]]
module = _paged(pages, filters=Filters(country="one"))
listing = _runner.read_listing(module, "acme", _pages_web(pages),
Ask(countries=("DK", "SE")), limits=NO_DELAY)
assert module.asked == [(0, ("DK",)), (0, ("SE",))]
assert (listing.ids, listing.complete) == (["1", "2"], True)
# ------------------------------------------------------------------ sweep
def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
pages = [["1"], ["2"], ["3"], ["4"], ["5"]]
module = _paged(pages, total=5)
web, state = _pages_web(pages), {}
limits = _runner.Limits(max_pages=2, page_delay=0)
seen: list[tuple[list[str], bool]] = []
for _pull in range(3):
listing = _runner.read(state, "ats", "fake:acme", module, "acme", web, limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
seen.append((listing.ids, listing.complete))
assert seen == [(["1", "2"], False), (["3", "4"], False), (["1", "2", "3", "4", "5"], True)]
assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4]
assert "ats:sweep:fake:acme" not in state
assert json.loads(state["ats:count:fake:acme"])["count"] == 5
def test_a_sweep_starts_again_when_the_ask_changed() -> None:
pages = [["1"], ["2"], ["3"]]
module = _paged(pages, filters=Filters(country="many"))
web, state = _pages_web(pages), {}
limits = _runner.Limits(max_pages=2, page_delay=0)
first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits)
_runner.stamp(state, "ats", "t", "r", first, first.ids)
module.asked.clear()
_runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), limits=limits)
assert module.asked[0][0] == 0
def test_a_board_at_its_ceiling_is_not_swept() -> None:
pages = [["1", "2"], ["3", "4"], ["5"]]
module = _paged(pages, total=5, page_size=2, ceiling=4)
listing = _runner.read({}, "ats", "t", module, "acme", _pages_web(pages),
limits=_runner.Limits(max_pages=2, page_delay=0))
assert (listing.complete, listing.sweep) == (False, None)
def test_a_search_is_not_swept() -> None:
pages = [["1"], ["2"]]
module = _paged(pages, total=2)
listing = _runner.read_listing(module, "acme", _pages_web(pages), can_sweep=False,
limits=_runner.Limits(max_pages=1, page_delay=0))
assert listing.sweep is None
# ----------------------------------------------------------- the 10% guard
def test_a_suspect_read_far_under_the_last_count_fails() -> None:
module = _whole(empty="suspect")
web = Web({"https://jobs.fake.example/acme/list": json.dumps(
[str(n) for n in range(20)]).encode()})
state: dict[str, str] = {}
listing = _runner.read(state, "ats", "t", module, "acme", web)
_runner.stamp(state, "ats", "t", "r", listing, listing.ids)
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
_runner.read(state, "ats", "t", module, "acme", web)
# A reader whose empty answer is trusted reads it as it is.
module.READS = module.READS.model_copy(update={"empty": "trusted"})
assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"]
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
"""A board read whole last time and with a filter now lists far fewer
roles; that is the filter, not a failure."""
module = _whole(filters=Filters(country="one"))
web = Web({"https://jobs.fake.example/acme/list": json.dumps(
[str(n) for n in range(50)]).encode(),
"https://jobs.fake.example/acme/list?c=DK": b'["1"]'})
state: dict[str, str] = {}
listing = _runner.read(state, "ats", "t", module, "acme", web)
_runner.stamp(state, "ats", "t", "r", listing, listing.ids)
filtered = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)))
assert filtered.ids == ["1"] and filtered.filtered
# ------------------------------------------------------ through ATSSource
@pytest.fixture
def board(monkeypatch: pytest.MonkeyPatch) -> Any:
pages = [["1"], ["2"], ["3"]]
module = _paged(pages, total=3, empty="suspect")
monkeypatch.setitem(readers.HOSTS, "fake", module)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
web = _pages_web(pages)
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: web(url))
module.web = web
return module
def _source(state: dict[str, str], params: dict[str, Any] | None = None) -> ats.ATSSource:
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
"params": params or {}}], "run": "r"})
source.state = state
return source
def test_ats_source_sweeps_and_stamps_every_id_once_the_sweep_ends(board: Any) -> None:
state: dict[str, str] = {}
items = list(_source(state).fetch())
assert [i.source_id for i in items] == ["fake:acme:1", "fake:acme:2"]
assert state["ats:complete:fake:acme"] == ""
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == 2
items = list(_source(state).fetch())
assert [i.source_id for i in items] == ["fake:acme:3"]
assert state["ats:complete:fake:acme"] == "1"
assert json.loads(state["ats:ids:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
def test_ats_source_records_a_suspect_read_as_failing(board: Any) -> None:
state = {"ats:count:fake:acme": json.dumps(
{"asks": _runner.fingerprint([Ask()]), "count": 100})}
board.web.answers = {"https://jobs.fake.example/acme/page/0": b"[]"}
assert list(_source(state).fetch()) == []
assert "under 10%" in json.loads(state["ats:failing:fake:acme"])["reason"]
assert "ats:ok:fake:acme" not in state
def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
board.READS = board.READS.model_copy(update={"filters": Filters(country="many")})
state: dict[str, str] = {}
list(_source(state, {"countries": ["dk"]}).fetch())
assert board.asked[0] == (0, ("DK",))
assert state["ats:filtered:fake:acme"] == "1"
# -------------------------------------------------------------------- raw
def test_raw_keeps_every_stated_key_and_the_places() -> None:
row = _row("1", places=[{"text": "Berlin", "remote": False},
{"text": "Remote (DE)", "remote": True}],
teaser="Short.", seniority="senior", employment="full_time", country="DE",
description="Long.", posted=None, location="Berlin; Remote (DE)")
raw = _runner.raw_of(row)
assert raw["places"] == row["places"]
assert raw["place"] == "Berlin; Remote (DE)"
assert (raw["teaser"], raw["seniority"], raw["employment"], raw["country"]) == (
"Short.", "senior", "full_time", "DE")
assert raw["location"] == "Berlin; Remote (DE)"
assert not {"description", "title", "url", "posted", "source_id"} & set(raw)
assert (raw["department"], raw["salary_text"], raw["currency"], raw["salary_min"]) == (
"", "", "", None)
def test_a_row_without_places_has_its_location_as_the_one_place() -> None:
raw = _runner.raw_of({"source_id": "1", "title": "t", "url": "u",
"location": "Berlin, Munich"})
assert raw["places"] == [{"text": "Berlin, Munich", "remote": None}]
assert raw["place"] == "Berlin, Munich"
assert _runner.raw_of({"source_id": "1", "title": "t", "url": "u"})["places"] == []
def test_a_detail_location_without_places_becomes_the_one_place() -> None:
module = _whole(text="detail")
module.detail = lambda get, slug, row: {"location": "Aarhus", "employment": "contract",
"description": "Full."}
rows, done, failed = _runner.read_details(module, "fake", "acme", Web(), [_row("1")],
set(), set(), limits=NO_DELAY)
assert rows[0]["places"] == [{"text": "Aarhus", "remote": None}]
assert (rows[0]["employment"], rows[0]["description"]) == ("contract", "Full.")
assert (done, failed) == ({"fake:acme:1"}, set())
+160 -254
View File
@@ -1,15 +1,17 @@
"""The `search` plugin, `country_of`, and what the pull and the CLI do with
searches. No network: Jobindex's feed is a trimmed copy of a real one
(2026-09-23), Adzuna's is written from its documented response, Platsbanken's
is trimmed from a live answer (2026-09-29)."""
"""The `search` plugin over the runner and the provider readers, `country_of`,
and what the pull does with searches. No network: `_runner.fetch` is a stub,
and a fake provider stands in where the pull is what is tested. The readers'
own parsing is tested in `test_readers_<provider>.py`."""
from __future__ import annotations
import json
import sys
import types
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, ClassVar
from urllib.parse import parse_qs, urlsplit
import pytest
@@ -18,7 +20,8 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from plugins import search # noqa: E402
from plugins import readers, search # noqa: E402
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
@@ -111,246 +114,129 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) ->
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
# ----------------------------------------------------------------- parsers
# ----------------------------------------------------- read through the runner
def test_jobindex_reads_title_company_town_and_date() -> None:
rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes())
assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"]
paid, bare, robot = rows
# A paid ad: the company is its logo's alt text, the title its <h4>.
assert paid["company"] == "DSV"
assert paid["title"].startswith("Dangerous Goods Compliance Manager")
assert paid["place"] == "Hedehusene"
assert paid["country"] == "DK"
assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23"
assert paid["description"].startswith("Do you want to minimise")
# A crawled ad with no company after the title's comma has none.
assert bare["company"] == ""
assert bare["place"] == "København"
# A crawled ad names its company after the title.
assert robot["title"] == "Marketing Content Creator"
assert robot["company"] == "Clerk.IO ApS"
def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]:
"""`_runner.fetch` answering `answer(url)`, remembering the URLs asked."""
asked: list[str] = []
def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes:
assert allow(urlsplit(url).hostname), url
asked.append(url)
return answer(url)
monkeypatch.setattr(_runner, "fetch", fake_fetch)
monkeypatch.setattr(search, "PAGE_DELAY", 0)
return asked
def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None:
assert search._remote("Birkerød, Hybrid position") == "hybrid"
assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
assert search._remote("Remote, Danmark") == "remote"
assert search._remote("Hedehusene") is None
def _jobindex_page(url: str) -> bytes:
"""A full Jobindex page (20 items) whatever page is asked."""
page = int(parse_qs(urlsplit(url).query)["page"][0])
items = "".join(f"<item><title>Role {page}-{n}, Acme</title>"
f"<link>https://www.jobindex.dk/vis-job/r{page}x{n}</link>"
f"<description>&lt;p&gt;Teaser {n}&lt;/p&gt;</description></item>"
for n in range(20))
return f"<rss><channel><title>Jobindex</title>{items}</channel></rss>".encode()
def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None:
rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de")
assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused
first, predicted = rows
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
assert first["description"] == "Build our backend services in Python & Go."
assert first["company"] == "Acme GmbH"
assert first["place"] == "Berlin, Deutschland"
assert first["country"] == "DE"
assert first["remote"] == "remote"
assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR")
assert (predicted["salary_min"], predicted["currency"]) == (None, "")
@pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)])
def test_the_saved_pages_param_bounds_the_pages_read(
monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None:
urls = _web(monkeypatch, _jobindex_page)
params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else
{"pages": pages})
rows, complete = search.run_search("jobindex:cph", params)
assert len(urls) == asked and len(rows) == 20 * asked
assert not complete # Jobindex states no total: never complete
query = parse_qs(urlsplit(urls[0]).query)
assert (query["q"], query["geoareaid"]) == (["python"], ["1"])
def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
with pytest.raises(ValueError, match="app_id"):
search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None:
pages: list[int] = []
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
pages.append(params["page"])
return b"short" if params["page"] == 3 else b"full"
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setattr(search, "_jobindex_page",
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
# Stopped by the page limit with every page full: not complete.
pages.clear()
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert (pages, complete) == ([1, 2], False)
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body(
monkeypatch: pytest.MonkeyPatch) -> None:
"""A full page whose parser dropped one item still has 20 items, so the
search reads on instead of calling itself complete at 19 rows."""
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert not complete
_web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes())
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
"params": {"q": "x", "pages": 1}}]})
items = list(source.fetch())
paid = items[0]
assert paid.source_id == "jobindex:h1701165"
assert (paid.body, paid.excerpt) == ("", "")
assert paid.raw["teaser"].startswith("Do you want to minimise")
assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK"
assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}]
assert paid.raw["place"] == "Hedehusene"
assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph")
assert source.state["search:complete:jobindex:cph"] == ""
def test_a_jobindex_answer_without_a_channel_raises() -> None:
with pytest.raises(ValueError, match="channel"):
search.parse_jobindex(b"<rss version='2.0'></rss>")
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
asked: list[int] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
page = int(url.rsplit("/", 1)[1])
asked.append(page)
start = (page - 1) * search.ADZUNA_PAGE
def _adzuna(count: int | None, stock: int = 60) -> Any:
def answer(url: str) -> bytes:
page = int(urlsplit(url).path.rsplit("/", 1)[1])
start = (page - 1) * 50
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
for n in range(start, min(start + 50, stock))]
body: dict[str, Any] = {"results": results}
if count is not None:
body["count"] = count
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setenv("ADZUNA_APP_ID", "id")
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
return asked
return answer
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
def test_a_search_without_its_key_is_refused_before_any_request(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
urls = _web(monkeypatch, _adzuna(60))
with pytest.raises(ValueError, match="app_id"):
search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {})
assert urls == []
def test_the_key_comes_from_the_config_before_the_environment(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("ADZUNA_APP_ID", "env-id")
monkeypatch.setenv("ADZUNA_APP_KEY", "env-key")
urls = _web(monkeypatch, _adzuna(60))
search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
{"adzuna": {"app_id": "cfg-id"}})
query = parse_qs(urlsplit(urls[0]).query)
assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"])
assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1"
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)])
def test_an_adzuna_search_is_complete_when_it_read_its_count(
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
_adzuna(monkeypatch, count)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None:
_web(monkeypatch, _adzuna(count))
rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
{"adzuna": {"app_id": "i", "app_key": "k"}})
assert (len(rows), done) == (60, complete)
assert rows[0]["place"] == ""
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
#: Two hits of a live `/search?q=data science&region=CaRE_1nn_cSU` (2026-09-29),
#: trimmed to the fields the parser reads, and a third the way an ad with no
#: municipality and a remote headline comes.
PLATSBANKEN = {
"total": {"value": 32},
"hits": [
{"id": "31457942", "headline": "Postdoctoral Researcher",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942",
"publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59",
"description": {"text": "The European Spallation Source (ESS) is one of the largest.",
"text_formatted": "<p>The European Spallation Source (ESS)</p>"},
"employer": {"name": "European Spallation Source Eric"},
"application_details": {"url": "https://europeanspallationsource.se/careers"},
"workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"},
"workplace_address": {"municipality": "Lund", "region": "Skåne län",
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
{"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504",
"publication_date": "2026-09-03T15:46:06",
"description": {"text": "Robotics at ESS."},
"employer": {"name": "European Spallation Source Eric"},
"workplace_model": {"concept_id": None, "label": None},
"workplace_address": {"municipality": None, "region": "Skåne län",
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
{"id": "31400001", "headline": "Data scientist, distansarbete möjligt",
"webpage_url": "javascript:alert(1)", "workplace_address": {}},
{"id": "31400002", "headline": "Data scientist, distansarbete möjligt",
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002",
"employer": {"name": None}, "workplace_address": {}},
],
}
def test_platsbanken_reads_title_company_place_and_date() -> None:
rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode())
assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
first, no_town, remote = rows
assert first["title"] == "Postdoctoral Researcher"
assert first["company"] == "European Spallation Source Eric"
assert first["place"] == "Lund"
assert first["country"] == "SE"
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09"
assert first["description"].startswith("The European Spallation Source")
assert first["remote"] is None
assert no_town["place"] == "Skåne län" # the region when there is no municipality
assert remote["remote"] == "remote"
assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None)
def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None,
stock: int = 250) -> list[dict[str, Any]]:
"""Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request,
stating `total`. Returns the params of each request."""
asked: list[dict[str, Any]] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
asked.append(params)
start = params["offset"]
hits = [{"id": str(n), "headline": f"Role {n}",
"webpage_url": f"https://x.example/{n}"}
for n in range(start, min(start + params["limit"], stock))]
body: dict[str, Any] = {"hits": hits}
if total is not None:
body["total"] = {"value": total}
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
return asked
@pytest.mark.parametrize(("total", "pages", "found", "complete"), [
(250, 5, 250, True), # a short third page ends it
(300, 5, 250, False), # the stated total is more than there was
(250, 2, 200, False), # stopped by the page limit
(None, 5, 250, True), # no total: a short page is the end
])
def test_a_platsbanken_search_is_complete_when_it_read_its_total(
monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int,
complete: bool) -> None:
_platsbanken(monkeypatch, total)
rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (found, complete)
def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset(
def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 200, stock=200)
rows, done = search.search_platsbanken(
None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type]
def answer(url: str) -> bytes:
query = parse_qs(urlsplit(url).query)
start = int(query["offset"][0])
hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}",
"workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]}
for n in range(start, min(start + 100, 200))]
return json.dumps({"total": {"value": 200}, "hits": hits}).encode()
urls = _web(monkeypatch, answer)
rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"})
assert (len(rows), done) == (200, True)
assert [p["offset"] for p in asked] == [0, 100]
assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU"
assert asked[0]["limit"] == search.PLATSBANKEN_PAGE
offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls]
assert offsets == [["0"], ["100"]]
assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"]
assert rows[0]["place"] == "Lund; Malmö"
def test_a_platsbanken_search_without_a_region_sends_none(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 5, stock=5)
search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type]
assert "region" not in asked[0]
def test_a_platsbanken_search_never_asks_past_the_offset_limit(
monkeypatch: pytest.MonkeyPatch) -> None:
asked = _platsbanken(monkeypatch, 5000, stock=5000)
monkeypatch.setattr(search, "MAX_PAGES", 30)
_rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type]
assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET
assert not done
def test_a_platsbanken_search_refuses_an_unknown_region() -> None:
with pytest.raises(ValueError, match="region"):
search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type]
def test_platsbanken_regions_are_the_21_lan() -> None:
assert len(search.PLATSBANKEN_REGIONS) == 21
assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län"
def test_an_unknown_provider_is_refused() -> None:
with pytest.raises(ValueError, match="provider"):
search.run_search("indeed:x", {"q": "x"})
def test_provider_of() -> None:
@@ -369,15 +255,27 @@ class FakeConfig:
path = None
def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
def fake(client: object, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København",
"country": "DK", "remote": None, "url": f"https://x.example/{i}",
"posted": None, "description": "", "salary_min": None,
"salary_max": None, "currency": ""} for i in ids], complete
def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
"""A provider `fake` that lists `ids` in one page and states a total of
as many, or of one more when the search is to be incomplete."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL = "fake", "Fake"
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
places="one", fields=Fields(country="stated"), robots="test")
module.allowed = lambda slug: lambda host: host == "search.fake.example"
monkeypatch.setitem(search._SEARCHES, "jobindex", fake)
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
assert ask.params["q"]
return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None
module.page = page
module.rows = lambda decoded, slug: [
{"source_id": i, "title": f"Engineer {i}", "company": "Acme",
"url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}],
"country": "DK", "remote": None, "posted": None, "description": ""}
for i in decoded]
assert not readers.problems(module, "fake")
monkeypatch.setitem(readers.PROVIDERS, "fake", module)
def _pull(store: Store) -> dict[str, Any]:
@@ -388,36 +286,36 @@ def _pull(store: Store) -> dict[str, Any]:
def test_a_complete_search_closes_what_it_stopped_listing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend", "area": 1})
_stub_search(monkeypatch, ["h1", "h2"], complete=True)
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
_pull(store)
roles = {r.source_id: r for r in store.roles()}
assert set(roles) == {"jobindex:h1", "jobindex:h2"}
assert roles["jobindex:h1"].country == "DK"
assert roles["jobindex:h1"].subscription_id is not None
assert set(roles) == {"fake:h1", "fake:h2"}
assert roles["fake:h1"].country == "DK"
assert roles["fake:h1"].subscription_id is not None
_stub_search(monkeypatch, ["h2"], complete=True)
_fake_provider(monkeypatch, ["h2"], complete=True)
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
assert {r.source_id for r in store.roles()} == {"fake:h2"}
assert (result["searches"], result["closed"]) == (1, 1)
def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale(
def test_an_incomplete_search_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_stub_search(monkeypatch, ["h1", "h2"], complete=False)
_fake_provider(monkeypatch, ["h1", "h2"], complete=False)
_pull(store)
_stub_search(monkeypatch, ["h2"], complete=False)
_fake_provider(monkeypatch, ["h2"], complete=False)
_pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"}
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,))
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
_pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
assert {r.source_id for r in store.roles()} == {"fake:h2"}
def test_seed_searches_imports_once(store: Store) -> None:
@@ -434,24 +332,32 @@ def test_seed_searches_imports_once(store: Store) -> None:
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
# -------------------------------------------------------------------- cli
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
_stub_search(monkeypatch, ["h1"], complete=True)
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
_fake_provider(monkeypatch, ["h1"], complete=True)
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
"params": {"q": "x"}}]})
items = list(source.fetch())
assert [i.source_id for i in items] == ["jobindex:h1"]
assert items[0].raw["search"] == "jobindex:cph"
assert source.state["search:complete:jobindex:cph"] == "1"
assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'
assert [i.source_id for i in items] == ["fake:h1"]
assert items[0].raw["search"] == "fake:cph"
assert source.state["search:complete:fake:cph"] == "1"
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin",
"params": {"q": "x", "country": "de"}}]})
assert list(source.fetch()) == []
assert "search:ok:adzuna:berlin" not in source.state
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen",
board="jobindex", params={"q": "backend"})
_stub_search(monkeypatch, ["h1"], complete=False)
sub = store.add_subscription("search", "fake:cph", label="Copenhagen",
board="fake", params={"q": "backend"})
_fake_provider(monkeypatch, ["h1"], complete=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
store.update_subscription(sub.id, subscribed=False, fetch=False)