From 902089a5e240eac74c14bc38b7f7d7213fcaabf4 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 09:28:33 +0200 Subject: [PATCH] Runner: readers read from their declaration, ats and search thin over it plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 --- app/jobs/pull.py | 40 +- app/web/pages.py | 8 +- plugins/ats.py | 720 ++++--------------- plugins/readers/__init__.py | 28 +- plugins/readers/_runner.py | 712 ++++++++++++++++++ plugins/search.py | 445 ++---------- tests/fixtures/ats/successfactors-robots.txt | 11 - tests/test_ats.py | 11 +- tests/test_ats_hibob.py | 13 - tests/test_ats_hrmanager.py | 19 - tests/test_pull.py | 55 +- tests/test_readers.py | 204 +----- tests/test_runner.py | 420 +++++++++++ tests/test_search.py | 414 +++++------ 14 files changed, 1608 insertions(+), 1492 deletions(-) create mode 100644 plugins/readers/_runner.py delete mode 100644 tests/fixtures/ats/successfactors-robots.txt create mode 100644 tests/test_runner.py diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 46a0588..41d8819 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -13,7 +13,16 @@ on a successful parse (`ats:ok:`, `ats:complete:`, still leaves the boards it had already finished stamped, so one bad board in a batch of ten does not cost the other nine their closure. A board that answered but was not read to its stated total (a paged host that hit a 429 or the page -cap) stores what it read and closes nothing. +cap) stores what it read and closes nothing; a paged board over the page cap +is swept across pulls and closes once the sweep reached its end (see +`plugins/readers/_runner.py`). + +A board read with a filter (`ats:filtered:`, from its subscription's +`params`) lists only the roles in reach, so its close keeps open the roles a +person acted on: those with a vote or an application (`_close_unlisted`). A +filtered read, or one of a reader that declares `empty="suspect"`, that lists +under a tenth of what the same read listed last time is recorded by the +runner as a failure, not as a list to close against. A search closes the same way when it listed everything that matches (see `plugins/search.py` for how each provider knows). One that did not closes its @@ -175,7 +184,11 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}] plugin = SearchSource(options) else: - options["boards"] = [{"url": target, "label": sub.company or sub.label}] + # A board's params are its filter; a reader's key is kept with + # the searches' (`sources.search.`). + options["boards"] = [{"url": target, "label": sub.company or sub.label, + "params": sub.params}] + options["keys"] = dict(config.sources.get("search") or {}) plugin = ATSSource(options) plugin.state = store.state for item in plugin.fetch(): @@ -252,7 +265,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *, incomplete += 1 continue seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) - closed += store.mark_closed(source, seen_ids, sub.id) + if store.state.get(f"{source}:filtered:{sub.target}", ""): + closed += _close_unlisted(store, source, seen_ids, sub.id) + else: + closed += store.mark_closed(source, seen_ids, sub.id) if searched: closed += store.close_unseen("search", stale) if run_marker: @@ -263,6 +279,24 @@ def close_boards(config: Config, store: Store, tally: Tally, *, "incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies} +def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int: + """`Store.mark_closed` for a filtered read: close every open role of the + board it did not list, except a role someone acted on, one with a vote + (any, neutral included) or an application (which holds the notes). A + filtered read cannot tell a role gone from a role outside its filter, and + a role acted on stays until the board is read without the filter or + fails for `STALE_DAYS`.""" + with store.transaction() as conn: + return conn.execute( + """UPDATE roles SET closed_at = ? + WHERE source = ? AND subscription_id = ? AND closed_at IS NULL + AND source_id NOT IN (SELECT value FROM json_each(?)) + AND uid NOT IN (SELECT uid FROM role_votes) + AND uid NOT IN (SELECT role_uid FROM applications + WHERE role_uid IS NOT NULL)""", + (store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount + + def failing_boards(store: Store) -> dict[str, tuple[str, str]]: """`target -> (since, reason)` for every `ats` board whose last fetch failed: when the failures began, and what the latest one said.""" diff --git a/app/web/pages.py b/app/web/pages.py index 476a18a..b4f9da5 100644 --- a/app/web/pages.py +++ b/app/web/pages.py @@ -28,7 +28,7 @@ functions are the classifier's vocabulary and the hosts are what it parses, and this reaches `plugins/` the same way `app.cli` does, since a plugin is app-agnostic and never reaches back into `app`. The Jobindex areas, Adzuna countries and Platsbanken regions on the Sources page -come from `plugins.search` for the same reason. +come from the search readers in `plugins/readers/` for the same reason. """ from __future__ import annotations @@ -49,9 +49,9 @@ if str(_ROOT) not in sys.path: sys.path.insert(0, str(_ROOT)) from plugins.ats import FUNCTIONS, HOSTS # noqa: E402 -from plugins.search import ( # noqa: E402 - ADZUNA_COUNTRIES, JOBINDEX_AREAS, PLATSBANKEN_REGIONS, -) +from plugins.readers.adzuna import COUNTRIES as ADZUNA_COUNTRIES # noqa: E402 +from plugins.readers.jobindex import AREAS as JOBINDEX_AREAS # noqa: E402 +from plugins.readers.platsbanken import REGIONS as PLATSBANKEN_REGIONS # noqa: E402 TEMPLATE_DIR = Path(__file__).resolve().parent.parent / "templates" STATIC_DIR = Path(__file__).resolve().parent.parent / "static" diff --git a/plugins/ats.py b/plugins/ats.py index 7725a9c..167835d 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -2,50 +2,44 @@ `plugins/readers/`, one module per host. Every one of these hosts serves a company's own list of roles as public, -unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that -drops off the board is *closed*, which is the cheapest staleness check there -is. What a host says and how its feed reads is its module's; this file holds -what every host shares: the fetch and its allow-list, paging and -completeness, the per-role detail reads, robots.txt, the classifier, and the -plugin itself. +unauthenticated JSON, XML or RSS, and a role that drops off the board is +*closed*, which is the cheapest staleness check there is. What a host says +and how its feed reads is its module's, what every reader shares is +`plugins/readers/_runner.py`'s; this file is the `ats` Source over the runner, +the classifier, and the names the app and the tests reach for on `ats`. The allow-list and the classifier are ported from `openings.go` and `cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including -`classify`'s nine functions and their keywords. The body cap is not a -straight port — see `MAX_BYTES` below. +`classify`'s nine functions and their keywords. A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what -its slug is. Every board is fetched independently; one that fails or times -out is logged, recorded as failing under `ats:failing:` (see -`ATSSource.fetch`), and skipped, and the rest continue. +its slug is. Every board is read independently; one that fails is logged, +recorded as failing under `ats:failing:`, and skipped. A board's +subscription `params` (`{"countries": ["DK"], "remote": true}`) are its +filter, narrowed to what its reader declares (`_runner.ask_for`). -A host that pages states a total or does not. A board with a total counts as -complete only when it read as many distinct roles as the total says, one -without when it read up to an empty page (see `_paged`); a host that answers -with the whole board at once is complete whenever it parses. +The limits below are read when a board is read, so a test that replaces one +on this module reaches the runner. """ from __future__ import annotations -import ipaddress import json import logging import re import sys -import time -from collections.abc import Callable, Iterator, MutableMapping -from datetime import UTC, datetime, timedelta -from typing import Any +from collections.abc import Callable, Iterator +from datetime import UTC, datetime from pathlib import Path -from urllib.parse import unquote +from typing import Any import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, - SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, + SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, # noqa: F401 ) try: @@ -58,6 +52,7 @@ except ModuleNotFoundError: # file. So the root goes on the path then, and only then. sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from plugins import readers +from plugins.readers import _runner from plugins.readers._shared import Get, Row # The names the parsers had here before they moved to their host modules, @@ -70,156 +65,57 @@ from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401 log = logging.getLogger(__name__) -#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every -#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's -#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers — -#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and -#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised -#: to 96 MB so those boards fit; a board that still exceeds it gets one retry -#: at its host's `fallback_url` where it has one (Greenhouse, without -#: `?content=true`, see `_fetch_rows`), so at least the titles land. -MAX_BYTES = 96 << 20 +#: The runner's limits, kept here as the `ats` Source's own: see `_runner`. +MAX_BYTES = _runner.MAX_BYTES +TIMEOUTS = _runner.TIMEOUTS +DEADLINE = _runner.DEADLINE +MAX_PAGES = _runner.MAX_PAGES +PAGE_DELAY = _runner.PAGE_DELAY +DETAIL_CAP = _runner.DETAIL_CAP +DETAIL_FIELDS = _runner.DETAIL_FIELDS -#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai` -#: failed a pull of 2026-09-28 with a read timeout. Measured live on -#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse -#: boards with the most open roles (databricks, stripe, anthropic, feverup, -#: datadog), with and without `pay_transparency=true`, cached and with a -#: cache-busting parameter (84 requests; CloudFront answered every one -#: "Miss", so all reached the origin): the body arrives in under 1 s once it -#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the -#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic, -#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise. -#: The flag made no difference that three runs could tell apart from that -#: noise. So a read may wait 30 s, about twice the worst first byte seen, -#: and a whole request, redirects and body included, 60 s (see `_get`), -#: about four times the slowest seen, which still leaves a 96 MB body room -#: on a link far slower than the one measured. -TIMEOUTS = httpx.Timeout(10.0, read=30.0) -DEADLINE = 60.0 +#: The runner's errors, under the names they had here. +_Refused = _runner.Refused +_TooLarge = _runner.TooLarge +_local_host = _runner.local_host #: host -> the name the host gives itself: the hosts this plugin reads, in #: the order a select offers them. What the Sources page's host select is #: rendered from, so it cannot offer a host `is_known_target` refuses. HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()} +#: host -> parser, for the hosts whose module parses a whole body. +_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { + name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")} + + +def _limits() -> _runner.Limits: + return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP) + def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: """The hosts a request or a redirect may land on for one board: what its - host module's `allowed` says, which is one fixed API host or the - company's own site. Anything else is refused, at the first hop and at - every redirect: a feed is external content, and a board row must never - be able to point this fetch at an internal address.""" + host module's `allowed` says. Anything else is refused, at the first hop + and at every redirect.""" module = readers.HOSTS.get(host) return module.allowed(slug) if module is not None else lambda h: False -class _Refused(Exception): - """A board answered somewhere its allow-list does not cover.""" - - -class _TooLarge(ValueError): - """A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only - knows the old shape still catches it; `_fetch_rows` catches it by type to - retry at the host's `fallback_url`.""" - - -class _Disallowed(Exception): - """The site's robots.txt disallows the page a board is read from (see - `_check_robots`).""" - - -class _RobotsUnreachable(Exception): - """The site's robots.txt could not be read: a 5xx, a 429, a timeout, a - connection error or a redirect off the site. RFC 9309 says to assume - the site disallows everything then, so the board is not read this pull.""" - - -#: Host name suffixes that name a machine on a private network, never a -#: company's careers site. -_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa") - - -def _local_host(name: str) -> bool: - """Whether a host name is an address or a local name rather than a public - DNS name: an IP literal (v4 or v6), anything whose last label is a number - (`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the - resolver), `localhost`, or a private suffix. A Teamtailor slug can be any - host name, so the allow-list alone would let one point the fetch here.""" - name = name.strip("[]").rstrip(".").lower() - try: - ipaddress.ip_address(name) - return True - except ValueError: - pass - last = name.rsplit(".", 1)[-1] - return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last)) - or name == "localhost" or name.endswith(_LOCAL_SUFFIXES)) - - def _get(client: httpx.Client, url: str, host: str, slug: str, *, method: str = "GET", payload: dict[str, Any] | None = None, - headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes: - """Fetch one URL for a board: https only on the default port, never an - address or a local name (`_local_host`), the host's own allow-list, at - most three redirect hops each checked the same way, and the body cut off - at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST - sends `payload` as JSON, `{}` when there is none. `headers` are sent on - top of the User-Agent and Accept every request carries, and replace - either where they name it. With `www_twin`, a redirect may also land on - the first URL's host with `www.` added or taken off, which only the - robots.txt read asks for (see `_check_robots`). - - The body is streamed and counted as it arrives, so an oversized one is - refused at `MAX_BYTES` rather than after it has been read whole, and - the whole request, redirects included, must end within `DEADLINE`: the - per-read `TIMEOUTS` alone let a board that trickles bytes run on for - good. The deadline is checked between chunks, so it can be overrun by - at most one read timeout.""" - allow = _allowed_host(host, slug) - if www_twin: - bare = httpx.URL(url).host.removeprefix("www.") - board_allow = allow - allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731 - sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {}) - deadline = time.monotonic() + DEADLINE - current = url - for _hop in range(4): - parsed = httpx.URL(current) - if (parsed.scheme != "https" or parsed.port not in (None, 443) - or _local_host(parsed.host) or not allow(parsed.host)): - raise _Refused(f"{current} is not an allowed host for {host}") - with client.stream( - method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent, - json=(payload or {}) if method == "POST" else None, - ) as response: - if response.is_redirect: - location = response.headers.get("location") or "" - if not location: - raise ValueError("redirect with no location") - current = str(httpx.URL(current).join(location)) - continue - response.raise_for_status() - chunks: list[bytes] = [] - size = 0 - for chunk in response.iter_bytes(): - size += len(chunk) - if size > MAX_BYTES: - raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)") - if time.monotonic() > deadline: - raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s", - request=response.request) - chunks.append(chunk) - return b"".join(chunks) - raise ValueError("too many redirects") + headers: dict[str, str] | None = None) -> bytes: + """`_runner.fetch` for one board, with this module's `MAX_BYTES` and + `DEADLINE`.""" + return _runner.fetch(client, url, _allowed_host(host, slug), method=method, + payload=payload, headers=headers, max_bytes=MAX_BYTES, + deadline=DEADLINE, name=host) def _getter(client: httpx.Client, host: str, slug: str) -> Get: """The `get` a host module is handed: `_get` bound to one board. `_get` - is looked up when a request is made, not when this is built, so a test - that replaces `ats._get` reaches every host; `payload` and `headers` are - passed on only when given, which keeps a replacement that predates them - working.""" + is looked up when a request is made, so a test that replaces `ats._get` + reaches every host; `payload` and `headers` are passed on only when + given, which keeps a replacement that predates them working.""" def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> bytes: extra: dict[str, Any] = {} @@ -231,12 +127,6 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get: return get -#: host -> parser, for the hosts whose module parses a whole body. Every one -#: takes the raw response body and the slug. -_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { - name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")} - - # --------------------------------------------------------- the classifier #: The fixed set of functions, and the keyword rules that map a title @@ -381,362 +271,40 @@ def careers_url(target: str) -> str: return module.careers_url(slug) if module is not None else "" -#: Pages read from one paged board at most, and the pause between two of -#: them. Workable answers ten roles a page, SmartRecruiters a hundred and -#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause -#: keeps Workable, which answers a burst with 429, from seeing one. The same -#: pause separates two detail reads. -MAX_PAGES = 100 -PAGE_DELAY = 0.5 - - -def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: - """Every page of a board whose host pages, `PAGE_DELAY` apart, until the - host has no next page, the stated total is reached, or `MAX_PAGES`. - Returns the rows and whether the board is complete: as many distinct - results read as the host's stated total, or, for a host that states none - (Teamtailor), the read ended on an empty page (a page with nothing new in - it ends the read short of that, incomplete, so an offset the host ignores - cannot loop). Counted on the raw results, so a row the parser drops does - not make a board incomplete, and on distinct ids, so a page that shifted - under an offset and repeated a role does not make it complete. - - A 429 after the first page stops the read with what it has, incomplete; - one on the first page raises like any other failure.""" - host, _, slug = target.partition(":") - module = readers.HOSTS[host] - get = _getter(client, host, slug) - rows: dict[str, Row] = {} - seen: set[str] = set() - total: Any = None - cursor: Any = None - for page in range(MAX_PAGES): - if page: - time.sleep(PAGE_DELAY) - try: - data, results, total, cursor = module.page(get, slug, cursor) - except httpx.HTTPStatusError as exc: - if not page or exc.response.status_code != 429: - raise - log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete", - target, page + 1, len(seen), total) - return list(rows.values()), False - before = len(seen) - seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results} - for row in module.rows(data, slug): - rows.setdefault(row["source_id"], row) - if (cursor is None or (isinstance(total, int) and len(seen) >= total) - or (total is None and len(seen) == before)): - break - complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None - if not complete: - log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), - "an unstated total" if total is None else total) - return list(rows.values()), complete - - -def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: - """One board's listing, fetched and parsed: its rows and whether they are - the whole board. Raises on any failure — refused host, timeout, a feed - shape the host's parser does not understand. A host that does not page - answers with the whole board, so it is complete whenever it parses; a - paged one is complete when `_paged` says so. - - A body that exceeds `MAX_BYTES` gets one retry at the host's - `fallback_url` where it has one (Greenhouse, without `?content=true`): - the description is lost, but the titles land rather than the whole board - being skipped.""" +def _fetch_rows(client: httpx.Client, target: str, + ask: readers.Ask | None = None) -> tuple[list[Row], bool]: + """One board's listing, read and parsed, unfiltered unless `ask` says: + its rows and whether they are the whole board. Raises on any failure. + No sweep and no state: what `fetch_board` and the host tests read.""" if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") host, _, slug = target.partition(":") - module = readers.HOSTS[host] - if hasattr(module, "page"): - return _paged(client, target) - get = _getter(client, host, slug) - try: - body = get(module.feed_url(slug)) - except _TooLarge: - if not hasattr(module, "fallback_url"): - raise - log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed", - target, MAX_BYTES) - body = get(module.fallback_url(slug)) - return module.parse(body, slug), True - - -# -------------------------------------------------------------- robots.txt - -#: How long a robots.txt answer holds for one board before it is read again. -ROBOTS_TTL = timedelta(days=1) - - -#: This app's product token, the name a robots.txt group addresses it by: -#: the `desk` of `desk/ (personal aggregator)`. -ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower() - -#: One rule of a group: `(allow, path pattern)`. -_Rule = tuple[bool, str] - - -def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]: - """The groups of a robots.txt, per RFC 9309: one or more `user-agent` - lines (their tokens lowercased) and the `allow` and `disallow` rules that - follow them, up to the next `user-agent` line after a rule. Comments, - other keys and rules before any `user-agent` line are left out, and so is - an empty rule, which allows everything and so says nothing.""" - groups: list[tuple[set[str], list[_Rule]]] = [] - in_agents = False - for line in text.splitlines(): - key, sep, value = line.split("#", 1)[0].partition(":") - key, value = key.strip().lower(), value.strip() - if not sep: - continue - if key == "user-agent": - if not in_agents: - groups.append((set(), [])) - in_agents = True - groups[-1][0].add(value.lower()) - elif key in ("allow", "disallow") and groups: - in_agents = False - if value: - groups[-1][1].append((key == "allow", value)) - return groups - - -def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]: - """The rules for `token`: every group that names it (matched whole and - ignoring case, as RFC 9309 says, not as a substring), merged; else every - `*` group, merged; else none, which allows everything.""" - for wanted in (token, "*"): - rules = [rule for agents, group in groups if wanted in agents for rule in group] - if rules or any(wanted in agents for agents, _group in groups): - return rules - return [] - - -def _robots_pattern(path: str) -> re.Pattern[str]: - """A rule's path as a regex anchored at the start: `*` is any run of - characters, and a `$` at the end anchors the end.""" - anchored = path.endswith("$") - body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*")) - return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL) - - -def _robots_verdict(rules: list[_Rule], path: str) -> bool: - """Whether `path` is allowed by `rules`: the longest matching rule wins, - an allow wins a tie, and no match allows. Both sides are compared - percent-decoded, so `%7E` and `~` are one character.""" - path = unquote(path) - best: tuple[int, bool] | None = None - for allow, pattern in rules: - if _robots_pattern(unquote(pattern)).match(path): - weight = (len(pattern), allow) - best = weight if best is None or weight > best else best - return best is None or best[1] - - -def _robots_allows(body: bytes, page: str) -> bool: - """Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules - for every agent (`*`), and by those it has for this app's own token - (`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this - app by name is honoured as well. Paths are case-sensitive; a byte order - mark is dropped.""" - groups = _robots_groups(body.decode("utf-8-sig", "replace")) - url = httpx.URL(page) - path = (url.raw_path or b"/").decode("ascii", "replace") - return all(_robots_verdict(_robots_rules(groups, token), path) - for token in ("*", ROBOTS_TOKEN)) - - -def _check_robots(client: httpx.Client, target: str, - state: MutableMapping[str, str] | None = None) -> None: - """Raise `_Disallowed` when the site of a host with a `robots_page` has a - robots.txt that disallows that page, and `_RobotsUnreachable` when it - cannot be read. Read through `_get`, so from the board's own allowed - hosts only, except that a redirect between the host and its `www.` twin - is followed (at most three hops, https only); any other redirect off the - site is unreachable. With `state`, the verdict and when it was reached - are kept under `ats:robots:` and held for `ROBOTS_TTL`. - - By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey, - which allows, and is kept like any other verdict. A 5xx, a 429, a - timeout or a connection error means the file is unreachable, which the - RFC says to treat as disallowing everything: the board is not read on - this pull, fails with "robots.txt could not be read", and nothing is - kept, so the next pull asks again. A 429 counts as unreachable although - it is a 4xx, since it says "not now" rather than "there is none".""" - host, _, slug = target.partition(":") - module = readers.HOSTS.get(host) - if module is None or not hasattr(module, "robots_page") or not is_known_target(target): - return - page = module.robots_page(slug) - key = f"ats:robots:{target}" - now = datetime.now(UTC) - if state is not None: - try: - kept = json.loads(state.get(key) or "{}") - fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL - except (ValueError, KeyError, TypeError): - kept, fresh = {}, False - if fresh: - if not kept.get("allowed", True): - raise _Disallowed(f"robots.txt disallows {page}") - return - url = f"https://{httpx.URL(page).host}/robots.txt" - try: - body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True) - except httpx.HTTPStatusError as exc: - code = exc.response.status_code - if not 400 <= code < 500 or code == 429: - raise _RobotsUnreachable(f"robots.txt answered {code}") from exc - allowed = True - except Exception as exc: - raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc - else: - # A host that serves a path whatever its case is matched without - # case, so a rule written `/Jobs/` forbids a site named `jobs` too. - if getattr(module, "ROBOTS_ANY_CASE", False): - allowed = _robots_allows(body.lower(), page.lower()) - else: - allowed = _robots_allows(body, page) - if state is not None: - state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"), - "allowed": allowed}) - if not allowed: - raise _Disallowed(f"robots.txt disallows {page}") - - -# ---------------------------------------------------------------- details - -#: Roles read in detail from one board in one pull at most. The rest of a -#: large new board is read over the next pulls, 200 at a time. -DETAIL_CAP = 200 - -#: The row keys a host's `detail` may fill: those whose column survives a -#: later pull that sends the listing's empty value for them, so what a role's -#: own page said is not wiped the pull after it was read. `description`, -#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are -#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only -#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the -#: earliest date a role ever had. A filled column is still overwritten by -#: another filled value, so `_read_details` withholds the listing's place and -#: remote for a role already read. `title` and `url` are not here, since the -#: listing always states them and would overwrite them; nor are `department` -#: (no column of its own, only `raw` and the classifier's guess) and -#: `salary_text`, both rewritten from the listing on every pull. -DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min", - "salary_max", "posted") + listing = _runner.read_listing(readers.HOSTS[host], slug, _getter(client, host, slug), ask, + limits=_limits()) + return listing.rows, listing.complete def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool, detailed: set[str], failed: set[str] ) -> tuple[list[Row], set[str], set[str]]: - """The rows of a board whose host has `detail`, with the roles not yet - read in detail filled from their own pages: at most `DETAIL_CAP` of them, - `PAGE_DELAY` apart, first the roles never tried and then those whose - detail failed before (`failed`), so a few roles that always fail cannot - take the whole cap pull after pull. Returns the rows and the compound ids - read (for `ats:detailed:`) and failed (for - `ats:detailfailed:`). - - Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key - is left out with a debug line. Every row not read in detail in this call - goes out with an empty description: a role read on an earlier pull keeps - the one it has, since `upsert_role` never overwrites a filled column with - an empty value, and a teaser from the listing must not replace it. A role - read on an earlier pull goes out without the listing's place and remote - as well, for the same reason: the listing states them every pull - ("2 Locations"), and would replace what the role's own page said. - - A detail that answers 404 or 410 counts as read: the role is gone or has - no page of its own, and asking again would not change that. One that - raises otherwise is logged and the role kept in `failed`; a 429 stops the - reads for this board until the next pull, and does not mark the role as - failing. None of this touches completeness, which is the listing's alone. - The ids of roles no longer listed are dropped from both sets, but only - after a complete listing, so one cut short does not make the next pull - read its missing roles again.""" + """`_runner.read_details` for one board, with `rows` as the whole listing + when `complete`.""" host, _, slug = target.partition(":") - module = readers.HOSTS[host] - get = _getter(client, host, slug) - done, failing = set(detailed), set(failed) - set(detailed) - - def compound(row: Row) -> str: - return f"{host}:{slug}:{row['source_id']}" - - unread = [row for row in rows if compound(row) not in done] - queue = ([row for row in unread if compound(row) not in failing] - + [row for row in unread if compound(row) in failing])[:DETAIL_CAP] - filled: dict[str, Row] = {} - for row in queue: - key = compound(row) - time.sleep(PAGE_DELAY) - try: - answer = module.detail(get, slug, row) - except httpx.HTTPStatusError as exc: - code = exc.response.status_code - if code == 429: - log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull", - target) - break - if code in (404, 410): - done.add(key) - failing.discard(key) - else: - log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code) - failing.add(key) - continue - except Exception as exc: - log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc) - failing.add(key) - continue - if ignored := sorted(set(answer) - set(DETAIL_FIELDS)): - log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS", - target, row["source_id"], ", ".join(ignored)) - filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} - done.add(key) - failing.discard(key) - out = [] - for row in rows: - key = compound(row) - if key in filled: - out.append(row | {"description": ""} | filled[key]) - elif key in detailed: - out.append(row | {"description": "", "location": "", "remote": None}) - else: - out.append(row | {"description": ""}) - if complete: - listed = {compound(row) for row in rows} - done &= listed - failing &= listed - return out, done, failing + listed = {f"{host}:{slug}:{row['source_id']}" for row in rows} if complete else None + return _runner.read_details(readers.HOSTS[host], host, slug, _getter(client, host, slug), + rows, detailed, failed, listed=listed, limits=_limits()) def fetch_board(target: str) -> tuple[list[Row], bool]: - """Fetch and parse one board directly: no `Source`, no `self.state`, just - "does this board answer, what does it list, and is that all of it". For - `app companies add|check`, which ask that question about one board at a - time outside any pull. The listing only, with no detail reads, and - robots.txt read afresh. Raises the same way `_fetch_rows` does, and - `_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or - that cannot be checked.""" + """Fetch and parse one board directly: no `Source`, no state, just "does + this board answer, what does it list, and is that all of it". For `app + companies add|check`. The listing only, unfiltered, no detail reads. + Raises the same way `_fetch_rows` does.""" with httpx.Client() as client: - _check_robots(client, target) return _fetch_rows(client, target) -def failure_reason(exc: BaseException) -> str: - """One short line for why a board did not answer: the status of an HTTP - error, else the exception's class and the first line of its message.""" - if isinstance(exc, _Disallowed): - return "robots.txt disallows it" - if isinstance(exc, _RobotsUnreachable): - return "robots.txt could not be read" - if isinstance(exc, httpx.HTTPStatusError): - return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() - lines = str(exc).strip().splitlines() - text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__ - return text[:200] +failure_reason = _runner.failure_reason # -------------------------------------------------------------- the plugin @@ -745,9 +313,8 @@ class ATSSource(Source): name = "ats" description = f"Company job boards: {', '.join(HOSTS.values())}." #: The raw entries under `sources.ats.boards` are `{"url": target, "label": - #: company}` — `url` carries the `host:slug` target and `label` the - #: company name, which is how `Source.targets()`'s generic dict reading - #: hands both back without a plugin-specific override. + #: company, "params": {...}}`: `url` carries the `host:slug` target, + #: `label` the company name and `params` the board's filter. subscription_field = "boards" spec = PluginSpec( @@ -763,99 +330,74 @@ class ATSSource(Source): )}, ) - def _id_set(self, key: str) -> set[str]: - """A JSON list of ids kept in state, as a set; empty when there is - none or it does not parse.""" - try: - return {str(i) for i in json.loads(self.state.get(key) or "[]")} - except (ValueError, TypeError): - return set() - def fetch(self) -> Iterator[Item]: with httpx.Client() as client: # Handed in by the batch that runs this plugin, as the marker of - # the pull run that queued it (see `app/jobs/pull.py`), and never - # read from state, where a later run may have replaced it. A - # plugin run on its own — a test, a dry run — has no run to stamp - # itself against, so it makes one up. + # the pull run that queued it (see `app/jobs/pull.py`). A plugin + # run on its own (a test, a dry run) makes one up. run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( timespec="seconds") - for target, label in self.targets(): - host, _, slug = target.partition(":") - failing = f"ats:failing:{target}" - try: - _check_robots(client, target, self.state) - rows, complete = _fetch_rows(client, target) - except Exception as exc: - log.warning("ats: %s failed (%s)", target, exc) - # The first failure in a row sets `since`, later ones only - # the reason; `app.jobs.pull` closes the board's roles - # once `since` is `STALE_DAYS` old. - try: - since = json.loads(self.state.get(failing, "") or "{}").get("since") - except ValueError: - since = None - self.state[failing] = json.dumps({ - "since": since or datetime.now(UTC).isoformat(timespec="seconds"), - "reason": failure_reason(exc)}) - continue - self.state.pop(failing, None) + yield from self._board(client, target, label, run_marker) - detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}") - detail_sets: tuple[set[str], set[str]] | None = None - if hasattr(readers.HOSTS.get(host), "detail"): - detailed, failed = (self._id_set(key) for key in detail_keys) - rows, detailed, failed = _read_details(client, target, rows, complete, - detailed, failed) - detail_sets = (detailed, failed) + def _board(self, client: httpx.Client, target: str, label: str, + run_marker: str) -> Iterator[Item]: + host, _, slug = target.partition(":") + failing = f"ats:failing:{target}" + try: + if not is_known_target(target): + raise ValueError(f"{target!r} is not host:slug for a known ATS host") + module = readers.HOSTS[host] + get = _getter(client, host, slug) + ask = _runner.ask_for(module, self.params_for(target), + keys=self.options.get("keys")) + listing = _runner.read(self.state, "ats", target, module, slug, get, ask, + limits=_limits()) + except Exception as exc: + log.warning("ats: %s failed (%s)", target, exc) + _runner.record_failure(self.state, failing, exc) + return + self.state.pop(failing, None) - seen_ids = [] - for row in rows: - # The compound id, matching `roles.source_id` — what - # `Store.mark_closed` compares against, not the board's own - # bare job id. - compound_id = f"{host}:{slug}:{row['source_id']}" - seen_ids.append(compound_id) - yield Item( - source_id=compound_id, - location=row["url"], - title=row["title"], - excerpt=(row["description"] or "")[:300], - body=row["description"], - published_at=row["posted"], - subscription=label, - kind="role", - raw={ - "company": label, - "place": row["location"], - "department": row["department"], - "function": classify(row["department"], row["title"]), - "remote": row["remote"], - # Every parser sets salary_text, empty or not (see - # "salary_text" in `readers/_shared.py`); only - # the hosts that state structured pay also set the - # normalised figures, so the rest leave them at - # row.get's default of no figure. - "salary_text": row.get("salary_text", ""), - "salary_min": row.get("salary_min"), - "salary_max": row.get("salary_max"), - "currency": row.get("currency", ""), - "board": host, - "slug": slug, - }, - ) - # Stamped only once the board has parsed without raising, so a - # board that blew up half way through is neither marked ok nor - # has a partial id list treated as the whole of it. `complete` - # says whether the ids are the whole board, which the pull - # needs before it closes what they leave out. The ids read in - # detail are kept here too, after every row went out, so a - # pull stopped half way reads the rest again rather than - # losing their descriptions. - self.state[f"ats:ok:{target}"] = run_marker - self.state[f"ats:complete:{target}"] = "1" if complete else "" - self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) - if detail_sets is not None: - for key, ids in zip(detail_keys, detail_sets, strict=True): - self.state[key] = json.dumps(sorted(ids)) + def compound(source_id: str) -> str: + # Matches `roles.source_id`, which `Store.mark_closed` compares. + return f"{host}:{slug}:{source_id}" + + rows = listing.rows + detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}") + detail_sets: tuple[set[str], set[str]] | None = None + if hasattr(module, "detail"): + detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys) + listed = {compound(i) for i in listing.ids} if listing.complete else None + rows, detailed, failed = _runner.read_details( + module, host, slug, get, rows, detailed, failed, listed=listed, + limits=_limits()) + detail_sets = (detailed, failed) + + for row in rows: + description = row.get("description") or "" + yield Item( + source_id=compound(row["source_id"]), + location=row["url"], + title=row["title"], + excerpt=description[:300], + body=description, + published_at=row.get("posted"), + subscription=label, + kind="role", + raw=_runner.raw_of(row) | { + "company": label, + "function": classify(row.get("department") or "", row["title"]), + "board": host, + "slug": slug, + }, + ) + # Stamped only once every row went out, so a board that blew up half + # way through is neither marked ok nor has a partial id list treated + # as the whole of it, and a pull stopped half way reads the rest of + # its details again. + _runner.stamp(self.state, "ats", target, run_marker, listing, + [compound(i) for i in listing.ids]) + if detail_sets is not None: + for key, ids in zip(detail_keys, detail_sets, strict=True): + self.state[key] = json.dumps(sorted(ids)) diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py index 29021ff..9a4a2e2 100644 --- a/plugins/readers/__init__.py +++ b/plugins/readers/__init__.py @@ -129,7 +129,8 @@ exists. **Fetching.** `allowed(slug)` returns a predicate over a lowercased host name; `_shared.only(host)` is the usual one. Every request goes through -`get`, which is `ats._get` bound to the board: +`get`, which is `_runner.fetch` bound to the target (`_runner.getter`; the +`ats` Source binds it through `ats._get`): get(url, *, method="GET", payload=None, headers=None) -> bytes @@ -138,7 +139,7 @@ default User-Agent and `Accept: application/json, text/xml`, and replace either where they name it. `get` is https only on port 443, never to an IP address or a local name (`localhost`, `.local`, `.internal`, ...), follows at most three redirects each checked against `allowed(slug)`, cuts the body at -`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns +`_runner.MAX_BYTES` and ends the whole request within `_runner.DEADLINE`. It returns the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read `exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection failure, and another exception (`ValueError` among them) for a refused host, @@ -151,8 +152,9 @@ id, to decide completeness), the total the host states or None, and the cursor of the next page or None after the last. The first call gets cursor None. A board with a total is complete when as many distinct ids were read; one without, when the read ended on an empty page. A 429 after the first page -ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY` -apart. A paged module may also have `parse(body, slug)` for one page's body. +ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most, +`_runner.PAGE_DELAY` apart, and a board cut there is swept across pulls +(`_runner.read_listing`). A paged module may also have `parse(body, slug)` for one page's body. **Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's own id for the role, unique on the board), `title`, `url` (http or https, @@ -183,11 +185,11 @@ keeps it. **Detail.** `detail(get, slug, row)` reads one role's own page and returns a dict of the fields it found. The shared code calls it for roles not yet read -(`ats:detailed:`), at most `ats.DETAIL_CAP` (200) a board a pull, -`ats.PAGE_DELAY` apart, the roles never tried before those that failed -(`ats:detailfailed:`). Only `ats.DETAIL_FIELDS` are taken: -`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` -and `posted`, the keys whose columns survive a later pull that sends the +(`ats:detailed:`), at most `_runner.DETAIL_CAP` (200) a board a +pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that +failed (`ats:detailfailed:`). Only `_runner.DETAIL_FIELDS` are taken: +`description`, `location`, `places`, `teaser`, `remote`, `country`, +`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the listing's empty value; any other key is left out with a debug log line. `title`, `url`, `department` and `salary_text` cannot be filled from `detail`: the listing rewrites them every pull. A role not read in detail @@ -203,11 +205,9 @@ Whoever builds a reader reads the host's robots.txt for the paths the reader requests and reports what it disallows, and the owner decides whether the host is read anyway. `robots` records that decision with its date, e.g. `"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is -disallowed: owner, 2026-09-30"`. The runtime check in `ats.py`, which -`robots_page` and `ROBOTS_ANY_CASE` feed, predates this rule and goes with -the runner; a new reader defines neither. Until then it reads -`https:///robots.txt` at most once a day per board and fails the board -when the page is disallowed for `*` or the app's token. +disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time; +`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are +read by nothing and a new reader defines neither. **Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when the slug is a board; `probe_ok(body)` narrows that when a host answers an diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py new file mode 100644 index 0000000..73bf67e --- /dev/null +++ b/plugins/readers/_runner.py @@ -0,0 +1,712 @@ +"""The runner: what the app does for every reader, board or search, from what +the reader declares in `READS`. + +A reader says how its host answers; this module reads it: the guarded fetch +(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves +to (`ask_for`) and its series (`series`), the listing whole or paged, one +series per `one`-filter value unioned by id (`read_listing`), completeness, +the sweep of an oversized paged board across pulls, the 10% guard on a +filtered or `empty="suspect"` read (`read`), the per-role detail reads +(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure +bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and +`plugins/search.py` are thin `Source` classes over it: they pass their own +limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay +as they were. + +A module without `READS` is read with the older signatures, `feed_url(slug)` +and `page(get, slug, cursor)`, and never gets an `Ask`. + +State keys, under the Source's prefix and the target (`ats:sweep:`): + + sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor", + "seen", "rows", "whole"}`, continued by the next pull + count `{"asks", "count"}`, the roles the last whole read listed, for + the 10% guard + facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL` +""" + +from __future__ import annotations + +import ipaddress +import itertools +import json +import logging +import os +import re +import time +from collections.abc import Callable, Iterable, Mapping, MutableMapping +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from types import ModuleType +from typing import Any + +import httpx + +from desk.sources import USER_AGENT + +from ._contract import Ask, Key, Place, Reads +from ._shared import Get, Row + +log = logging.getLogger(__name__) + +#: openings.go's 8 MB body cap is not kept: every URL fetched here is a feed +#: on a fixed allow-list, and the real run of 2026-09-23 hit 8 MB on the +#: biggest employers (Anduril at 41 MB, Anthropic, OpenAI, Databricks between +#: 9 and 19 MB). 96 MB fits them; a board still over it gets one retry at its +#: reader's `fallback_url` where it has one. +MAX_BYTES = 96 << 20 + +#: The first byte of a Greenhouse board waited up to 16.6 s in 84 requests +#: measured on 2026-09-29, and the body then arrived in under 1 s. So a read +#: may wait 30 s and a whole request, redirects and body included, 60 s. +TIMEOUTS = httpx.Timeout(10.0, read=30.0) +DEADLINE = 60.0 + +#: Pages read from one target in one pull at most, and the pause between two +#: requests (pages and detail reads), which keeps Workable from answering 429. +MAX_PAGES = 100 +PAGE_DELAY = 0.5 + +#: Roles read in detail from one board in one pull at most. +DETAIL_CAP = 200 + +#: The row keys a reader's `detail` may fill: those whose column survives a +#: later pull that sends the listing's empty value, so what a role's own page +#: said is not wiped the pull after it was read. `title`, `url`, `department` +#: and `salary_text` are left out, since the listing rewrites them every pull. +#: Workday, SuccessFactors and Rippling state `country` and `employment` on a +#: role's own page only. +DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "country", + "employment", "currency", "salary_min", "salary_max", "posted") + +#: How long a board's `facets` are kept before they are asked again. +FACETS_TTL = timedelta(days=1) + +#: A filtered or `empty="suspect"` read listing under this share of what the +#: same read listed last time counts as a failure. +SUSPECT_SHARE = 0.1 + +#: The keys of a row that an `Item` carries in its own fields, not in `raw`. +ITEM_KEYS = ("source_id", "title", "url", "description", "posted") + + +class Refused(Exception): + """A request or a redirect aimed at a host its allow-list does not cover.""" + + +class TooLarge(ValueError): + """A response over the byte cap. A `ValueError`, as it was before it had a + type of its own.""" + + +class Suspect(Exception): + """A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE` + of what it listed last time.""" + + +class MissingKey(ValueError): + """A reader's `READS.key` has no value in the config or the environment.""" + + +@dataclass(frozen=True, slots=True) +class Limits: + """What one Source allows a read: pages, the pause between requests, and + the detail reads a pull.""" + + max_pages: int = MAX_PAGES + page_delay: float = PAGE_DELAY + detail_cap: int = DETAIL_CAP + + +@dataclass(slots=True) +class Listing: + """One read of a target's listing. `rows` are this pull's rows, `ids` the + `source_id`s the read listed: this pull's, or the whole sweep's once it + reached the end. `complete` says `ids` are the whole list. `sweep` is the + state to keep for the next pull, None when no sweep is under way. + `filtered` says the read asked the host for a part of the list.""" + + rows: list[Row] + complete: bool + ids: list[str] = field(default_factory=list) + filtered: bool = False + sweep: dict[str, Any] | None = None + asks: str = "" + + +# --------------------------------------------------------------- modules + +def declared(module: ModuleType) -> bool: + """Whether `module` declares `READS`, and so takes `ask`.""" + return isinstance(getattr(module, "READS", None), Reads) + + +def module_of(name: str) -> ModuleType | None: + """The reader named `name`, a board host first, else a search provider.""" + from plugins import readers + + return readers.HOSTS.get(name) or readers.PROVIDERS.get(name) + + +# ----------------------------------------------------------------- fetch + +#: Host name suffixes that name a machine on a private network. +_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa") + + +def local_host(name: str) -> bool: + """Whether a host name is an address or a local name rather than a public + DNS name: an IP literal, anything whose last label is a number + (`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1), `localhost` + or a private suffix.""" + name = name.strip("[]").rstrip(".").lower() + try: + ipaddress.ip_address(name) + return True + except ValueError: + pass + last = name.rsplit(".", 1)[-1] + return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last)) + or name == "localhost" or name.endswith(_LOCAL_SUFFIXES)) + + +def fetch(client: httpx.Client, url: str, allow: Callable[[str], bool], *, + method: str = "GET", payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, max_bytes: int | None = None, + deadline: float | None = None, name: str = "") -> bytes: + """Fetch one URL for one target: https on port 443 only, never an address + or a local name, only hosts `allow` accepts, at most three redirects each + checked the same way, the body cut at `max_bytes` (`MAX_BYTES`) and the + whole request ended within `deadline` (`DEADLINE`) seconds. A POST sends + `payload` as JSON, `{}` when None; `headers` are added to the + User-Agent and Accept and replace either where they name it. + + The body is streamed and counted as it arrives, and the deadline checked + between chunks, so a trickling answer cannot run on. Raises `Refused`, + `TooLarge`, `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError` + on a timeout or a connection failure, and ValueError on a broken + redirect.""" + cap = MAX_BYTES if max_bytes is None else max_bytes + limit = DEADLINE if deadline is None else deadline + sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {}) + ends = time.monotonic() + limit + current = url + for _hop in range(4): + parsed = httpx.URL(current) + if (parsed.scheme != "https" or parsed.port not in (None, 443) + or local_host(parsed.host) or not allow(parsed.host)): + raise Refused(f"{current} is not an allowed host for {name or 'this reader'}") + with client.stream( + method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent, + json=(payload or {}) if method == "POST" else None, + ) as response: + if response.is_redirect: + location = response.headers.get("location") or "" + if not location: + raise ValueError("redirect with no location") + current = str(httpx.URL(current).join(location)) + continue + response.raise_for_status() + chunks: list[bytes] = [] + size = 0 + for chunk in response.iter_bytes(): + size += len(chunk) + if size > cap: + raise TooLarge(f"response too large (over {cap} bytes)") + if time.monotonic() > ends: + raise httpx.ReadTimeout(f"no whole answer within {limit:.0f} s", + request=response.request) + chunks.append(chunk) + return b"".join(chunks) + raise ValueError("too many redirects") + + +def getter(client: httpx.Client, allow: Callable[[str], bool], name: str = "") -> Get: + """The `get` a reader is handed: `fetch` bound to one target's allow-list.""" + def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None) -> bytes: + return fetch(client, url, allow, method=method, payload=payload, headers=headers, + name=name) + return get + + +# ------------------------------------------------------------------- ask + +#: A filter dimension of `Filters` -> the field of `Ask` it fills. +_ASK_FIELD = {"country": "countries", "place": "places", "remote": "remote", "query": "query"} + +#: Keys a subscription's params or a reach may use, -> the dimension. +_PARAM_DIMENSION = {"countries": "country", "country": "country", "places": "place", + "place": "place", "remote": "remote", "query": "query", "q": "query"} + + +def _values(value: Any) -> tuple[str, ...]: + """A param's value as a tuple of non-empty strings.""" + items = value if isinstance(value, list | tuple | set) else [value] + return tuple(str(v).strip() for v in items if v not in (None, "", 0) and str(v).strip()) + + +def _dimensions(module: ModuleType, given: Mapping[str, Any]) -> dict[str, Any]: + """`dimension -> value` from a params or reach mapping: the generic keys + (`countries`, `places`, `remote`, `q`) and the reader's own `PARAMS` + (saved param -> dimension, e.g. Jobindex's `area` -> `place`).""" + names = _PARAM_DIMENSION | dict(getattr(module, "PARAMS", {}) or {}) + out: dict[str, Any] = {} + for key, value in given.items(): + dimension = names.get(key) + if dimension == "remote": + if isinstance(value, bool): + out["remote"] = value + elif dimension == "query": + if str(value or "").strip(): + out["query"] = str(value).strip() + elif dimension in ("country", "place"): + values = _values(value) + if dimension == "country": + values = tuple(v.upper() for v in values) + if values: + out[dimension] = values + return out + + +def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]: + """A reader's key: each of `key.fields` from the config entry + `sources.search.` (`keys` is `sources.search`), else from the + environment variable at the same position in `key.env`. Raises + `MissingKey` naming both places when a field has neither.""" + entry = (keys or {}).get(key.name) or {} + out: dict[str, str] = {} + for index, name in enumerate(key.fields): + env = key.env[index] if index < len(key.env) else "" + value = str((entry.get(name) if isinstance(entry, Mapping) else "") or "") \ + or (os.environ.get(env, "") if env else "") + if not value: + where = f"sources.search.{key.name}: {', '.join(key.fields)}" + if key.env: + where += f", or {' and '.join(key.env)}" + raise MissingKey(f"{key.name} needs a key: {where}") + out[name] = value + return out + + +def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None, + reach: Mapping[str, Any] | None = None, *, + keys: Mapping[str, Any] | None = None) -> Ask: + """The `Ask` one target resolves to. `reach` (from the agents, package F) + and then the subscription's own `params` (a board's override, e.g. + `{"countries": ["DK"], "remote": true}`, or a search's saved params) set + the dimensions, params winning per dimension, narrowed to the ones the + reader declares in `READS.filters`. A search's params go whole into + `Ask.params`; a reader with `READS.key` gets the resolved key under + `Ask.params["key"]` (see `resolve_key`, which raises when it is missing). + A module without `READS` gets `Ask()`.""" + if not declared(module): + return Ask() + reads: Reads = module.READS + params = dict(subscription_params or {}) + wanted = _dimensions(module, reach or {}) | _dimensions(module, params) + fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items() + if name in reads.filters.declared()} + extra: dict[str, Any] = dict(params) if reads.scope == "search" else {} + if reads.key is not None: + extra["key"] = resolve_key(reads.key, keys) + return Ask(**fields, params=extra) + + +def filtered(ask: Ask) -> bool: + """Whether `ask` reads a part of a board: a country, a place or remote.""" + return bool(ask.countries or ask.places or ask.remote is not None) + + +def series(module: ModuleType, ask: Ask, + facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]: + """The series of requests `ask` takes on this reader: one per value of a + dimension declared `one` (the product when two are), one with every + value for `many`, and host ids from `facets` for `ids` (values `facets` + does not know are dropped; an empty list when none is left, since the + host then has nothing in reach). `[ask]` for a module without `READS`.""" + if not declared(module): + return [ask] + reach = module.READS.filters + countries, places = ask.countries, ask.places + for dimension in ("country", "place"): + values = countries if dimension == "country" else places + if getattr(reach, dimension) != "ids" or not values: + continue + known = (facets or {}).get(dimension) or {} + mapped = tuple(known[v] for v in values if v in known) + if not mapped: + log.info("readers: %s knows none of %s %s", module.NAME, dimension, + ", ".join(values)) + return [] + if dimension == "country": + countries = mapped + else: + places = mapped + by_country = [(c,) for c in countries] if reach.country == "one" and countries \ + else [countries] + by_place = [(p,) for p in places] if reach.place == "one" and places else [places] + return [ask.model_copy(update={"countries": c, "places": p}) + for c, p in itertools.product(by_country, by_place)] + + +def fingerprint(asks: Iterable[Ask]) -> str: + """What a read asked, without its params (which may hold a key): the same + string for the same series, so a sweep or a count is only compared with + a read that asked the same.""" + return json.dumps([ask.model_dump(exclude={"params"}, mode="json") for ask in asks], + sort_keys=True) + + +# --------------------------------------------------------------- listing + +def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]: + return {str(r.get("id") or r.get("shortcode") or "") for r in results} + + +def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask], + limits: Limits) -> dict[str, Row]: + """A whole listing, one request a series, unioned by `source_id`. A body + over the cap gets one retry at the reader's `fallback_url`.""" + rows: dict[str, Row] = {} + for index, ask in enumerate(asks): + if index: + time.sleep(limits.page_delay) + url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug) + try: + body = get(url) + except TooLarge: + if not hasattr(module, "fallback_url"): + raise + log.warning("readers: %s:%s: body over the cap; reading the smaller feed", + module.NAME, slug) + body = get(module.fallback_url(slug)) + for row in module.parse(body, slug): + rows.setdefault(row["source_id"], row) + return rows + + +def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *, + limits: Limits | None = None, sweep: Mapping[str, Any] | None = None, + can_sweep: bool = False, + facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing: + """One target's listing, read and parsed. Raises on any failure of the + first request, and on a reader that does not understand an answer. + + A whole listing is complete whenever it parses. A paged one reads each + series' pages, `limits.page_delay` apart, until the host has no next + page, the stated total is reached, a page brings nothing new (a host + that ignores the cursor, incomplete) or `limits.max_pages` pages in all. + A series is complete when it read as many distinct result ids as its + stated total, or, without a total, when it ended on the host's last + page; never for `complete="never"`. The read is complete when every + series is. A 429 after the first page ends the read, incomplete. + + With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns + the `sweep` to keep, and the next pull, handed it back, continues from + its series and cursor; the sweep is complete when it reaches the end, + and `ids` then hold every id it listed. A read at the reader's `ceiling` + is not swept: the host answers nothing past it.""" + limits = limits or Limits() + ask = ask or Ask() + asks = series(module, ask, facets) + is_filtered = filtered(ask) + key = fingerprint(asks) + if not asks: + return Listing([], True, [], is_filtered, None, key) + if not hasattr(module, "page"): + rows = _whole(module, slug, get, asks, limits) + return Listing(list(rows.values()), True, list(rows), is_filtered, None, key) + + reads: Reads | None = module.READS if declared(module) else None + never = reads is not None and reads.complete == "never" + ceiling = reads.ceiling if reads is not None else None + start, cursor, seen, earlier, whole = 0, None, set(), [], True + if sweep and sweep.get("asks") == key: + start, cursor = int(sweep.get("series") or 0), sweep.get("cursor") + seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or []) + whole = bool(sweep.get("whole", True)) + rows: dict[str, Row] = {} + pages = 0 + stop: str = "" + index = start + for index in range(start, len(asks)): + total: Any = None + ended = stalled = False + while True: + if pages >= limits.max_pages: + stop = "cap" + break + if pages: + time.sleep(limits.page_delay) + try: + if reads is not None: + data, results, total, cursor = module.page(get, slug, cursor, asks[index]) + else: + data, results, total, cursor = module.page(get, slug, cursor) + except httpx.HTTPStatusError as exc: + if not pages or exc.response.status_code != 429: + raise + log.warning("readers: %s:%s: 429 on page %d; kept %d, not complete", + module.NAME, slug, pages + 1, len(rows)) + stop = "429" + break + pages += 1 + before = len(seen) + seen |= _result_ids(results) + for row in module.rows(data, slug): + rows.setdefault(row["source_id"], row) + if cursor is None or (isinstance(total, int) and len(seen) >= total): + ended = True + break + if total is None and len(seen) == before: + stalled = True + break + if stop: + break + done = (not never and not stalled + and (len(seen) >= total if isinstance(total, int) else ended)) + if not done: + log.warning("readers: %s:%s: read %d of %s; not complete", module.NAME, slug, + len(seen), "an unstated total" if total is None else total) + whole = whole and done + cursor, seen = None, set() + + listed = list(dict.fromkeys(earlier + list(rows))) + if not stop: + return Listing(list(rows.values()), whole, listed, is_filtered, None, key) + log.warning("readers: %s:%s: stopped after %d pages (%s); not complete", + module.NAME, slug, pages, stop) + at_ceiling = ceiling is not None and len(seen) >= ceiling + if not can_sweep or never or at_ceiling or not _storable(cursor): + return Listing(list(rows.values()), False, list(rows), is_filtered, None, key) + keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen), + "rows": listed, "whole": whole} + return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key) + + +def _storable(cursor: Any) -> bool: + """Whether a cursor survives a trip through JSON in state.""" + try: + return json.loads(json.dumps(cursor)) == cursor + except (TypeError, ValueError): + return False + + +# ------------------------------------------------------------ with state + +def _load(state: Mapping[str, str], key: str) -> dict[str, Any]: + try: + value = json.loads(state.get(key) or "{}") + except (TypeError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + +def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask, + state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]: + """The reader's `facets` when `ask` sets a dimension it declares `ids`, + kept in `state[key]` for `FACETS_TTL`; `{}` otherwise.""" + if not declared(module) or not hasattr(module, "facets"): + return {} + ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"] + if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name in ("country", "place")): + return {} + now = datetime.now(UTC) + if state is not None and key: + kept = _load(state, key) + try: + if now - datetime.fromisoformat(kept["checked"]) < FACETS_TTL: + return dict(kept.get("facets") or {}) + except (KeyError, TypeError, ValueError): + pass + found = dict(module.facets(get, slug) or {}) + if state is not None and key: + state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"), + "facets": found}) + return found + + +def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType, + slug: str, get: Get, ask: Ask | None = None, *, + limits: Limits | None = None, can_sweep: bool = True) -> Listing: + """`read_listing` for one target with its state under `:...:`: + the facets it needs, the sweep it continues, and the 10% guard. A read + that is filtered or whose reader declares `empty="suspect"`, and that + listed under `SUSPECT_SHARE` of what the same read listed last time, + raises `Suspect`. Nothing is written here but the facets; `stamp` keeps + the rest once the rows are stored.""" + ask = ask or Ask() + facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}") + listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep, + sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets) + suspect = declared(module) and module.READS.empty == "suspect" + last = _load(state, f"{prefix}:count:{target}") + count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0 + if ((listing.filtered or suspect) and listing.sweep is None and count + and len(listing.ids) < SUSPECT_SHARE * count): + raise Suspect(f"listed {len(listing.ids)} roles, under " + f"{SUSPECT_SHARE:.0%} of the {count} it listed last time") + return listing + + +def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str, + listing: Listing, ids: list[str]) -> None: + """Record a read whose rows are stored: `ok` (the run's marker), + `complete`, `ids` (the compound ids, what the pull closes against), + `filtered`, the sweep to continue or none, and the count for the 10% + guard once a read is whole.""" + state[f"{prefix}:ok:{target}"] = run + state[f"{prefix}:complete:{target}"] = "1" if listing.complete else "" + state[f"{prefix}:ids:{target}"] = json.dumps(ids) + state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else "" + if listing.sweep is not None: + state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep) + else: + state.pop(f"{prefix}:sweep:{target}", None) + state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks, + "count": len(listing.ids)}) + + +# ---------------------------------------------------------------- details + +def id_set(state: Mapping[str, str], key: str) -> set[str]: + """A JSON list of ids kept in state, as a set; empty when there is none + or it does not parse.""" + try: + return {str(i) for i in json.loads(state.get(key) or "[]")} + except (TypeError, ValueError): + return set() + + +def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[Row], + detailed: set[str], failed: set[str], *, listed: set[str] | None = None, + limits: Limits | None = None) -> tuple[list[Row], set[str], set[str]]: + """The rows with the roles not yet read in detail filled from their own + pages, at most `limits.detail_cap`, `limits.page_delay` apart: first the + roles never tried, then those in `failed`. Ids are compound, + `::`. Returns the rows and the ids read and + failed, to keep as the caller's `detailed` and `failed` sets. + + Only `DETAIL_FIELDS` are taken from what `detail` returns. A `location` + without `places` becomes the one place. Every row not read now goes out + with an empty description, so one read earlier stays, and a row read on + an earlier pull without the listing's place and remote, so what its page + said stays. A 404 or 410 counts as read, a 429 stops the reads for this + pull, anything else keeps the role in `failed`. `listed` is every id of a + complete listing: ids outside it leave both sets.""" + limits = limits or Limits() + done, failing = set(detailed), set(failed) - set(detailed) + + def compound(row: Row) -> str: + return f"{name}:{slug}:{row['source_id']}" + + unread = [row for row in rows if compound(row) not in done] + queue = ([row for row in unread if compound(row) not in failing] + + [row for row in unread if compound(row) in failing])[:limits.detail_cap] + filled: dict[str, Row] = {} + for row in queue: + key = compound(row) + time.sleep(limits.page_delay) + try: + answer = module.detail(get, slug, row) + except httpx.HTTPStatusError as exc: + code = exc.response.status_code + if code == 429: + log.warning("readers: %s:%s: 429 on a detail read; the rest wait", name, slug) + break + if code in (404, 410): + done.add(key) + failing.discard(key) + else: + log.warning("readers: %s:%s: detail of %s answered %s", name, slug, + row["source_id"], code) + failing.add(key) + continue + except Exception as exc: + log.warning("readers: %s:%s: detail of %s failed (%s)", name, slug, + row["source_id"], exc) + failing.add(key) + continue + if ignored := sorted(set(answer) - set(DETAIL_FIELDS)): + log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS", + name, slug, row["source_id"], ", ".join(ignored)) + taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment] + if taken.get("location") and "places" not in taken: + taken["places"] = [{"text": taken["location"], "remote": None}] + filled[key] = taken + done.add(key) + failing.discard(key) + out: list[Row] = [] + for row in rows: + key = compound(row) + if key in filled: + out.append(row | {"description": ""} | filled[key]) # type: ignore[operator] + elif key in detailed: + out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator] + "remote": None}) + else: + out.append(row | {"description": ""}) # type: ignore[operator] + if listed is not None: + done &= listed + failing &= listed + return out, done, failing + + +# -------------------------------------------------------------------- raw + +def places_of(row: Mapping[str, Any]) -> list[Place]: + """A row's places: its `places`, or, for a row without (an undeclared + reader's), its `location` as the one place, split nothing.""" + places = row.get("places") + if not isinstance(places, list): + location = str(row.get("location") or "").strip() + return [{"text": location, "remote": None}] if location else [] + out: list[Place] = [] + for place in places: + if isinstance(place, Mapping) and str(place.get("text") or "").strip(): + out.append({"text": str(place["text"]).strip(), "remote": place.get("remote")}) + return out + + +def raw_of(row: Mapping[str, Any]) -> dict[str, Any]: + """The `raw` of an `Item` for one row: every key the row states except + those the `Item` carries itself (`ITEM_KEYS`), with `places` always a + list, `place` the display line of the places joined with "; ", and the + keys storage reads always present (`department`, `remote`, + `salary_text`, `salary_min`, `salary_max`, `currency`, `teaser`).""" + places = places_of(row) + raw = {k: v for k, v in row.items() if k not in ITEM_KEYS} + return raw | { + "places": places, + "place": "; ".join(place["text"] for place in places), + "department": row.get("department") or "", + "remote": row.get("remote"), + "salary_text": row.get("salary_text") or "", + "salary_min": row.get("salary_min"), + "salary_max": row.get("salary_max"), + "currency": row.get("currency") or "", + "teaser": row.get("teaser") or "", + } + + +# --------------------------------------------------------------- failures + +def failure_reason(exc: BaseException) -> str: + """One short line for why a target did not answer: the status of an HTTP + error, else the exception's class and the first line of its message.""" + if isinstance(exc, httpx.HTTPStatusError): + return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() + lines = str(exc).strip().splitlines() + text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__ + return text[:200] + + +def record_failure(state: MutableMapping[str, str], key: str, exc: BaseException) -> None: + """Keep a failure under `key` as `{"since", "reason"}`: the first failure + in a row sets `since`, later ones only the reason. The pull closes a + target's roles once `since` is old enough; its next answer removes the + key.""" + since = _load(state, key).get("since") + state[key] = json.dumps({"since": since or datetime.now(UTC).isoformat(timespec="seconds"), + "reason": failure_reason(exc)}) diff --git a/plugins/search.py b/plugins/search.py index 55de4a4..01a163f 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -5,421 +5,109 @@ roles at companies nobody has listed yet, by asking a cross-company board a saved question: "backend developer" in Region Hovedstaden, "python" in Berlin. A subscription is one saved search; its `params` are the question. -Three boards, checked live on 2026-09-23 and, for Platsbanken, 2026-09-29: - - jobindex Denmark's largest job board. Every search result page has an RSS - twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's - own syntax: words are OR, `"a b"` a phrase, `+word` required. - `area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region - Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. - 20 results a page. - adzuna An aggregator with a documented API for most of Europe, the UK - and the US, but not Denmark (adzuna.dk does not exist). Needs a - free key from developer.adzuna.com, as `sources.search.adzuna: - {app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built - against Adzuna's documented response and not yet run live. - platsbanken Sweden's public employment service, through the JobTech search - API at jobsearch.api.jobtechdev.se, no key needed. `region` is - the id of a län in the JobTech taxonomy (Skåne län is - `CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page, - and an `offset` above 2000 is refused. It states `total`. It has - no remote flag worth reading (`workplace_model` is mostly - empty), so `remote` comes from the headline. - -Considered and left out: Jobnet (its search endpoint answers 401 outside the -browser session of its own site) and Jooble (a key needs a manual -application, and its API host answered a Cloudflare challenge). +The providers are the readers of `scope="search"` in `plugins/readers/` +(`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other +reader: the search's params go in `Ask.params`, a declared key is resolved +from `sources.search.` or the environment and refused when missing, +and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most +`MAX_PAGES`). A search result list is never a board's complete list, so a role missing from one run is not closed here. A search that read everything that matches was -complete, and the pull closes what it stopped listing; the rest close once no -search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna and -Platsbanken state how many results match, so their searches are complete when -they read that many; Jobindex states no count, so a Jobindex search is -complete when a page within the page limit came back short. +complete (`complete="stated"`, as many distinct results as the stated total), +and the pull closes what it stopped listing; the rest close once no search +has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). """ from __future__ import annotations -import html import json import logging -import os -import re -import xml.etree.ElementTree as ET -from collections.abc import Callable, Iterator +import sys +from collections.abc import Iterator, Mapping from datetime import UTC, datetime -from email.utils import parsedate_to_datetime +from pathlib import Path +from types import ModuleType from typing import Any import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, - SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html, + SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, ) +try: + from plugins import readers +except ModuleNotFoundError: + # Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`. + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from plugins import readers +from plugins.readers import _runner +from plugins.readers._shared import Row + log = logging.getLogger(__name__) -TIMEOUT = 15.0 - -#: Pages fetched per search unless its params say otherwise, and the most a +#: Pages read per search unless its params say otherwise, and the most a #: search may ask for. A search that needs more than ten pages is too broad to #: read anyway. DEFAULT_PAGES = 5 MAX_PAGES = 10 -PROVIDERS = ("jobindex", "adzuna", "platsbanken") - -#: What every provider turns a result into. -Row = dict[str, Any] +#: The pause between two pages of one search. +PAGE_DELAY = _runner.PAGE_DELAY -def _web_url(url: str) -> bool: - parsed = httpx.URL((url or "").strip()) - return parsed.scheme in ("http", "https") and bool(parsed.host) - - -def _pages(params: dict[str, Any]) -> int: +def _pages(params: Mapping[str, Any]) -> int: try: return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES)) except (TypeError, ValueError): return DEFAULT_PAGES -def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes: - response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True, - headers={"User-Agent": USER_AGENT}) - response.raise_for_status() - return response.content - - -# ---------------------------------------------------------------- jobindex - -JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss" -JOBINDEX_PAGE = 20 - -#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked -#: against the titles its feeds answer with. 0 is all of Denmark. -JOBINDEX_AREAS: dict[int, str] = { - 0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland", - 2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland", - 8: "Aarhus", 7: "Aalborg", -} - -_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE) -_H4 = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) -_AREA = re.compile(r'class="jix_robotjob--area">(.*?)', re.IGNORECASE | re.DOTALL) -_PARAGRAPHS = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) -#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden". -_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende") - - -#: What a Danish ad's town line says about where the work is done: -#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". -_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) -_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE) - - -def _remote(text: str) -> str | None: - if _REMOTE.search(text): - return "remote" - return "hybrid" if _HYBRID.search(text) else None - - -def _text(fragment: str) -> str: - """HTML to plain text. Jobindex breaks titles after a slash with a - zero-width space, which would otherwise end up in `title_key`.""" - return html.unescape(strip_html(fragment)).replace("\u200b", "").strip() - - -def parse_jobindex(body: bytes) -> list[Row]: - return _jobindex_page(body)[0] - - -def _jobindex_page(body: bytes) -> tuple[list[Row], int]: - """One page of a Jobindex RSS search, and how many items it had before - any was dropped. An item is `` "Role, Company", - a `<link>` to Jobindex's own page for it, and a `<description>` of HTML - whose `<h4>` is the title alone, whose logo's `alt` is the company on a - paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad - ("robot job") has no logo, so its company is what follows the title's - last comma, or '' when there is none. A body with no `<channel>` is not - a search result and raises, since an empty list would read as a search - that ran out and close everything it listed before.""" - root = ET.fromstring(body) - channel = root.find("channel") - if channel is None: - raise ValueError("jobindex answered without an RSS channel") - area_match = _SEARCH_AREA.search(channel.findtext("title") or "") - search_area = area_match.group(1).strip() if area_match else "" - rows = [] - items = channel.findall("item") - for item in items: - link = (item.findtext("link") or "").strip() - full_title = (item.findtext("title") or "").strip() - if not full_title or not _web_url(link): - continue - description = item.findtext("description") or "" - h4 = _H4.search(description) - title = _text(h4.group(1)) if h4 else full_title - alt = _ALT.search(description) - company = html.unescape(alt.group(1)).strip() if alt else "" - if not company and full_title.startswith(title + ", "): - company = full_title[len(title) + 2:].strip() - area = _AREA.search(description) - posted = None - if pub := item.findtext("pubDate"): - try: - posted = parsedate_to_datetime(pub) - except (TypeError, ValueError): - posted = None - place = _text(area.group(1)) if area else search_area - rows.append({ - "id": link.rstrip("/").rsplit("/", 1)[-1], - "title": title or full_title, - "company": company, - "place": place, - "country": "DK", - "remote": _remote(f"{place} {title}"), - "url": link, - "posted": posted, - "description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)), - "salary_min": None, "salary_max": None, "currency": "", - }) - return rows, len(items) - - -def search_jobindex(client: httpx.Client, params: dict[str, Any], - keys: dict[str, Any]) -> tuple[list[Row], bool]: - """Every page of one search, newest first, until a short page or the page - limit. Returns the rows and whether the results ran out: a page with - fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so - a dropped item does not pass for the end. Jobindex states no total, so a - search that read the page limit full is never complete.""" - query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"} - if params.get("area"): - query["geoareaid"] = params["area"] - rows: list[Row] = [] - for page in range(1, _pages(params) + 1): - got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page})) - rows += got - if items < JOBINDEX_PAGE: - return rows, True - return rows, False - - -# ------------------------------------------------------------------ adzuna - -ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}" -ADZUNA_PAGE = 50 - -#: Adzuna's countries in Europe, and the currency its salaries are in. -ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR", - "be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"} - - -def parse_adzuna(body: bytes, country: str) -> list[Row]: - """One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles - and snippets carry `<strong>` highlights. A salary Adzuna estimated - (`salary_is_predicted`) is not the posting's, so it is dropped.""" - return _adzuna_rows(json.loads(body), country) - - -def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]: - rows = [] - for job in data.get("results") or []: - title = _text(job.get("title") or "") - url = (job.get("redirect_url") or "").strip() - if not title or not _web_url(url): - continue - predicted = str(job.get("salary_is_predicted") or "0") == "1" - rows.append({ - "id": str(job.get("id") or ""), - "title": title, - "company": ((job.get("company") or {}).get("display_name") or "").strip(), - "place": ((job.get("location") or {}).get("display_name") or "").strip(), - "country": country.upper(), - "remote": _remote(title), - "url": url, - "posted": _parse_iso(job.get("created")), - "description": _text(job.get("description") or ""), - "salary_min": None if predicted else job.get("salary_min"), - "salary_max": None if predicted else job.get("salary_max"), - "currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""), - }) - return rows - - -def _parse_iso(value: Any) -> datetime | None: - try: - parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) - except ValueError: - return None - return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) - - -def search_adzuna(client: httpx.Client, params: dict[str, Any], - keys: dict[str, Any]) -> tuple[list[Row], bool]: - """Every page of one search, until a short page, the stated `count` or - the page limit. Returns the rows and whether they are all `count` says - match, or, on an answer with no `count`, whether a page came back short.""" - app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "") - app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "") - if not (app_id and app_key): - raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, " - "or ADZUNA_APP_ID and ADZUNA_APP_KEY") - country = str(params.get("country") or "").lower() - if country not in ADZUNA_COUNTRIES: - raise ValueError(f"adzuna country {country!r} is not one of " - f"{', '.join(sorted(ADZUNA_COUNTRIES))}") - query: dict[str, Any] = {"app_id": app_id, "app_key": app_key, - "results_per_page": ADZUNA_PAGE, "sort_by": "date", - "what": str(params.get("q") or "")} - if params.get("where"): - query["where"] = params["where"] - rows: list[Row] = [] - seen: set[str] = set() - count: Any = None - ran_out = False - for page in range(1, _pages(params) + 1): - url = ADZUNA_SEARCH.format(country=country, page=page) - data = json.loads(_get(client, url, query)) - results = data.get("results") or [] - count = data.get("count", count) - seen |= {str(job.get("id") or "") for job in results} - rows += _adzuna_rows(data, country) - if len(results) < ADZUNA_PAGE: - ran_out = True - break - if isinstance(count, int) and len(seen) >= count: - break - # Adzuna states how many results match; the search is complete when it - # read that many distinct ones. Without a count, a short page is the end. - return rows, (len(seen) == count) if isinstance(count, int) else ran_out - - -PLATSBANKEN_SEARCH = "https://jobsearch.api.jobtechdev.se/search" -PLATSBANKEN_PAGE = 100 -#: The API refuses an `offset` above 2000, which no page limit reaches. -PLATSBANKEN_MAX_OFFSET = 2000 - -#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region` -#: concepts (which also list regions outside Sweden). "" is all of Sweden and -#: is not a key here. -PLATSBANKEN_REGIONS: dict[str, str] = { - "DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län", - "K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län", - "wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län", - "MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län", - "tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län", - "CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län", - "s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län", - "EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län", - "NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län", - "zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län", - "oLT3_Q9p_3nn": "Östergötlands län", -} - - -def parse_platsbanken(body: bytes) -> list[Row]: - """One page of JobTech's `/search`: `{"total": {"value"}, "hits": [...]}`. - A hit's `webpage_url` is Platsbanken's own page for the ad. `remote` is - read from the headline and from `workplace_model`, which the ads mostly - leave empty.""" - return _platsbanken_rows(json.loads(body)) - - -def _platsbanken_rows(data: dict[str, Any]) -> list[Row]: - rows = [] - for hit in data.get("hits") or []: - title = _text(hit.get("headline") or "") - url = (hit.get("webpage_url") or "").strip() - if not title or not _web_url(url): - continue - address = hit.get("workplace_address") or {} - model = (hit.get("workplace_model") or {}).get("label") or "" - rows.append({ - "id": str(hit.get("id") or ""), - "title": title, - "company": ((hit.get("employer") or {}).get("name") or "").strip(), - "place": (address.get("municipality") or address.get("region") or "").strip(), - "country": "SE", - "remote": _remote(f"{title} {model}"), - "url": url, - "posted": _parse_iso(hit.get("publication_date")), - "description": _text((hit.get("description") or {}).get("text") or ""), - "salary_min": None, "salary_max": None, "currency": "", - }) - return rows - - -def search_platsbanken(client: httpx.Client, params: dict[str, Any], - keys: dict[str, Any]) -> tuple[list[Row], bool]: - """Every page of one search, newest first, until a short page, the stated - total or the page limit. Returns the rows and whether they are all the - `total` says match, as for Adzuna. `region` is a key of - `PLATSBANKEN_REGIONS`, or empty for all of Sweden.""" - region = str(params.get("region") or "") - if region and region not in PLATSBANKEN_REGIONS: - raise ValueError(f"platsbanken region {region!r} is not one of the ids in " - "PLATSBANKEN_REGIONS") - query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "pubdate-desc", - "limit": PLATSBANKEN_PAGE} - if region: - query["region"] = region - rows: list[Row] = [] - seen: set[str] = set() - total: Any = None - ran_out = False - for page in range(_pages(params)): - offset = page * PLATSBANKEN_PAGE - if offset > PLATSBANKEN_MAX_OFFSET: - break - data = json.loads(_get(client, PLATSBANKEN_SEARCH, query | {"offset": offset})) - hits = data.get("hits") or [] - total = (data.get("total") or {}).get("value", total) - seen |= {str(hit.get("id") or "") for hit in hits} - rows += _platsbanken_rows(data) - if len(hits) < PLATSBANKEN_PAGE: - ran_out = True - break - if isinstance(total, int) and len(seen) >= total: - break - return rows, (len(seen) == total) if isinstance(total, int) else ran_out - - -_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]], - tuple[list[Row], bool]]] = { - "jobindex": search_jobindex, - "adzuna": search_adzuna, - "platsbanken": search_platsbanken, -} +def _limits(params: Mapping[str, Any]) -> _runner.Limits: + return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY) def provider_of(target: str) -> str: """`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one.""" provider, sep, name = target.partition(":") - return provider if sep and name.strip() and provider in _SEARCHES else "" + return provider if sep and name.strip() and provider in readers.PROVIDERS else "" + + +def _read(client: httpx.Client, target: str, params: Mapping[str, Any], + keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]: + """One search read through the runner: its provider and its listing. + Raises ValueError for an unknown provider or a missing key, before any + request.""" + provider = provider_of(target) + if not provider: + raise ValueError(f"{target!r} is not provider:name with provider one of " + f"{', '.join(readers.PROVIDERS)}") + module = readers.PROVIDERS[provider] + slug = target.partition(":")[2] + ask = _runner.ask_for(module, params, keys=keys) + get = _runner.getter(client, module.allowed(slug), provider) + return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params)) def run_search(target: str, params: dict[str, Any], keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]: """One search, outside any pull: for `app searches add`, which runs a - search once before saving it.""" - provider = provider_of(target) - if not provider: - raise ValueError(f"{target!r} is not provider:name with provider one of " - f"{', '.join(PROVIDERS)}") + search once before saving it. `keys` is the config's `sources.search`. + Each row also carries `place`, the display line of its places.""" with httpx.Client() as client: - return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {}) + _module, listing = _read(client, target, params, keys) + rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows] + return rows, listing.complete # -------------------------------------------------------------- the plugin class SearchSource(Source): name = "search" - description = "Saved searches on cross-company job boards: Jobindex, Adzuna, Platsbanken." + description = "Saved searches on cross-company job boards: " + ", ".join( + module.LABEL for module in readers.PROVIDERS.values()) + "." #: Entries are `{"url": target, "label": name, "params": {...}}`. subscription_field = "searches" @@ -447,37 +135,32 @@ class SearchSource(Source): log.warning("search: %r is not provider:name; skipped", target) continue try: - rows, complete = _SEARCHES[provider]( - client, self.params_for(target), self.options.get(provider) or {}) + _module, listing = _read(client, target, self.params_for(target), + self.options) except Exception as exc: log.warning("search: %s failed (%s)", target, exc) continue seen_ids = [] - for row in rows: - source_id = f"{provider}:{row['id']}" + for row in listing.rows: + source_id = f"{provider}:{row['source_id']}" seen_ids.append(source_id) + description = row.get("description") or "" yield Item( source_id=source_id, location=row["url"], title=row["title"], - excerpt=(row["description"] or "")[:300], - body=row["description"], - published_at=row["posted"], + excerpt=description[:300], + body=description, + published_at=row.get("posted"), subscription=label, kind="role", - raw={ - "company": row["company"], - "place": row["place"], - "country": row["country"], - "remote": row["remote"], - "salary_min": row["salary_min"], - "salary_max": row["salary_max"], - "currency": row["currency"], + raw=_runner.raw_of(row) | { + "company": row.get("company") or "", "provider": provider, "search": target, }, ) self.state[f"search:ok:{target}"] = run_marker - self.state[f"search:complete:{target}"] = "1" if complete else "" + self.state[f"search:complete:{target}"] = "1" if listing.complete else "" self.state[f"search:ids:{target}"] = json.dumps(seen_ids) diff --git a/tests/fixtures/ats/successfactors-robots.txt b/tests/fixtures/ats/successfactors-robots.txt deleted file mode 100644 index 02e3127..0000000 --- a/tests/fixtures/ats/successfactors-robots.txt +++ /dev/null @@ -1,11 +0,0 @@ -User-agent: * -Disallow: /applybutton/ -Disallow: /talentcommunity/ -Disallow: /mobile/talentcommunity/ -Disallow: /emailsubscribe/ -Disallow: /email/image/ -Disallow: /services/ -Disallow: /preapply/ -Disallow: /error -Disallow: /unsubscribe/ -Disallow: /reset/ \ No newline at end of file diff --git a/tests/test_ats.py b/tests/test_ats.py index 7285d52..3f855fe 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -19,7 +19,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats # noqa: E402 -from plugins.readers import smartrecruiters # noqa: E402 +from plugins.readers import _runner, smartrecruiters # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -487,7 +487,7 @@ def test_get_stops_reading_at_max_bytes_rather_than_after_the_body( def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive( monkeypatch: pytest.MonkeyPatch) -> None: clock = iter(range(0, 1000, 10)) - monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock))) + monkeypatch.setattr(_runner.time, "monotonic", lambda: float(next(clock))) monkeypatch.setattr(ats, "DEADLINE", 25.0) body = (b"x" for _ in range(100)) with _client(lambda request: httpx.Response(200, content=body)) as client, \ @@ -752,12 +752,13 @@ def test_teamtailor_is_read_through_the_streamed_get_and_a_redirect_elsewhere_is def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource: """One board, `greenhouse:gone`, whose fetch answers `answer` or raises it.""" - def fake(client: object, target: str) -> object: + def fake(*args: object, **kwargs: object) -> _runner.Listing: if isinstance(answer, Exception): raise answer - return answer + rows, complete = answer # type: ignore[misc] + return _runner.Listing(rows, complete, [row["source_id"] for row in rows]) - monkeypatch.setattr(ats, "_fetch_rows", fake) + monkeypatch.setattr(_runner, "read_listing", fake) source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}], "run": "run-1"}) return source diff --git a/tests/test_ats_hibob.py b/tests/test_ats_hibob.py index 9df2125..539c2b1 100644 --- a/tests/test_ats_hibob.py +++ b/tests/test_ats_hibob.py @@ -155,19 +155,6 @@ def test_an_empty_board_is_complete_and_a_401_is_a_failure(web: dict[str, Any]) ats._fetch_rows(client, "hibob:liquidia") -def test_the_robots_txt_that_is_the_apps_own_page_allows(web: dict[str, Any]) -> None: - """The host answers /robots.txt with its single-page app's HTML, a 200 - with no rules.""" - web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \ - b'<!DOCTYPE html><html><head><title>Careers' - with httpx.Client() as client: - ats._check_robots(client, "hibob:dixa") - web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \ - b"User-agent: *\nDisallow: /\n" - with pytest.raises(ats._Disallowed): - ats._check_robots(client, "hibob:dixa") - - # ------------------------------------------------------- slugs and hosts def test_the_module_is_registered_as_a_paged_host() -> None: diff --git a/tests/test_ats_hrmanager.py b/tests/test_ats_hrmanager.py index 99b38ff..1a2b123 100644 --- a/tests/test_ats_hrmanager.py +++ b/tests/test_ats_hrmanager.py @@ -133,25 +133,6 @@ def test_a_429_after_the_first_page_ends_the_read_incomplete(web: dict[str, Any] assert len(rows) == 5 and not complete -def test_robots_are_read_from_the_api_host_whose_404_allows(web: dict[str, Any]) -> None: - request = httpx.Request("GET", "https://api.hr-manager.net/robots.txt") - web["answers"]["https://api.hr-manager.net/robots.txt"] = httpx.HTTPStatusError( - "404", request=request, response=httpx.Response(404, request=request)) - state: dict[str, str] = {} - with httpx.Client() as client: - ats._check_robots(client, "hrmanager:stsgroup_tr", state) - assert '"allowed": true' in state["ats:robots:hrmanager:stsgroup_tr"] - assert web["asked"] == ["https://api.hr-manager.net/robots.txt"] - - -def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide( - web: dict[str, Any]) -> None: - web["answers"]["https://api.hr-manager.net/robots.txt"] = \ - b"User-agent: *\nDisallow: /JobPortal.svc/\n" - with httpx.Client() as client, pytest.raises(ats._Disallowed): - ats._check_robots(client, "hrmanager:stsgroup_tr") - - # ------------------------------------------------------- slugs and hosts def test_the_module_is_registered_as_a_paged_host() -> None: diff --git a/tests/test_pull.py b/tests/test_pull.py index 25bdfc5..8afdd1a 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -20,7 +20,9 @@ if str(ROOT) not in sys.path: from desk.models import Item # noqa: E402 from pipeline import Cancelled, RunContext, Tally # noqa: E402 -from plugins import ats # noqa: E402 +from app.models import Application # noqa: E402 +from plugins import readers # noqa: E402 +from plugins.readers import _runner # noqa: E402 from plugins.ats import ATSSource # noqa: E402 from app.jobs import cascade, pull # noqa: E402 from app.storage import Store # noqa: E402 @@ -202,14 +204,16 @@ def _row(job_id: str) -> dict[str, object]: "description": "", "remote": None, "salary_text": ""} -def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None: - """Every board's fetch returns `answer`, `(rows, complete)`, or raises it.""" - def fake(client: object, target: str) -> object: +def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None: + """Every board's listing is `answer`, `(rows, complete)`, or raises it; + with `filtered`, a read the board's filter narrowed.""" + def fake(*args: object, **kwargs: object) -> _runner.Listing: if isinstance(answer, Exception): raise answer - return answer + rows, complete = answer # type: ignore[misc] + return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered) - monkeypatch.setattr(ats, "_fetch_rows", fake) + monkeypatch.setattr(_runner, "read_listing", fake) def _pull(store: Store) -> dict[str, object]: @@ -275,6 +279,45 @@ def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens( assert pull.failing_boards(acme) == {} +def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row(i) for i in "1234"], True)) + _pull(acme) + uid = {r.source_id: r.uid for r in acme.roles()} + agent = acme.save_agent("Hunt") + acme.set_vote(agent.id, uid["workable:acme:2"], 0) + acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer")) + + _answer(monkeypatch, ([_row("1")], True), filtered=True) + result = _pull(acme) + assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"} + assert result["closed"] == 1 + assert acme.state["ats:filtered:workable:acme"] == "1" + + # Read without the filter again, the board closes as before. + _answer(monkeypatch, ([_row("1")], True)) + _pull(acme) + assert _open(acme) == {"workable:acme:1"} + + +def test_a_board_subscriptions_params_reach_the_runner_as_its_ask( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + store.add_subscription("ats", "workable:acme", label="Acme", company="Acme", + board="workable", params={"countries": ["dk"], "remote": True}) + asked: list[object] = [] + + def fake(module: object, slug: str, get: object, ask: object, **kwargs: object + ) -> _runner.Listing: + asked.append(ask) + return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type] + + monkeypatch.setattr(_runner, "read_listing", fake) + _pull(store) + [ask] = asked + declared = readers.HOSTS["workable"].READS.filters.declared() + assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined] + + # ------------------------------------------------------ one pull at a time def _ago(**delta: float) -> str: diff --git a/tests/test_readers.py b/tests/test_readers.py index fe1ccd8..6dd6894 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -1,6 +1,6 @@ """The host modules of `plugins/readers/` and what `plugins/ats.py` does for every host: the registry and its contract check, the fetch a host is handed, -the per-role detail reads, robots.txt, and Lever's EU instance. No network — +the per-role detail reads, and Lever's EU instance. No network — a fake host stands in where a behaviour belongs to no real host yet.""" from __future__ import annotations @@ -11,7 +11,7 @@ import subprocess import sys import types from collections.abc import Callable -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -23,6 +23,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import ats, readers # noqa: E402 +from plugins.readers import _runner # noqa: E402 from plugins.readers import lever # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -242,11 +243,9 @@ def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatc # ------------------------------------------------------- a fake host -def _fake_host(*, detail: Callable[..., Any] | None = None, - robots: bool = False) -> types.ModuleType: +def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType: """A host `fake` whose feed is a JSON list of ids, each role with a - teaser for a description, and, with `detail`, a per-role page; with - `robots`, its listing is the page robots.txt must allow.""" + teaser for a description, and, with `detail`, a per-role page.""" module: Any = types.ModuleType("fake") module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () module.slug_ok = lambda slug: slug.isalnum() @@ -260,8 +259,6 @@ def _fake_host(*, detail: Callable[..., Any] | None = None, for i in json.loads(body)] if detail is not None: module.detail = detail - if robots: - module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list" return module @@ -411,8 +408,8 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list( ids stay, and the next pull does not read them in detail again.""" base = "https://jobs.fake.example/acme" web["answers"].update({f"{base}/list": b'["2"]'}) - monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: ( - readers.HOSTS["fake"].parse(b'["2"]', "acme"), False)) + monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing( + readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"])) state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] @@ -511,7 +508,7 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps( "salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales", "title": "Another title"} - caplog.set_level("DEBUG", logger=ats.log.name) + caplog.set_level("DEBUG", logger=_runner.log.name) [item], _state = _run(monkeypatch, _fake_host(detail=detail)) assert (item.body, item.title, item.published_at) == ( "in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC)) @@ -519,8 +516,9 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps( item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000) assert (item.raw["salary_text"], item.raw["department"]) == ("", "") assert "department, salary_text, title left out" in caplog.text - assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency", - "salary_min", "salary_max", "posted"} + assert set(ats.DETAIL_FIELDS) == {"description", "location", "places", "teaser", "remote", + "country", "employment", "currency", "salary_min", + "salary_max", "posted"} def _paged_host() -> types.ModuleType: @@ -583,186 +581,6 @@ def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids( assert "ats:detailed:fake:acme" not in source.state -# ------------------------------------------------------------ robots.txt - -ROBOTS = "https://jobs.fake.example/robots.txt" - - -def test_robots_that_allow_the_page_let_the_board_be_read( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n", - "https://jobs.fake.example/acme/list": b'["1"]'}) - items, state = _run(monkeypatch, _fake_host(robots=True)) - assert len(items) == 1 - assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True - - -@pytest.mark.parametrize("robots", [ - b"User-agent: *\nDisallow: /acme/\n", - # A rule for this app by name counts as well as the one for everyone. - f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n" - .encode(), -]) -def test_robots_that_disallow_the_page_fail_the_board( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None: - web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'}) - items, state = _run(monkeypatch, _fake_host(robots=True)) - assert items == [] - assert web["asked"] == [ROBOTS] # the listing is never fetched - assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it" - assert "ats:ok:fake:acme" not in state - - -def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n", - "https://jobs.fake.example/acme/list": b'["1"]'}) - host = _fake_host(robots=True) - items, _state = _run(monkeypatch, host) - assert len(items) == 1 # paths are case-sensitive by default - host.ROBOTS_ANY_CASE = True - web["asked"].clear() - items, state = _run(monkeypatch, host) - assert items == [] - assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it" - - -@pytest.mark.parametrize(("robots", "page", "allowed"), [ - # The longest matching rule wins, whatever the order of the lines. - ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", - "https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True), - ("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True), - ("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False), - # An allow wins a tie. - ("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True), - # `*` inside a path, and the query counts as part of it. - ("User-agent: *\nDisallow: /*company\n", - "https://career5.successfactors.eu/career?company=x", False), - # `$` anchors the end. - ("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False), - ("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True), - # An empty Disallow allows. - ("User-agent: *\nDisallow:\n", "https://x.example/anything", True), - # A byte order mark and CRLF line ends. - ("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False), - # Paths are case-sensitive. - ("User-agent: *\nDisallow: /Novonesis_Careers/\n", - "https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False), - ("User-agent: *\nDisallow: /Novonesis_Careers/\n", - "https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True), - # A group for this app by name counts, a group for someone else does - # not, and a token matches whole, not as a substring. - (f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n", - "https://x.example/jobs", False), - ("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True), - ("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True), - # Two agents share one group, and a comment is no rule. - ("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n", - "https://x.example/jobs/1", False), - # A group for this app that allows does not undo the `*` group's refusal. - (f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n", - "https://x.example/jobs", False), - # Rules before any user-agent line belong to no group. - ("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True), -]) -def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None: - assert ats._robots_allows(robots.encode(), page) is allowed - - -def test_a_robots_verdict_is_kept_for_a_day( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", - "https://jobs.fake.example/acme/list": b'["1"]'}) - module = _fake_host(robots=True) - _items, state = _run(monkeypatch, module) - assert web["asked"] == [ROBOTS] - - # Within the day the kept verdict holds, even though the site changed. - web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n" - web["asked"].clear() - items, state = _run(monkeypatch, module, state) - assert (items, web["asked"]) == ([], []) - - # A day on, robots.txt is read again. - kept = json.loads(state["ats:robots:fake:acme"]) - kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat() - state["ats:robots:fake:acme"] = json.dumps(kept) - items, state = _run(monkeypatch, module, state) - assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"] - assert len(items) == 1 and "ats:failing:fake:acme" not in state - - -@pytest.mark.parametrize("code", [404, 410, 401, 403]) -def test_a_robots_txt_answering_4xx_allows_and_is_kept( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None: - web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'}) - items, state = _run(monkeypatch, _fake_host(robots=True)) - assert len(items) == 1 - assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True - - -@pytest.mark.parametrize("failure", [ - _status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"), - httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"), -]) -def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: - web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'}) - items, state = _run(monkeypatch, _fake_host(robots=True)) - assert items == [] and web["asked"] == [ROBOTS] - assert json.loads(state["ats:failing:fake:acme"])["reason"] == \ - "robots.txt could not be read" - assert "ats:robots:fake:acme" not in state - - -def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]: - """A site whose robots.txt on jobs.fake.example redirects to `location`, - where a robots.txt that disallows everything is served.""" - def handler(request: httpx.Request) -> httpx.Response: - if str(request.url) == ROBOTS: - return httpx.Response(301, headers={"location": location}) - if request.url.path == "/robots.txt": - return httpx.Response(200, text="User-agent: *\nDisallow: /\n") - return httpx.Response(200, text='["1"]') - return handler - - -def test_a_robots_txt_redirect_to_the_www_twin_is_followed( - monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True)) - with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ - pytest.raises(ats._Disallowed): - ats._check_robots(client, "fake:acme", {}) - - -@pytest.mark.parametrize("location", [ - "https://evil.example/robots.txt", "https://other.fake.example/robots.txt", - "http://www.jobs.fake.example/robots.txt", -]) -def test_a_robots_txt_redirect_anywhere_else_is_unreachable( - monkeypatch: pytest.MonkeyPatch, location: str) -> None: - monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True)) - with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable): - ats._check_robots(client, "fake:acme", {}) - - -def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True)) - with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \ - pytest.raises(ats._Refused): - ats._get(client, ROBOTS, "fake", "acme") - - -def test_fetch_board_honours_robots_txt( - monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: - web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n", - "https://jobs.fake.example/acme/list": b'["1"]'}) - monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True)) - with pytest.raises(ats._Disallowed) as caught: - ats.fetch_board("fake:acme") - assert ats.failure_reason(caught.value) == "robots.txt disallows it" - - # ----------------------------------------------------------------- Lever EU def test_a_lever_slug_may_name_the_eu_instance() -> None: diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..5a0ba43 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,420 @@ +"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a +subscription resolves to, series and their union, whole and paged listings, +completeness from the declaration, the sweep across pulls, the 10% guard, +the facets kept in state, the detail reads and the `raw` an item carries. No +network: `get` is a dict of URL -> body or exception.""" + +from __future__ import annotations + +import json +import sys +import types +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import ats, readers # noqa: E402 +from plugins.readers import Ask, Filters, Key, Reads, _runner # noqa: E402 + +NO_DELAY = _runner.Limits(page_delay=0) + + +def _status(code: int) -> httpx.HTTPStatusError: + request = httpx.Request("GET", "https://jobs.fake.example/") + return httpx.HTTPStatusError(str(code), request=request, + response=httpx.Response(code, request=request)) + + +class Web: + """A `get` answering from a dict of URL -> body or exception.""" + + def __init__(self, answers: dict[str, Any] | None = None) -> None: + self.answers = answers or {} + self.asked: list[str] = [] + + def __call__(self, url: str, **kwargs: Any) -> bytes: + self.asked.append(url) + answer = self.answers.get(url, _status(404)) + if isinstance(answer, BaseException): + raise answer + return answer + + +def _row(i: str, **extra: Any) -> dict[str, Any]: + return {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}", + "places": [{"text": "Berlin", "remote": None}]} | extra + + +def _whole(**reads: Any) -> Any: + """A declared whole-listing board: the feed is a JSON list of ids, one + feed per `ask.countries` value.""" + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () + module.slug_ok = lambda slug: slug.isalnum() + module.allowed = lambda slug: lambda h: h == "jobs.fake.example" + module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" + module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing", + "complete": "whole", "places": "many", "robots": "test"} | reads)) + module.feed_url = lambda slug, ask: (f"https://jobs.fake.example/{slug}/list" + + "".join(f"?c={c}" for c in ask.countries)) + module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] + return module + + +def _paged(pages: list[list[str]], total: int | None = None, **reads: Any) -> Any: + """A declared paged board over `pages`, the cursor being the page index.""" + module = _whole(**({"listing": "paged", "complete": "stated" if total is not None + else "end"} | reads)) + del module.feed_url, module.parse + module.asked = [] + + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + index = int(cursor or 0) + module.asked.append((index, ask.countries)) + body = get(f"https://jobs.fake.example/{slug}/page/{index}") + ids = json.loads(body) if body else pages[index] + more = index + 1 < len(pages) and bool(ids) + return ids, [{"id": i} for i in ids], total, index + 1 if more else None + + module.page = page + module.rows = lambda ids, slug: [_row(i) for i in ids] + return module + + +def _pages_web(pages: list[list[str]]) -> Web: + return Web({f"https://jobs.fake.example/acme/page/{n}": json.dumps(ids).encode() + for n, ids in enumerate(pages)}) + + +# -------------------------------------------------------------------- ask + +def test_ask_for_narrows_the_params_to_the_declared_filters() -> None: + board = _whole(filters=Filters(country="one")) + ask = _runner.ask_for(board, {"countries": ["dk", "se"], "remote": True, "places": ["X"]}) + assert (ask.countries, ask.remote, ask.places, ask.params) == (("DK", "SE"), None, (), {}) + # The params override the reach per dimension; the reach fills the rest. + both = _whole(filters=Filters(country="many", remote="many")) + ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True}) + assert (ask.countries, ask.remote) == (("DE",), True) + # A module without READS reads the whole board. + assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask() + + +def test_a_searchs_params_go_whole_into_the_ask_with_its_key( + monkeypatch: pytest.MonkeyPatch) -> None: + module = _whole(scope="search", listing="paged", complete="stated", text="excerpt", + filters=Filters(query="one", place="one"), + key=Key(name="fake", fields=("id", "secret"), env=("FAKE_ID", "FAKE_SECRET"))) + module.PARAMS = {"q": "query", "area": "place"} + monkeypatch.delenv("FAKE_ID", raising=False) + monkeypatch.setenv("FAKE_SECRET", "s") + ask = _runner.ask_for(module, {"q": "python", "area": 1, "pages": 2}, + keys={"fake": {"id": "i"}}) + assert (ask.query, ask.places) == ("python", ("1",)) + assert ask.params == {"q": "python", "area": 1, "pages": 2, + "key": {"id": "i", "secret": "s"}} + with pytest.raises(ValueError, match=r"sources\.search\.fake: id, secret, or FAKE_ID"): + _runner.ask_for(module, {"q": "x"}, keys={}) + + +def test_series_are_one_a_value_for_one_and_one_for_many() -> None: + ask = Ask(countries=("DK", "SE"), places=("Aarhus",)) + one = _whole(filters=Filters(country="one", place="many")) + assert [(a.countries, a.places) for a in _runner.series(one, ask)] == [ + (("DK",), ("Aarhus",)), (("SE",), ("Aarhus",))] + many = _whole(filters=Filters(country="many")) + assert _runner.series(many, ask) == [ask] + + +def test_ids_filters_are_mapped_through_the_facets() -> None: + module = _whole(filters=Filters(country="ids")) + module.facets = lambda get, slug: {} + ask = Ask(countries=("DK", "XX")) + [mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}}) + assert mapped.countries == ("h-1",) + # None known: the host has nothing in reach, and the read is empty. + assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == [] + listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)), + facets={"country": {}}) + assert (listing.rows, listing.complete, listing.filtered) == ([], True, True) + + +def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None: + module = _whole(filters=Filters(country="ids")) + calls: list[str] = [] + module.facets = lambda get, slug: calls.append(slug) or {"country": {"DK": "7"}} + state: dict[str, str] = {} + assert _runner.facets_for(module, "acme", Web(), Ask(), state, "k") == {} + for _ in range(2): + found = _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k") + assert found == {"country": {"DK": "7"}} + assert calls == ["acme"] + kept = json.loads(state["k"]) + kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat() + state["k"] = json.dumps(kept) + _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k") + assert calls == ["acme", "acme"] + + +# ---------------------------------------------------------------- listing + +def test_a_whole_listing_reads_one_feed_a_series_and_unions_by_id() -> None: + module = _whole(filters=Filters(country="one")) + web = Web({"https://jobs.fake.example/acme/list?c=DK": b'["1", "2"]', + "https://jobs.fake.example/acme/list?c=SE": b'["2", "3"]'}) + listing = _runner.read_listing(module, "acme", web, Ask(countries=("DK", "SE")), + limits=NO_DELAY) + assert [r["source_id"] for r in listing.rows] == ["1", "2", "3"] + assert (listing.complete, listing.filtered, listing.ids) == (True, True, ["1", "2", "3"]) + + +def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None: + """Greenhouse's shape: the feed with the text, and one without it.""" + module = _whole() + module.fallback_url = lambda slug: f"https://jobs.fake.example/{slug}/small" + web = Web({"https://jobs.fake.example/acme/list": _runner.TooLarge("over"), + "https://jobs.fake.example/acme/small": b'["1"]'}) + listing = _runner.read_listing(module, "acme", web) + assert ([r["source_id"] for r in listing.rows], listing.complete) == (["1"], True) + # A reader without a fallback fails. + del module.fallback_url + with pytest.raises(_runner.TooLarge): + _runner.read_listing(module, "acme", web) + + +def test_a_module_without_reads_gets_the_older_signatures() -> None: + module: Any = types.ModuleType("old") + module.NAME = "old" + module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list" + module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] + listing = _runner.read_listing(module, "acme", + Web({"https://jobs.fake.example/acme/list": b'["1"]'}), + Ask(countries=("DK",))) + assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete + + +def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None: + pages = [["1", "2"], ["3"]] + listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages), + limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2", "3"], True) + short = _runner.read_listing(_paged(pages, total=4), "acme", _pages_web(pages), + limits=NO_DELAY) + assert not short.complete + + +def test_the_page_cap_leaves_a_board_incomplete() -> None: + pages = [["1"], ["2"], ["3"]] + listing = _runner.read_listing(_paged(pages), "acme", _pages_web(pages), + limits=_runner.Limits(max_pages=2, page_delay=0)) + assert (listing.ids, listing.complete, listing.sweep) == (["1", "2"], False, None) + + +def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None: + """A host that ignores the cursor answers page one again; without a total + the read would never end, so a page with nothing new stops it.""" + pages = [["1", "2"], ["1", "2"], ["3"]] + module = _paged(pages) + listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2"], False) + assert [index for index, _ in module.asked] == [0, 1] + + +def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None: + pages = [["1"], ["2"]] + web = _pages_web(pages) + web.answers["https://jobs.fake.example/acme/page/1"] = _status(429) + listing = _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1"], False) + web.answers["https://jobs.fake.example/acme/page/0"] = _status(429) + with pytest.raises(httpx.HTTPStatusError): + _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY) + + +def test_a_search_is_never_complete_when_it_says_so() -> None: + pages = [["1"]] + module = _paged(pages, scope="search", complete="never", text="excerpt") + assert not _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY).complete + + +def test_paged_series_each_read_to_the_end_and_union() -> None: + pages = [["1", "2"]] + module = _paged(pages, filters=Filters(country="one")) + listing = _runner.read_listing(module, "acme", _pages_web(pages), + Ask(countries=("DK", "SE")), limits=NO_DELAY) + assert module.asked == [(0, ("DK",)), (0, ("SE",))] + assert (listing.ids, listing.complete) == (["1", "2"], True) + + +# ------------------------------------------------------------------ sweep + +def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None: + pages = [["1"], ["2"], ["3"], ["4"], ["5"]] + module = _paged(pages, total=5) + web, state = _pages_web(pages), {} + limits = _runner.Limits(max_pages=2, page_delay=0) + seen: list[tuple[list[str], bool]] = [] + for _pull in range(3): + listing = _runner.read(state, "ats", "fake:acme", module, "acme", web, limits=limits) + _runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids) + seen.append((listing.ids, listing.complete)) + assert seen == [(["1", "2"], False), (["3", "4"], False), (["1", "2", "3", "4", "5"], True)] + assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4] + assert "ats:sweep:fake:acme" not in state + assert json.loads(state["ats:count:fake:acme"])["count"] == 5 + + +def test_a_sweep_starts_again_when_the_ask_changed() -> None: + pages = [["1"], ["2"], ["3"]] + module = _paged(pages, filters=Filters(country="many")) + web, state = _pages_web(pages), {} + limits = _runner.Limits(max_pages=2, page_delay=0) + first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits) + _runner.stamp(state, "ats", "t", "r", first, first.ids) + module.asked.clear() + _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), limits=limits) + assert module.asked[0][0] == 0 + + +def test_a_board_at_its_ceiling_is_not_swept() -> None: + pages = [["1", "2"], ["3", "4"], ["5"]] + module = _paged(pages, total=5, page_size=2, ceiling=4) + listing = _runner.read({}, "ats", "t", module, "acme", _pages_web(pages), + limits=_runner.Limits(max_pages=2, page_delay=0)) + assert (listing.complete, listing.sweep) == (False, None) + + +def test_a_search_is_not_swept() -> None: + pages = [["1"], ["2"]] + module = _paged(pages, total=2) + listing = _runner.read_listing(module, "acme", _pages_web(pages), can_sweep=False, + limits=_runner.Limits(max_pages=1, page_delay=0)) + assert listing.sweep is None + + +# ----------------------------------------------------------- the 10% guard + +def test_a_suspect_read_far_under_the_last_count_fails() -> None: + module = _whole(empty="suspect") + web = Web({"https://jobs.fake.example/acme/list": json.dumps( + [str(n) for n in range(20)]).encode()}) + state: dict[str, str] = {} + listing = _runner.read(state, "ats", "t", module, "acme", web) + _runner.stamp(state, "ats", "t", "r", listing, listing.ids) + web.answers["https://jobs.fake.example/acme/list"] = b'["1"]' + with pytest.raises(_runner.Suspect, match="under 10% of the 20"): + _runner.read(state, "ats", "t", module, "acme", web) + # A reader whose empty answer is trusted reads it as it is. + module.READS = module.READS.model_copy(update={"empty": "trusted"}) + assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"] + + +def test_the_guard_compares_only_reads_that_asked_the_same() -> None: + """A board read whole last time and with a filter now lists far fewer + roles; that is the filter, not a failure.""" + module = _whole(filters=Filters(country="one")) + web = Web({"https://jobs.fake.example/acme/list": json.dumps( + [str(n) for n in range(50)]).encode(), + "https://jobs.fake.example/acme/list?c=DK": b'["1"]'}) + state: dict[str, str] = {} + listing = _runner.read(state, "ats", "t", module, "acme", web) + _runner.stamp(state, "ats", "t", "r", listing, listing.ids) + filtered = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",))) + assert filtered.ids == ["1"] and filtered.filtered + + +# ------------------------------------------------------ through ATSSource + +@pytest.fixture +def board(monkeypatch: pytest.MonkeyPatch) -> Any: + pages = [["1"], ["2"], ["3"]] + module = _paged(pages, total=3, empty="suspect") + monkeypatch.setitem(readers.HOSTS, "fake", module) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + monkeypatch.setattr(ats, "MAX_PAGES", 2) + web = _pages_web(pages) + monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: web(url)) + module.web = web + return module + + +def _source(state: dict[str, str], params: dict[str, Any] | None = None) -> ats.ATSSource: + source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme", + "params": params or {}}], "run": "r"}) + source.state = state + return source + + +def test_ats_source_sweeps_and_stamps_every_id_once_the_sweep_ends(board: Any) -> None: + state: dict[str, str] = {} + items = list(_source(state).fetch()) + assert [i.source_id for i in items] == ["fake:acme:1", "fake:acme:2"] + assert state["ats:complete:fake:acme"] == "" + assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == 2 + + items = list(_source(state).fetch()) + assert [i.source_id for i in items] == ["fake:acme:3"] + assert state["ats:complete:fake:acme"] == "1" + assert json.loads(state["ats:ids:fake:acme"]) == [ + "fake:acme:1", "fake:acme:2", "fake:acme:3"] + + +def test_ats_source_records_a_suspect_read_as_failing(board: Any) -> None: + state = {"ats:count:fake:acme": json.dumps( + {"asks": _runner.fingerprint([Ask()]), "count": 100})} + board.web.answers = {"https://jobs.fake.example/acme/page/0": b"[]"} + assert list(_source(state).fetch()) == [] + assert "under 10%" in json.loads(state["ats:failing:fake:acme"])["reason"] + assert "ats:ok:fake:acme" not in state + + +def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None: + board.READS = board.READS.model_copy(update={"filters": Filters(country="many")}) + state: dict[str, str] = {} + list(_source(state, {"countries": ["dk"]}).fetch()) + assert board.asked[0] == (0, ("DK",)) + assert state["ats:filtered:fake:acme"] == "1" + + +# -------------------------------------------------------------------- raw + +def test_raw_keeps_every_stated_key_and_the_places() -> None: + row = _row("1", places=[{"text": "Berlin", "remote": False}, + {"text": "Remote (DE)", "remote": True}], + teaser="Short.", seniority="senior", employment="full_time", country="DE", + description="Long.", posted=None, location="Berlin; Remote (DE)") + raw = _runner.raw_of(row) + assert raw["places"] == row["places"] + assert raw["place"] == "Berlin; Remote (DE)" + assert (raw["teaser"], raw["seniority"], raw["employment"], raw["country"]) == ( + "Short.", "senior", "full_time", "DE") + assert raw["location"] == "Berlin; Remote (DE)" + assert not {"description", "title", "url", "posted", "source_id"} & set(raw) + assert (raw["department"], raw["salary_text"], raw["currency"], raw["salary_min"]) == ( + "", "", "", None) + + +def test_a_row_without_places_has_its_location_as_the_one_place() -> None: + raw = _runner.raw_of({"source_id": "1", "title": "t", "url": "u", + "location": "Berlin, Munich"}) + assert raw["places"] == [{"text": "Berlin, Munich", "remote": None}] + assert raw["place"] == "Berlin, Munich" + assert _runner.raw_of({"source_id": "1", "title": "t", "url": "u"})["places"] == [] + + +def test_a_detail_location_without_places_becomes_the_one_place() -> None: + module = _whole(text="detail") + module.detail = lambda get, slug, row: {"location": "Aarhus", "employment": "contract", + "description": "Full."} + rows, done, failed = _runner.read_details(module, "fake", "acme", Web(), [_row("1")], + set(), set(), limits=NO_DELAY) + assert rows[0]["places"] == [{"text": "Aarhus", "remote": None}] + assert (rows[0]["employment"], rows[0]["description"]) == ("contract", "Full.") + assert (done, failed) == ({"fake:acme:1"}, set()) diff --git a/tests/test_search.py b/tests/test_search.py index 15fd7a3..d253985 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1,15 +1,17 @@ -"""The `search` plugin, `country_of`, and what the pull and the CLI do with -searches. No network: Jobindex's feed is a trimmed copy of a real one -(2026-09-23), Adzuna's is written from its documented response, Platsbanken's -is trimmed from a live answer (2026-09-29).""" +"""The `search` plugin over the runner and the provider readers, `country_of`, +and what the pull does with searches. No network: `_runner.fetch` is a stub, +and a fake provider stands in where the pull is what is tested. The readers' +own parsing is tested in `test_readers_.py`.""" from __future__ import annotations import json import sys +import types from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, ClassVar +from urllib.parse import parse_qs, urlsplit import pytest @@ -18,7 +20,8 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 -from plugins import search # noqa: E402 +from plugins import readers, search # noqa: E402 +from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 @@ -111,246 +114,129 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} -# ----------------------------------------------------------------- parsers +# ----------------------------------------------------- read through the runner -def test_jobindex_reads_title_company_town_and_date() -> None: - rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes()) - assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"] - paid, bare, robot = rows - # A paid ad: the company is its logo's alt text, the title its

. - assert paid["company"] == "DSV" - assert paid["title"].startswith("Dangerous Goods Compliance Manager") - assert paid["place"] == "Hedehusene" - assert paid["country"] == "DK" - assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23" - assert paid["description"].startswith("Do you want to minimise") - # A crawled ad with no company after the title's comma has none. - assert bare["company"] == "" - assert bare["place"] == "København" - # A crawled ad names its company after the title. - assert robot["title"] == "Marketing Content Creator" - assert robot["company"] == "Clerk.IO ApS" +def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]: + """`_runner.fetch` answering `answer(url)`, remembering the URLs asked.""" + asked: list[str] = [] + + def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes: + assert allow(urlsplit(url).hostname), url + asked.append(url) + return answer(url) + + monkeypatch.setattr(_runner, "fetch", fake_fetch) + monkeypatch.setattr(search, "PAGE_DELAY", 0) + return asked -def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None: - assert search._remote("Birkerød, Hybrid position") == "hybrid" - assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid" - assert search._remote("Remote, Danmark") == "remote" - assert search._remote("Hedehusene") is None +def _jobindex_page(url: str) -> bytes: + """A full Jobindex page (20 items) whatever page is asked.""" + page = int(parse_qs(urlsplit(url).query)["page"][0]) + items = "".join(f"Role {page}-{n}, Acme" + f"https://www.jobindex.dk/vis-job/r{page}x{n}" + f"<p>Teaser {n}</p>" + for n in range(20)) + return f"Jobindex{items}".encode() -def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None: - rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de") - assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused - first, predicted = rows - assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote" - assert first["description"] == "Build our backend services in Python & Go." - assert first["company"] == "Acme GmbH" - assert first["place"] == "Berlin, Deutschland" - assert first["country"] == "DE" - assert first["remote"] == "remote" - assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR") - assert (predicted["salary_min"], predicted["currency"]) == (None, "") +@pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)]) +def test_the_saved_pages_param_bounds_the_pages_read( + monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None: + urls = _web(monkeypatch, _jobindex_page) + params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else + {"pages": pages}) + rows, complete = search.run_search("jobindex:cph", params) + assert len(urls) == asked and len(rows) == 20 * asked + assert not complete # Jobindex states no total: never complete + query = parse_qs(urlsplit(urls[0]).query) + assert (query["q"], query["geoareaid"]) == (["python"], ["1"]) -def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("ADZUNA_APP_ID", raising=False) - monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) - with pytest.raises(ValueError, match="app_id"): - search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] - - -def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None: - pages: list[int] = [] - full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)] - - def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: - pages.append(params["page"]) - return b"short" if params["page"] == 3 else b"full" - - monkeypatch.setattr(search, "_get", fake_get) - monkeypatch.setattr(search, "_jobindex_page", - lambda body: (full[:5], 5) if body == b"short" else (full, len(full))) - rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type] - assert (pages, len(rows), complete) == ([1, 2, 3], 45, True) - - # Stopped by the page limit with every page full: not complete. - pages.clear() - rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] - assert (pages, complete) == ([1, 2], False) - - -def test_a_dropped_jobindex_item_does_not_pass_for_the_end( +def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body( monkeypatch: pytest.MonkeyPatch) -> None: - """A full page whose parser dropped one item still has 20 items, so the - search reads on instead of calling itself complete at 19 rows.""" - full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)] - monkeypatch.setattr(search, "_get", lambda client, url, params: b"") - monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full))) - _rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] - assert not complete + _web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes()) + source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", + "params": {"q": "x", "pages": 1}}]}) + items = list(source.fetch()) + paid = items[0] + assert paid.source_id == "jobindex:h1701165" + assert (paid.body, paid.excerpt) == ("", "") + assert paid.raw["teaser"].startswith("Do you want to minimise") + assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK" + assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}] + assert paid.raw["place"] == "Hedehusene" + assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph") + assert source.state["search:complete:jobindex:cph"] == "" -def test_a_jobindex_answer_without_a_channel_raises() -> None: - with pytest.raises(ValueError, match="channel"): - search.parse_jobindex(b"") - - -def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]: - """Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`.""" - asked: list[int] = [] - - def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: - page = int(url.rsplit("/", 1)[1]) - asked.append(page) - start = (page - 1) * search.ADZUNA_PAGE +def _adzuna(count: int | None, stock: int = 60) -> Any: + def answer(url: str) -> bytes: + page = int(urlsplit(url).path.rsplit("/", 1)[1]) + start = (page - 1) * 50 results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"} - for n in range(start, min(start + search.ADZUNA_PAGE, 60))] + for n in range(start, min(start + 50, stock))] body: dict[str, Any] = {"results": results} if count is not None: body["count"] = count return json.dumps(body).encode() - - monkeypatch.setattr(search, "_get", fake_get) - monkeypatch.setenv("ADZUNA_APP_ID", "id") - monkeypatch.setenv("ADZUNA_APP_KEY", "key") - return asked + return answer -@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)]) +def test_a_search_without_its_key_is_refused_before_any_request( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ADZUNA_APP_ID", raising=False) + monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) + urls = _web(monkeypatch, _adzuna(60)) + with pytest.raises(ValueError, match="app_id"): + search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {}) + assert urls == [] + + +def test_the_key_comes_from_the_config_before_the_environment( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ADZUNA_APP_ID", "env-id") + monkeypatch.setenv("ADZUNA_APP_KEY", "env-key") + urls = _web(monkeypatch, _adzuna(60)) + search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, + {"adzuna": {"app_id": "cfg-id"}}) + query = parse_qs(urlsplit(urls[0]).query) + assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"]) + assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1" + + +@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)]) def test_an_adzuna_search_is_complete_when_it_read_its_count( - monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None: - _adzuna(monkeypatch, count) - rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] + monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None: + _web(monkeypatch, _adzuna(count)) + rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, + {"adzuna": {"app_id": "i", "app_key": "k"}}) assert (len(rows), done) == (60, complete) + assert rows[0]["place"] == "" -def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None: - asked = _adzuna(monkeypatch, search.ADZUNA_PAGE) - rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] - assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True) - - -#: Two hits of a live `/search?q=data science®ion=CaRE_1nn_cSU` (2026-09-29), -#: trimmed to the fields the parser reads, and a third the way an ad with no -#: municipality and a remote headline comes. -PLATSBANKEN = { - "total": {"value": 32}, - "hits": [ - {"id": "31457942", "headline": "Postdoctoral Researcher", - "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942", - "publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59", - "description": {"text": "The European Spallation Source (ESS) is one of the largest.", - "text_formatted": "

The European Spallation Source (ESS)

"}, - "employer": {"name": "European Spallation Source Eric"}, - "application_details": {"url": "https://europeanspallationsource.se/careers"}, - "workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"}, - "workplace_address": {"municipality": "Lund", "region": "Skåne län", - "region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}}, - {"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision", - "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504", - "publication_date": "2026-09-03T15:46:06", - "description": {"text": "Robotics at ESS."}, - "employer": {"name": "European Spallation Source Eric"}, - "workplace_model": {"concept_id": None, "label": None}, - "workplace_address": {"municipality": None, "region": "Skåne län", - "region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}}, - {"id": "31400001", "headline": "Data scientist, distansarbete möjligt", - "webpage_url": "javascript:alert(1)", "workplace_address": {}}, - {"id": "31400002", "headline": "Data scientist, distansarbete möjligt", - "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002", - "employer": {"name": None}, "workplace_address": {}}, - ], -} - - -def test_platsbanken_reads_title_company_place_and_date() -> None: - rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode()) - assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused - first, no_town, remote = rows - assert first["title"] == "Postdoctoral Researcher" - assert first["company"] == "European Spallation Source Eric" - assert first["place"] == "Lund" - assert first["country"] == "SE" - assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942" - assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09" - assert first["description"].startswith("The European Spallation Source") - assert first["remote"] is None - assert no_town["place"] == "Skåne län" # the region when there is no municipality - assert remote["remote"] == "remote" - assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None) - - -def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None, - stock: int = 250) -> list[dict[str, Any]]: - """Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request, - stating `total`. Returns the params of each request.""" - asked: list[dict[str, Any]] = [] - - def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: - asked.append(params) - start = params["offset"] - hits = [{"id": str(n), "headline": f"Role {n}", - "webpage_url": f"https://x.example/{n}"} - for n in range(start, min(start + params["limit"], stock))] - body: dict[str, Any] = {"hits": hits} - if total is not None: - body["total"] = {"value": total} - return json.dumps(body).encode() - - monkeypatch.setattr(search, "_get", fake_get) - return asked - - -@pytest.mark.parametrize(("total", "pages", "found", "complete"), [ - (250, 5, 250, True), # a short third page ends it - (300, 5, 250, False), # the stated total is more than there was - (250, 2, 200, False), # stopped by the page limit - (None, 5, 250, True), # no total: a short page is the end -]) -def test_a_platsbanken_search_is_complete_when_it_read_its_total( - monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int, - complete: bool) -> None: - _platsbanken(monkeypatch, total) - rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type] - assert (len(rows), done) == (found, complete) - - -def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset( +def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total( monkeypatch: pytest.MonkeyPatch) -> None: - asked = _platsbanken(monkeypatch, 200, stock=200) - rows, done = search.search_platsbanken( - None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type] + def answer(url: str) -> bytes: + query = parse_qs(urlsplit(url).query) + start = int(query["offset"][0]) + hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}", + "workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]} + for n in range(start, min(start + 100, 200))] + return json.dumps({"total": {"value": 200}, "hits": hits}).encode() + + urls = _web(monkeypatch, answer) + rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"}) assert (len(rows), done) == (200, True) - assert [p["offset"] for p in asked] == [0, 100] - assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU" - assert asked[0]["limit"] == search.PLATSBANKEN_PAGE + offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls] + assert offsets == [["0"], ["100"]] + assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"] + assert rows[0]["place"] == "Lund; Malmö" -def test_a_platsbanken_search_without_a_region_sends_none( - monkeypatch: pytest.MonkeyPatch) -> None: - asked = _platsbanken(monkeypatch, 5, stock=5) - search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type] - assert "region" not in asked[0] - - -def test_a_platsbanken_search_never_asks_past_the_offset_limit( - monkeypatch: pytest.MonkeyPatch) -> None: - asked = _platsbanken(monkeypatch, 5000, stock=5000) - monkeypatch.setattr(search, "MAX_PAGES", 30) - _rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type] - assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET - assert not done - - -def test_a_platsbanken_search_refuses_an_unknown_region() -> None: - with pytest.raises(ValueError, match="region"): - search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type] - - -def test_platsbanken_regions_are_the_21_lan() -> None: - assert len(search.PLATSBANKEN_REGIONS) == 21 - assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län" +def test_an_unknown_provider_is_refused() -> None: + with pytest.raises(ValueError, match="provider"): + search.run_search("indeed:x", {"q": "x"}) def test_provider_of() -> None: @@ -369,15 +255,27 @@ class FakeConfig: path = None -def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None: - def fake(client: object, params: dict[str, Any], - keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: - return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København", - "country": "DK", "remote": None, "url": f"https://x.example/{i}", - "posted": None, "description": "", "salary_min": None, - "salary_max": None, "currency": ""} for i in ids], complete +def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None: + """A provider `fake` that lists `ids` in one page and states a total of + as many, or of one more when the search is to be incomplete.""" + module: Any = types.ModuleType("fake") + module.NAME, module.LABEL = "fake", "Fake" + module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated", + places="one", fields=Fields(country="stated"), robots="test") + module.allowed = lambda slug: lambda host: host == "search.fake.example" - monkeypatch.setitem(search._SEARCHES, "jobindex", fake) + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + assert ask.params["q"] + return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None + + module.page = page + module.rows = lambda decoded, slug: [ + {"source_id": i, "title": f"Engineer {i}", "company": "Acme", + "url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}], + "country": "DK", "remote": None, "posted": None, "description": ""} + for i in decoded] + assert not readers.problems(module, "fake") + monkeypatch.setitem(readers.PROVIDERS, "fake", module) def _pull(store: Store) -> dict[str, Any]: @@ -388,36 +286,36 @@ def _pull(store: Store) -> dict[str, Any]: def test_a_complete_search_closes_what_it_stopped_listing( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: - store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", - params={"q": "backend", "area": 1}) - _stub_search(monkeypatch, ["h1", "h2"], complete=True) + store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", + params={"q": "backend"}) + _fake_provider(monkeypatch, ["h1", "h2"], complete=True) _pull(store) roles = {r.source_id: r for r in store.roles()} - assert set(roles) == {"jobindex:h1", "jobindex:h2"} - assert roles["jobindex:h1"].country == "DK" - assert roles["jobindex:h1"].subscription_id is not None + assert set(roles) == {"fake:h1", "fake:h2"} + assert roles["fake:h1"].country == "DK" + assert roles["fake:h1"].subscription_id is not None - _stub_search(monkeypatch, ["h2"], complete=True) + _fake_provider(monkeypatch, ["h2"], complete=True) result = _pull(store) - assert {r.source_id for r in store.roles()} == {"jobindex:h2"} + assert {r.source_id for r in store.roles()} == {"fake:h2"} assert (result["searches"], result["closed"]) == (1, 1) -def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale( +def test_an_incomplete_search_closes_nothing_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: - store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", + store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) - _stub_search(monkeypatch, ["h1", "h2"], complete=False) + _fake_provider(monkeypatch, ["h1", "h2"], complete=False) _pull(store) - _stub_search(monkeypatch, ["h2"], complete=False) + _fake_provider(monkeypatch, ["h2"], complete=False) _pull(store) - assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"} + assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"} old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() with store.transaction() as conn: - conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,)) + conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,)) _pull(store) - assert {r.source_id for r in store.roles()} == {"jobindex:h2"} + assert {r.source_id for r in store.roles()} == {"fake:h2"} def test_seed_searches_imports_once(store: Store) -> None: @@ -434,24 +332,32 @@ def test_seed_searches_imports_once(store: Store) -> None: assert pull.seed_searches(config, store) == [] # type: ignore[arg-type] -# -------------------------------------------------------------------- cli - def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None: - _stub_search(monkeypatch, ["h1"], complete=True) - source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", + _fake_provider(monkeypatch, ["h1"], complete=True) + source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen", "params": {"q": "x"}}]}) items = list(source.fetch()) - assert [i.source_id for i in items] == ["jobindex:h1"] - assert items[0].raw["search"] == "jobindex:cph" - assert source.state["search:complete:jobindex:cph"] == "1" - assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]' + assert [i.source_id for i in items] == ["fake:h1"] + assert items[0].raw["search"] == "fake:cph" + assert source.state["search:complete:fake:cph"] == "1" + assert source.state["search:ids:fake:cph"] == '["fake:h1"]' + + +def test_a_search_that_fails_is_skipped_and_stamps_nothing( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ADZUNA_APP_ID", raising=False) + monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) + source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin", + "params": {"q": "x", "country": "de"}}]}) + assert list(source.fetch()) == [] + assert "search:ok:adzuna:berlin" not in source.state def test_a_search_switched_off_mid_pull_ends_the_pull_closed( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: - sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen", - board="jobindex", params={"q": "backend"}) - _stub_search(monkeypatch, ["h1"], complete=False) + sub = store.add_subscription("search", "fake:cph", label="Copenhagen", + board="fake", params={"q": "backend"}) + _fake_provider(monkeypatch, ["h1"], complete=False) batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] store.update_subscription(sub.id, subscribed=False, fetch=False)