Merge branch 'wp-b-runner'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
650ae3f13a
14 files changed
+1608
-1492
No files matched your search
+37
-3
@@ -13,7 +13,16 @@ on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
|
||||
still leaves the boards it had already finished stamped, so one bad board in a
|
||||
batch of ten does not cost the other nine their closure. A board that answered
|
||||
but was not read to its stated total (a paged host that hit a 429 or the page
|
||||
cap) stores what it read and closes nothing.
|
||||
cap) stores what it read and closes nothing; a paged board over the page cap
|
||||
is swept across pulls and closes once the sweep reached its end (see
|
||||
`plugins/readers/_runner.py`).
|
||||
|
||||
A board read with a filter (`ats:filtered:<target>`, from its subscription's
|
||||
`params`) lists only the roles in reach, so its close keeps open the roles a
|
||||
person acted on: those with a vote or an application (`_close_unlisted`). A
|
||||
filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
||||
under a tenth of what the same read listed last time is recorded by the
|
||||
runner as a failure, not as a list to close against.
|
||||
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
@@ -175,7 +184,11 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
|
||||
# A board's params are its filter; a reader's key is kept with
|
||||
# the searches' (`sources.search.<name>`).
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
@@ -252,7 +265,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
incomplete += 1
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if store.state.get(f"{source}:filtered:{sub.target}", ""):
|
||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
||||
else:
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if searched:
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
@@ -263,6 +279,24 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
||||
"""`Store.mark_closed` for a filtered read: close every open role of the
|
||||
board it did not list, except a role someone acted on, one with a vote
|
||||
(any, neutral included) or an application (which holds the notes). A
|
||||
filtered read cannot tell a role gone from a role outside its filter, and
|
||||
a role acted on stays until the board is read without the filter or
|
||||
fails for `STALE_DAYS`."""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))
|
||||
AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)""",
|
||||
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||
failed: when the failures began, and what the latest one said."""
|
||||
|
||||
+4
-4
@@ -28,7 +28,7 @@ functions are the classifier's vocabulary and the hosts are what it parses,
|
||||
and this reaches `plugins/` the same way `app.cli` does, since a plugin is
|
||||
app-agnostic and never reaches back into `app`. The
|
||||
Jobindex areas, Adzuna countries and Platsbanken regions on the Sources page
|
||||
come from `plugins.search` for the same reason.
|
||||
come from the search readers in `plugins/readers/` for the same reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -49,9 +49,9 @@ if str(_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_ROOT))
|
||||
|
||||
from plugins.ats import FUNCTIONS, HOSTS # noqa: E402
|
||||
from plugins.search import ( # noqa: E402
|
||||
ADZUNA_COUNTRIES, JOBINDEX_AREAS, PLATSBANKEN_REGIONS,
|
||||
)
|
||||
from plugins.readers.adzuna import COUNTRIES as ADZUNA_COUNTRIES # noqa: E402
|
||||
from plugins.readers.jobindex import AREAS as JOBINDEX_AREAS # noqa: E402
|
||||
from plugins.readers.platsbanken import REGIONS as PLATSBANKEN_REGIONS # noqa: E402
|
||||
|
||||
TEMPLATE_DIR = Path(__file__).resolve().parent.parent / "templates"
|
||||
STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
|
||||
|
||||
+131
-589
@@ -2,50 +2,44 @@
|
||||
`plugins/readers/`, one module per host.
|
||||
|
||||
Every one of these hosts serves a company's own list of roles as public,
|
||||
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
||||
drops off the board is *closed*, which is the cheapest staleness check there
|
||||
is. What a host says and how its feed reads is its module's; this file holds
|
||||
what every host shares: the fetch and its allow-list, paging and
|
||||
completeness, the per-role detail reads, robots.txt, the classifier, and the
|
||||
plugin itself.
|
||||
unauthenticated JSON, XML or RSS, and a role that drops off the board is
|
||||
*closed*, which is the cheapest staleness check there is. What a host says
|
||||
and how its feed reads is its module's, what every reader shares is
|
||||
`plugins/readers/_runner.py`'s; this file is the `ats` Source over the runner,
|
||||
the classifier, and the names the app and the tests reach for on `ats`.
|
||||
|
||||
The allow-list and the classifier are ported from `openings.go` and
|
||||
`cmd/seedgen/main.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
||||
`classify`'s nine functions and their keywords. The body cap is not a
|
||||
straight port — see `MAX_BYTES` below.
|
||||
`classify`'s nine functions and their keywords.
|
||||
|
||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
||||
its slug is. Every board is fetched independently; one that fails or times
|
||||
out is logged, recorded as failing under `ats:failing:<target>` (see
|
||||
`ATSSource.fetch`), and skipped, and the rest continue.
|
||||
its slug is. Every board is read independently; one that fails is logged,
|
||||
recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
||||
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
|
||||
filter, narrowed to what its reader declares (`_runner.ask_for`).
|
||||
|
||||
A host that pages states a total or does not. A board with a total counts as
|
||||
complete only when it read as many distinct roles as the total says, one
|
||||
without when it read up to an empty page (see `_paged`); a host that answers
|
||||
with the whole board at once is complete whenever it parses.
|
||||
The limits below are read when a board is read, so a test that replaces one
|
||||
on this module reaches the runner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable, Iterator, MutableMapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import (
|
||||
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, # noqa: F401
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -58,6 +52,7 @@ except ModuleNotFoundError:
|
||||
# file. So the root goes on the path then, and only then.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plugins import readers
|
||||
from plugins.readers import _runner
|
||||
from plugins.readers._shared import Get, Row
|
||||
|
||||
# The names the parsers had here before they moved to their host modules,
|
||||
@@ -70,156 +65,57 @@ from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every
|
||||
#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's
|
||||
#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers —
|
||||
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
|
||||
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
|
||||
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
|
||||
#: at its host's `fallback_url` where it has one (Greenhouse, without
|
||||
#: `?content=true`, see `_fetch_rows`), so at least the titles land.
|
||||
MAX_BYTES = 96 << 20
|
||||
#: The runner's limits, kept here as the `ats` Source's own: see `_runner`.
|
||||
MAX_BYTES = _runner.MAX_BYTES
|
||||
TIMEOUTS = _runner.TIMEOUTS
|
||||
DEADLINE = _runner.DEADLINE
|
||||
MAX_PAGES = _runner.MAX_PAGES
|
||||
PAGE_DELAY = _runner.PAGE_DELAY
|
||||
DETAIL_CAP = _runner.DETAIL_CAP
|
||||
DETAIL_FIELDS = _runner.DETAIL_FIELDS
|
||||
|
||||
#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai`
|
||||
#: failed a pull of 2026-09-28 with a read timeout. Measured live on
|
||||
#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse
|
||||
#: boards with the most open roles (databricks, stripe, anthropic, feverup,
|
||||
#: datadog), with and without `pay_transparency=true`, cached and with a
|
||||
#: cache-busting parameter (84 requests; CloudFront answered every one
|
||||
#: "Miss", so all reached the origin): the body arrives in under 1 s once it
|
||||
#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the
|
||||
#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic,
|
||||
#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise.
|
||||
#: The flag made no difference that three runs could tell apart from that
|
||||
#: noise. So a read may wait 30 s, about twice the worst first byte seen,
|
||||
#: and a whole request, redirects and body included, 60 s (see `_get`),
|
||||
#: about four times the slowest seen, which still leaves a 96 MB body room
|
||||
#: on a link far slower than the one measured.
|
||||
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
|
||||
DEADLINE = 60.0
|
||||
#: The runner's errors, under the names they had here.
|
||||
_Refused = _runner.Refused
|
||||
_TooLarge = _runner.TooLarge
|
||||
_local_host = _runner.local_host
|
||||
|
||||
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
||||
#: the order a select offers them. What the Sources page's host select is
|
||||
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
||||
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
|
||||
|
||||
#: host -> parser, for the hosts whose module parses a whole body.
|
||||
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||
|
||||
|
||||
def _limits() -> _runner.Limits:
|
||||
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
|
||||
|
||||
|
||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
"""The hosts a request or a redirect may land on for one board: what its
|
||||
host module's `allowed` says, which is one fixed API host or the
|
||||
company's own site. Anything else is refused, at the first hop and at
|
||||
every redirect: a feed is external content, and a board row must never
|
||||
be able to point this fetch at an internal address."""
|
||||
host module's `allowed` says. Anything else is refused, at the first hop
|
||||
and at every redirect."""
|
||||
module = readers.HOSTS.get(host)
|
||||
return module.allowed(slug) if module is not None else lambda h: False
|
||||
|
||||
|
||||
class _Refused(Exception):
|
||||
"""A board answered somewhere its allow-list does not cover."""
|
||||
|
||||
|
||||
class _TooLarge(ValueError):
|
||||
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
|
||||
knows the old shape still catches it; `_fetch_rows` catches it by type to
|
||||
retry at the host's `fallback_url`."""
|
||||
|
||||
|
||||
class _Disallowed(Exception):
|
||||
"""The site's robots.txt disallows the page a board is read from (see
|
||||
`_check_robots`)."""
|
||||
|
||||
|
||||
class _RobotsUnreachable(Exception):
|
||||
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
|
||||
connection error or a redirect off the site. RFC 9309 says to assume
|
||||
the site disallows everything then, so the board is not read this pull."""
|
||||
|
||||
|
||||
#: Host name suffixes that name a machine on a private network, never a
|
||||
#: company's careers site.
|
||||
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
||||
|
||||
|
||||
def _local_host(name: str) -> bool:
|
||||
"""Whether a host name is an address or a local name rather than a public
|
||||
DNS name: an IP literal (v4 or v6), anything whose last label is a number
|
||||
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the
|
||||
resolver), `localhost`, or a private suffix. A Teamtailor slug can be any
|
||||
host name, so the allow-list alone would let one point the fetch here."""
|
||||
name = name.strip("[]").rstrip(".").lower()
|
||||
try:
|
||||
ipaddress.ip_address(name)
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
last = name.rsplit(".", 1)[-1]
|
||||
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
|
||||
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
|
||||
"""Fetch one URL for a board: https only on the default port, never an
|
||||
address or a local name (`_local_host`), the host's own allow-list, at
|
||||
most three redirect hops each checked the same way, and the body cut off
|
||||
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
|
||||
top of the User-Agent and Accept every request carries, and replace
|
||||
either where they name it. With `www_twin`, a redirect may also land on
|
||||
the first URL's host with `www.` added or taken off, which only the
|
||||
robots.txt read asks for (see `_check_robots`).
|
||||
|
||||
The body is streamed and counted as it arrives, so an oversized one is
|
||||
refused at `MAX_BYTES` rather than after it has been read whole, and
|
||||
the whole request, redirects included, must end within `DEADLINE`: the
|
||||
per-read `TIMEOUTS` alone let a board that trickles bytes run on for
|
||||
good. The deadline is checked between chunks, so it can be overrun by
|
||||
at most one read timeout."""
|
||||
allow = _allowed_host(host, slug)
|
||||
if www_twin:
|
||||
bare = httpx.URL(url).host.removeprefix("www.")
|
||||
board_allow = allow
|
||||
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
|
||||
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
|
||||
deadline = time.monotonic() + DEADLINE
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
parsed = httpx.URL(current)
|
||||
if (parsed.scheme != "https" or parsed.port not in (None, 443)
|
||||
or _local_host(parsed.host) or not allow(parsed.host)):
|
||||
raise _Refused(f"{current} is not an allowed host for {host}")
|
||||
with client.stream(
|
||||
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
||||
json=(payload or {}) if method == "POST" else None,
|
||||
) as response:
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
if not location:
|
||||
raise ValueError("redirect with no location")
|
||||
current = str(httpx.URL(current).join(location))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
for chunk in response.iter_bytes():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)")
|
||||
if time.monotonic() > deadline:
|
||||
raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s",
|
||||
request=response.request)
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
raise ValueError("too many redirects")
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
"""`_runner.fetch` for one board, with this module's `MAX_BYTES` and
|
||||
`DEADLINE`."""
|
||||
return _runner.fetch(client, url, _allowed_host(host, slug), method=method,
|
||||
payload=payload, headers=headers, max_bytes=MAX_BYTES,
|
||||
deadline=DEADLINE, name=host)
|
||||
|
||||
|
||||
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
||||
"""The `get` a host module is handed: `_get` bound to one board. `_get`
|
||||
is looked up when a request is made, not when this is built, so a test
|
||||
that replaces `ats._get` reaches every host; `payload` and `headers` are
|
||||
passed on only when given, which keeps a replacement that predates them
|
||||
working."""
|
||||
is looked up when a request is made, so a test that replaces `ats._get`
|
||||
reaches every host; `payload` and `headers` are passed on only when
|
||||
given, which keeps a replacement that predates them working."""
|
||||
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
extra: dict[str, Any] = {}
|
||||
@@ -231,12 +127,6 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
||||
return get
|
||||
|
||||
|
||||
#: host -> parser, for the hosts whose module parses a whole body. Every one
|
||||
#: takes the raw response body and the slug.
|
||||
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||
|
||||
|
||||
# --------------------------------------------------------- the classifier
|
||||
|
||||
#: The fixed set of functions, and the keyword rules that map a title
|
||||
@@ -381,362 +271,40 @@ def careers_url(target: str) -> str:
|
||||
return module.careers_url(slug) if module is not None else ""
|
||||
|
||||
|
||||
#: Pages read from one paged board at most, and the pause between two of
|
||||
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
|
||||
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
|
||||
#: keeps Workable, which answers a burst with 429, from seeing one. The same
|
||||
#: pause separates two detail reads.
|
||||
MAX_PAGES = 100
|
||||
PAGE_DELAY = 0.5
|
||||
|
||||
|
||||
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""Every page of a board whose host pages, `PAGE_DELAY` apart, until the
|
||||
host has no next page, the stated total is reached, or `MAX_PAGES`.
|
||||
Returns the rows and whether the board is complete: as many distinct
|
||||
results read as the host's stated total, or, for a host that states none
|
||||
(Teamtailor), the read ended on an empty page (a page with nothing new in
|
||||
it ends the read short of that, incomplete, so an offset the host ignores
|
||||
cannot loop). Counted on the raw results, so a row the parser drops does
|
||||
not make a board incomplete, and on distinct ids, so a page that shifted
|
||||
under an offset and repeated a role does not make it complete.
|
||||
|
||||
A 429 after the first page stops the read with what it has, incomplete;
|
||||
one on the first page raises like any other failure."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
rows: dict[str, Row] = {}
|
||||
seen: set[str] = set()
|
||||
total: Any = None
|
||||
cursor: Any = None
|
||||
for page in range(MAX_PAGES):
|
||||
if page:
|
||||
time.sleep(PAGE_DELAY)
|
||||
try:
|
||||
data, results, total, cursor = module.page(get, slug, cursor)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if not page or exc.response.status_code != 429:
|
||||
raise
|
||||
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
|
||||
target, page + 1, len(seen), total)
|
||||
return list(rows.values()), False
|
||||
before = len(seen)
|
||||
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||
for row in module.rows(data, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
if (cursor is None or (isinstance(total, int) and len(seen) >= total)
|
||||
or (total is None and len(seen) == before)):
|
||||
break
|
||||
complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None
|
||||
if not complete:
|
||||
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen),
|
||||
"an unstated total" if total is None else total)
|
||||
return list(rows.values()), complete
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""One board's listing, fetched and parsed: its rows and whether they are
|
||||
the whole board. Raises on any failure — refused host, timeout, a feed
|
||||
shape the host's parser does not understand. A host that does not page
|
||||
answers with the whole board, so it is complete whenever it parses; a
|
||||
paged one is complete when `_paged` says so.
|
||||
|
||||
A body that exceeds `MAX_BYTES` gets one retry at the host's
|
||||
`fallback_url` where it has one (Greenhouse, without `?content=true`):
|
||||
the description is lost, but the titles land rather than the whole board
|
||||
being skipped."""
|
||||
def _fetch_rows(client: httpx.Client, target: str,
|
||||
ask: readers.Ask | None = None) -> tuple[list[Row], bool]:
|
||||
"""One board's listing, read and parsed, unfiltered unless `ask` says:
|
||||
its rows and whether they are the whole board. Raises on any failure.
|
||||
No sweep and no state: what `fetch_board` and the host tests read."""
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
if hasattr(module, "page"):
|
||||
return _paged(client, target)
|
||||
get = _getter(client, host, slug)
|
||||
try:
|
||||
body = get(module.feed_url(slug))
|
||||
except _TooLarge:
|
||||
if not hasattr(module, "fallback_url"):
|
||||
raise
|
||||
log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed",
|
||||
target, MAX_BYTES)
|
||||
body = get(module.fallback_url(slug))
|
||||
return module.parse(body, slug), True
|
||||
|
||||
|
||||
# -------------------------------------------------------------- robots.txt
|
||||
|
||||
#: How long a robots.txt answer holds for one board before it is read again.
|
||||
ROBOTS_TTL = timedelta(days=1)
|
||||
|
||||
|
||||
#: This app's product token, the name a robots.txt group addresses it by:
|
||||
#: the `desk` of `desk/<version> (personal aggregator)`.
|
||||
ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower()
|
||||
|
||||
#: One rule of a group: `(allow, path pattern)`.
|
||||
_Rule = tuple[bool, str]
|
||||
|
||||
|
||||
def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]:
|
||||
"""The groups of a robots.txt, per RFC 9309: one or more `user-agent`
|
||||
lines (their tokens lowercased) and the `allow` and `disallow` rules that
|
||||
follow them, up to the next `user-agent` line after a rule. Comments,
|
||||
other keys and rules before any `user-agent` line are left out, and so is
|
||||
an empty rule, which allows everything and so says nothing."""
|
||||
groups: list[tuple[set[str], list[_Rule]]] = []
|
||||
in_agents = False
|
||||
for line in text.splitlines():
|
||||
key, sep, value = line.split("#", 1)[0].partition(":")
|
||||
key, value = key.strip().lower(), value.strip()
|
||||
if not sep:
|
||||
continue
|
||||
if key == "user-agent":
|
||||
if not in_agents:
|
||||
groups.append((set(), []))
|
||||
in_agents = True
|
||||
groups[-1][0].add(value.lower())
|
||||
elif key in ("allow", "disallow") and groups:
|
||||
in_agents = False
|
||||
if value:
|
||||
groups[-1][1].append((key == "allow", value))
|
||||
return groups
|
||||
|
||||
|
||||
def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]:
|
||||
"""The rules for `token`: every group that names it (matched whole and
|
||||
ignoring case, as RFC 9309 says, not as a substring), merged; else every
|
||||
`*` group, merged; else none, which allows everything."""
|
||||
for wanted in (token, "*"):
|
||||
rules = [rule for agents, group in groups if wanted in agents for rule in group]
|
||||
if rules or any(wanted in agents for agents, _group in groups):
|
||||
return rules
|
||||
return []
|
||||
|
||||
|
||||
def _robots_pattern(path: str) -> re.Pattern[str]:
|
||||
"""A rule's path as a regex anchored at the start: `*` is any run of
|
||||
characters, and a `$` at the end anchors the end."""
|
||||
anchored = path.endswith("$")
|
||||
body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*"))
|
||||
return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL)
|
||||
|
||||
|
||||
def _robots_verdict(rules: list[_Rule], path: str) -> bool:
|
||||
"""Whether `path` is allowed by `rules`: the longest matching rule wins,
|
||||
an allow wins a tie, and no match allows. Both sides are compared
|
||||
percent-decoded, so `%7E` and `~` are one character."""
|
||||
path = unquote(path)
|
||||
best: tuple[int, bool] | None = None
|
||||
for allow, pattern in rules:
|
||||
if _robots_pattern(unquote(pattern)).match(path):
|
||||
weight = (len(pattern), allow)
|
||||
best = weight if best is None or weight > best else best
|
||||
return best is None or best[1]
|
||||
|
||||
|
||||
def _robots_allows(body: bytes, page: str) -> bool:
|
||||
"""Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules
|
||||
for every agent (`*`), and by those it has for this app's own token
|
||||
(`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this
|
||||
app by name is honoured as well. Paths are case-sensitive; a byte order
|
||||
mark is dropped."""
|
||||
groups = _robots_groups(body.decode("utf-8-sig", "replace"))
|
||||
url = httpx.URL(page)
|
||||
path = (url.raw_path or b"/").decode("ascii", "replace")
|
||||
return all(_robots_verdict(_robots_rules(groups, token), path)
|
||||
for token in ("*", ROBOTS_TOKEN))
|
||||
|
||||
|
||||
def _check_robots(client: httpx.Client, target: str,
|
||||
state: MutableMapping[str, str] | None = None) -> None:
|
||||
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
|
||||
robots.txt that disallows that page, and `_RobotsUnreachable` when it
|
||||
cannot be read. Read through `_get`, so from the board's own allowed
|
||||
hosts only, except that a redirect between the host and its `www.` twin
|
||||
is followed (at most three hops, https only); any other redirect off the
|
||||
site is unreachable. With `state`, the verdict and when it was reached
|
||||
are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
|
||||
|
||||
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
|
||||
which allows, and is kept like any other verdict. A 5xx, a 429, a
|
||||
timeout or a connection error means the file is unreachable, which the
|
||||
RFC says to treat as disallowing everything: the board is not read on
|
||||
this pull, fails with "robots.txt could not be read", and nothing is
|
||||
kept, so the next pull asks again. A 429 counts as unreachable although
|
||||
it is a 4xx, since it says "not now" rather than "there is none"."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS.get(host)
|
||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||
return
|
||||
page = module.robots_page(slug)
|
||||
key = f"ats:robots:{target}"
|
||||
now = datetime.now(UTC)
|
||||
if state is not None:
|
||||
try:
|
||||
kept = json.loads(state.get(key) or "{}")
|
||||
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
|
||||
except (ValueError, KeyError, TypeError):
|
||||
kept, fresh = {}, False
|
||||
if fresh:
|
||||
if not kept.get("allowed", True):
|
||||
raise _Disallowed(f"robots.txt disallows {page}")
|
||||
return
|
||||
url = f"https://{httpx.URL(page).host}/robots.txt"
|
||||
try:
|
||||
body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
code = exc.response.status_code
|
||||
if not 400 <= code < 500 or code == 429:
|
||||
raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
|
||||
allowed = True
|
||||
except Exception as exc:
|
||||
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
|
||||
else:
|
||||
# A host that serves a path whatever its case is matched without
|
||||
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
|
||||
if getattr(module, "ROBOTS_ANY_CASE", False):
|
||||
allowed = _robots_allows(body.lower(), page.lower())
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
if state is not None:
|
||||
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
||||
"allowed": allowed})
|
||||
if not allowed:
|
||||
raise _Disallowed(f"robots.txt disallows {page}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- details
|
||||
|
||||
#: Roles read in detail from one board in one pull at most. The rest of a
|
||||
#: large new board is read over the next pulls, 200 at a time.
|
||||
DETAIL_CAP = 200
|
||||
|
||||
#: The row keys a host's `detail` may fill: those whose column survives a
|
||||
#: later pull that sends the listing's empty value for them, so what a role's
|
||||
#: own page said is not wiped the pull after it was read. `description`,
|
||||
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
|
||||
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
|
||||
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
|
||||
#: earliest date a role ever had. A filled column is still overwritten by
|
||||
#: another filled value, so `_read_details` withholds the listing's place and
|
||||
#: remote for a role already read. `title` and `url` are not here, since the
|
||||
#: listing always states them and would overwrite them; nor are `department`
|
||||
#: (no column of its own, only `raw` and the classifier's guess) and
|
||||
#: `salary_text`, both rewritten from the listing on every pull.
|
||||
DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min",
|
||||
"salary_max", "posted")
|
||||
listing = _runner.read_listing(readers.HOSTS[host], slug, _getter(client, host, slug), ask,
|
||||
limits=_limits())
|
||||
return listing.rows, listing.complete
|
||||
|
||||
|
||||
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
||||
detailed: set[str], failed: set[str]
|
||||
) -> tuple[list[Row], set[str], set[str]]:
|
||||
"""The rows of a board whose host has `detail`, with the roles not yet
|
||||
read in detail filled from their own pages: at most `DETAIL_CAP` of them,
|
||||
`PAGE_DELAY` apart, first the roles never tried and then those whose
|
||||
detail failed before (`failed`), so a few roles that always fail cannot
|
||||
take the whole cap pull after pull. Returns the rows and the compound ids
|
||||
read (for `ats:detailed:<target>`) and failed (for
|
||||
`ats:detailfailed:<target>`).
|
||||
|
||||
Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key
|
||||
is left out with a debug line. Every row not read in detail in this call
|
||||
goes out with an empty description: a role read on an earlier pull keeps
|
||||
the one it has, since `upsert_role` never overwrites a filled column with
|
||||
an empty value, and a teaser from the listing must not replace it. A role
|
||||
read on an earlier pull goes out without the listing's place and remote
|
||||
as well, for the same reason: the listing states them every pull
|
||||
("2 Locations"), and would replace what the role's own page said.
|
||||
|
||||
A detail that answers 404 or 410 counts as read: the role is gone or has
|
||||
no page of its own, and asking again would not change that. One that
|
||||
raises otherwise is logged and the role kept in `failed`; a 429 stops the
|
||||
reads for this board until the next pull, and does not mark the role as
|
||||
failing. None of this touches completeness, which is the listing's alone.
|
||||
The ids of roles no longer listed are dropped from both sets, but only
|
||||
after a complete listing, so one cut short does not make the next pull
|
||||
read its missing roles again."""
|
||||
"""`_runner.read_details` for one board, with `rows` as the whole listing
|
||||
when `complete`."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
done, failing = set(detailed), set(failed) - set(detailed)
|
||||
|
||||
def compound(row: Row) -> str:
|
||||
return f"{host}:{slug}:{row['source_id']}"
|
||||
|
||||
unread = [row for row in rows if compound(row) not in done]
|
||||
queue = ([row for row in unread if compound(row) not in failing]
|
||||
+ [row for row in unread if compound(row) in failing])[:DETAIL_CAP]
|
||||
filled: dict[str, Row] = {}
|
||||
for row in queue:
|
||||
key = compound(row)
|
||||
time.sleep(PAGE_DELAY)
|
||||
try:
|
||||
answer = module.detail(get, slug, row)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
code = exc.response.status_code
|
||||
if code == 429:
|
||||
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
|
||||
target)
|
||||
break
|
||||
if code in (404, 410):
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
else:
|
||||
log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code)
|
||||
failing.add(key)
|
||||
continue
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
|
||||
failing.add(key)
|
||||
continue
|
||||
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
||||
log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
||||
target, row["source_id"], ", ".join(ignored))
|
||||
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
out = []
|
||||
for row in rows:
|
||||
key = compound(row)
|
||||
if key in filled:
|
||||
out.append(row | {"description": ""} | filled[key])
|
||||
elif key in detailed:
|
||||
out.append(row | {"description": "", "location": "", "remote": None})
|
||||
else:
|
||||
out.append(row | {"description": ""})
|
||||
if complete:
|
||||
listed = {compound(row) for row in rows}
|
||||
done &= listed
|
||||
failing &= listed
|
||||
return out, done, failing
|
||||
listed = {f"{host}:{slug}:{row['source_id']}" for row in rows} if complete else None
|
||||
return _runner.read_details(readers.HOSTS[host], host, slug, _getter(client, host, slug),
|
||||
rows, detailed, failed, listed=listed, limits=_limits())
|
||||
|
||||
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||
"does this board answer, what does it list, and is that all of it". For
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. The listing only, with no detail reads, and
|
||||
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
|
||||
`_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
|
||||
that cannot be checked."""
|
||||
"""Fetch and parse one board directly: no `Source`, no state, just "does
|
||||
this board answer, what does it list, and is that all of it". For `app
|
||||
companies add|check`. The listing only, unfiltered, no detail reads.
|
||||
Raises the same way `_fetch_rows` does."""
|
||||
with httpx.Client() as client:
|
||||
_check_robots(client, target)
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
def failure_reason(exc: BaseException) -> str:
|
||||
"""One short line for why a board did not answer: the status of an HTTP
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, _Disallowed):
|
||||
return "robots.txt disallows it"
|
||||
if isinstance(exc, _RobotsUnreachable):
|
||||
return "robots.txt could not be read"
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
||||
return text[:200]
|
||||
failure_reason = _runner.failure_reason
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
@@ -745,9 +313,8 @@ class ATSSource(Source):
|
||||
name = "ats"
|
||||
description = f"Company job boards: {', '.join(HOSTS.values())}."
|
||||
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
||||
#: company}` — `url` carries the `host:slug` target and `label` the
|
||||
#: company name, which is how `Source.targets()`'s generic dict reading
|
||||
#: hands both back without a plugin-specific override.
|
||||
#: company, "params": {...}}`: `url` carries the `host:slug` target,
|
||||
#: `label` the company name and `params` the board's filter.
|
||||
subscription_field = "boards"
|
||||
|
||||
spec = PluginSpec(
|
||||
@@ -763,99 +330,74 @@ class ATSSource(Source):
|
||||
)},
|
||||
)
|
||||
|
||||
def _id_set(self, key: str) -> set[str]:
|
||||
"""A JSON list of ids kept in state, as a set; empty when there is
|
||||
none or it does not parse."""
|
||||
try:
|
||||
return {str(i) for i in json.loads(self.state.get(key) or "[]")}
|
||||
except (ValueError, TypeError):
|
||||
return set()
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
with httpx.Client() as client:
|
||||
# Handed in by the batch that runs this plugin, as the marker of
|
||||
# the pull run that queued it (see `app/jobs/pull.py`), and never
|
||||
# read from state, where a later run may have replaced it. A
|
||||
# plugin run on its own — a test, a dry run — has no run to stamp
|
||||
# itself against, so it makes one up.
|
||||
# the pull run that queued it (see `app/jobs/pull.py`). A plugin
|
||||
# run on its own (a test, a dry run) makes one up.
|
||||
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
_check_robots(client, target, self.state)
|
||||
rows, complete = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
# The first failure in a row sets `since`, later ones only
|
||||
# the reason; `app.jobs.pull` closes the board's roles
|
||||
# once `since` is `STALE_DAYS` old.
|
||||
try:
|
||||
since = json.loads(self.state.get(failing, "") or "{}").get("since")
|
||||
except ValueError:
|
||||
since = None
|
||||
self.state[failing] = json.dumps({
|
||||
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"reason": failure_reason(exc)})
|
||||
continue
|
||||
self.state.pop(failing, None)
|
||||
yield from self._board(client, target, label, run_marker)
|
||||
|
||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||
detail_sets: tuple[set[str], set[str]] | None = None
|
||||
if hasattr(readers.HOSTS.get(host), "detail"):
|
||||
detailed, failed = (self._id_set(key) for key in detail_keys)
|
||||
rows, detailed, failed = _read_details(client, target, rows, complete,
|
||||
detailed, failed)
|
||||
detail_sets = (detailed, failed)
|
||||
def _board(self, client: httpx.Client, target: str, label: str,
|
||||
run_marker: str) -> Iterator[Item]:
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
ask = _runner.ask_for(module, self.params_for(target),
|
||||
keys=self.options.get("keys"))
|
||||
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
||||
limits=_limits())
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
_runner.record_failure(self.state, failing, exc)
|
||||
return
|
||||
self.state.pop(failing, None)
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
# `Store.mark_closed` compares against, not the board's own
|
||||
# bare job id.
|
||||
compound_id = f"{host}:{slug}:{row['source_id']}"
|
||||
seen_ids.append(compound_id)
|
||||
yield Item(
|
||||
source_id=compound_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": label,
|
||||
"place": row["location"],
|
||||
"department": row["department"],
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
# Every parser sets salary_text, empty or not (see
|
||||
# "salary_text" in `readers/_shared.py`); only
|
||||
# the hosts that state structured pay also set the
|
||||
# normalised figures, so the rest leave them at
|
||||
# row.get's default of no figure.
|
||||
"salary_text": row.get("salary_text", ""),
|
||||
"salary_min": row.get("salary_min"),
|
||||
"salary_max": row.get("salary_max"),
|
||||
"currency": row.get("currency", ""),
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it. `complete`
|
||||
# says whether the ids are the whole board, which the pull
|
||||
# needs before it closes what they leave out. The ids read in
|
||||
# detail are kept here too, after every row went out, so a
|
||||
# pull stopped half way reads the rest again rather than
|
||||
# losing their descriptions.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
if detail_sets is not None:
|
||||
for key, ids in zip(detail_keys, detail_sets, strict=True):
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
def compound(source_id: str) -> str:
|
||||
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
||||
return f"{host}:{slug}:{source_id}"
|
||||
|
||||
rows = listing.rows
|
||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||
detail_sets: tuple[set[str], set[str]] | None = None
|
||||
if hasattr(module, "detail"):
|
||||
detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys)
|
||||
listed = {compound(i) for i in listing.ids} if listing.complete else None
|
||||
rows, detailed, failed = _runner.read_details(
|
||||
module, host, slug, get, rows, detailed, failed, listed=listed,
|
||||
limits=_limits())
|
||||
detail_sets = (detailed, failed)
|
||||
|
||||
for row in rows:
|
||||
description = row.get("description") or ""
|
||||
yield Item(
|
||||
source_id=compound(row["source_id"]),
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted"),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {
|
||||
"company": label,
|
||||
"function": classify(row.get("department") or "", row["title"]),
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once every row went out, so a board that blew up half
|
||||
# way through is neither marked ok nor has a partial id list treated
|
||||
# as the whole of it, and a pull stopped half way reads the rest of
|
||||
# its details again.
|
||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||
[compound(i) for i in listing.ids])
|
||||
if detail_sets is not None:
|
||||
for key, ids in zip(detail_keys, detail_sets, strict=True):
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
+14
-14
@@ -129,7 +129,8 @@ exists.
|
||||
|
||||
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
||||
name; `_shared.only(host)` is the usual one. Every request goes through
|
||||
`get`, which is `ats._get` bound to the board:
|
||||
`get`, which is `_runner.fetch` bound to the target (`_runner.getter`; the
|
||||
`ats` Source binds it through `ats._get`):
|
||||
|
||||
get(url, *, method="GET", payload=None, headers=None) -> bytes
|
||||
|
||||
@@ -138,7 +139,7 @@ default User-Agent and `Accept: application/json, text/xml`, and replace
|
||||
either where they name it. `get` is https only on port 443, never to an IP
|
||||
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
|
||||
most three redirects each checked against `allowed(slug)`, cuts the body at
|
||||
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
|
||||
`_runner.MAX_BYTES` and ends the whole request within `_runner.DEADLINE`. It returns
|
||||
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
|
||||
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
|
||||
failure, and another exception (`ValueError` among them) for a refused host,
|
||||
@@ -151,8 +152,9 @@ id, to decide completeness), the total the host states or None, and the
|
||||
cursor of the next page or None after the last. The first call gets cursor
|
||||
None. A board with a total is complete when as many distinct ids were read;
|
||||
one without, when the read ended on an empty page. A 429 after the first page
|
||||
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
|
||||
apart. A paged module may also have `parse(body, slug)` for one page's body.
|
||||
ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
|
||||
`_runner.PAGE_DELAY` apart, and a board cut there is swept across pulls
|
||||
(`_runner.read_listing`). A paged module may also have `parse(body, slug)` for one page's body.
|
||||
|
||||
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
|
||||
own id for the role, unique on the board), `title`, `url` (http or https,
|
||||
@@ -183,11 +185,11 @@ keeps it.
|
||||
|
||||
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
||||
dict of the fields it found. The shared code calls it for roles not yet read
|
||||
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
|
||||
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
|
||||
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
|
||||
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
|
||||
and `posted`, the keys whose columns survive a later pull that sends the
|
||||
(`ats:detailed:<target>`), at most `_runner.DETAIL_CAP` (200) a board a
|
||||
pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that
|
||||
failed (`ats:detailfailed:<target>`). Only `_runner.DETAIL_FIELDS` are taken:
|
||||
`description`, `location`, `places`, `teaser`, `remote`, `country`,
|
||||
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the
|
||||
listing's empty value; any other key is left out with a debug log line.
|
||||
`title`, `url`, `department` and `salary_text` cannot be filled from
|
||||
`detail`: the listing rewrites them every pull. A role not read in detail
|
||||
@@ -203,11 +205,9 @@ Whoever builds a reader reads the host's robots.txt for the paths the reader
|
||||
requests and reports what it disallows, and the owner decides whether the
|
||||
host is read anyway. `robots` records that decision with its date, e.g.
|
||||
`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is
|
||||
disallowed: owner, 2026-09-30"`. The runtime check in `ats.py`, which
|
||||
`robots_page` and `ROBOTS_ANY_CASE` feed, predates this rule and goes with
|
||||
the runner; a new reader defines neither. Until then it reads
|
||||
`https://<host>/robots.txt` at most once a day per board and fails the board
|
||||
when the page is disallowed for `*` or the app's token.
|
||||
disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time;
|
||||
`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are
|
||||
read by nothing and a new reader defines neither.
|
||||
|
||||
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
||||
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
||||
|
||||
@@ -0,0 +1,712 @@
|
||||
"""The runner: what the app does for every reader, board or search, from what
|
||||
the reader declares in `READS`.
|
||||
|
||||
A reader says how its host answers; this module reads it: the guarded fetch
|
||||
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
||||
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
||||
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
||||
the sweep of an oversized paged board across pulls, the 10% guard on a
|
||||
filtered or `empty="suspect"` read (`read`), the per-role detail reads
|
||||
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
||||
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
||||
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
||||
as they were.
|
||||
|
||||
A module without `READS` is read with the older signatures, `feed_url(slug)`
|
||||
and `page(get, slug, cursor)`, and never gets an `Ask`.
|
||||
|
||||
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
||||
|
||||
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
|
||||
"seen", "rows", "whole"}`, continued by the next pull
|
||||
count `{"asks", "count"}`, the roles the last whole read listed, for
|
||||
the 10% guard
|
||||
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable, Iterable, Mapping, MutableMapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import USER_AGENT
|
||||
|
||||
from ._contract import Ask, Key, Place, Reads
|
||||
from ._shared import Get, Row
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: openings.go's 8 MB body cap is not kept: every URL fetched here is a feed
|
||||
#: on a fixed allow-list, and the real run of 2026-09-23 hit 8 MB on the
|
||||
#: biggest employers (Anduril at 41 MB, Anthropic, OpenAI, Databricks between
|
||||
#: 9 and 19 MB). 96 MB fits them; a board still over it gets one retry at its
|
||||
#: reader's `fallback_url` where it has one.
|
||||
MAX_BYTES = 96 << 20
|
||||
|
||||
#: The first byte of a Greenhouse board waited up to 16.6 s in 84 requests
|
||||
#: measured on 2026-09-29, and the body then arrived in under 1 s. So a read
|
||||
#: may wait 30 s and a whole request, redirects and body included, 60 s.
|
||||
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
|
||||
DEADLINE = 60.0
|
||||
|
||||
#: Pages read from one target in one pull at most, and the pause between two
|
||||
#: requests (pages and detail reads), which keeps Workable from answering 429.
|
||||
MAX_PAGES = 100
|
||||
PAGE_DELAY = 0.5
|
||||
|
||||
#: Roles read in detail from one board in one pull at most.
|
||||
DETAIL_CAP = 200
|
||||
|
||||
#: The row keys a reader's `detail` may fill: those whose column survives a
|
||||
#: later pull that sends the listing's empty value, so what a role's own page
|
||||
#: said is not wiped the pull after it was read. `title`, `url`, `department`
|
||||
#: and `salary_text` are left out, since the listing rewrites them every pull.
|
||||
#: Workday, SuccessFactors and Rippling state `country` and `employment` on a
|
||||
#: role's own page only.
|
||||
DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "country",
|
||||
"employment", "currency", "salary_min", "salary_max", "posted")
|
||||
|
||||
#: How long a board's `facets` are kept before they are asked again.
|
||||
FACETS_TTL = timedelta(days=1)
|
||||
|
||||
#: A filtered or `empty="suspect"` read listing under this share of what the
|
||||
#: same read listed last time counts as a failure.
|
||||
SUSPECT_SHARE = 0.1
|
||||
|
||||
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
||||
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
"""A request or a redirect aimed at a host its allow-list does not cover."""
|
||||
|
||||
|
||||
class TooLarge(ValueError):
|
||||
"""A response over the byte cap. A `ValueError`, as it was before it had a
|
||||
type of its own."""
|
||||
|
||||
|
||||
class Suspect(Exception):
|
||||
"""A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE`
|
||||
of what it listed last time."""
|
||||
|
||||
|
||||
class MissingKey(ValueError):
|
||||
"""A reader's `READS.key` has no value in the config or the environment."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Limits:
|
||||
"""What one Source allows a read: pages, the pause between requests, and
|
||||
the detail reads a pull."""
|
||||
|
||||
max_pages: int = MAX_PAGES
|
||||
page_delay: float = PAGE_DELAY
|
||||
detail_cap: int = DETAIL_CAP
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Listing:
|
||||
"""One read of a target's listing. `rows` are this pull's rows, `ids` the
|
||||
`source_id`s the read listed: this pull's, or the whole sweep's once it
|
||||
reached the end. `complete` says `ids` are the whole list. `sweep` is the
|
||||
state to keep for the next pull, None when no sweep is under way.
|
||||
`filtered` says the read asked the host for a part of the list."""
|
||||
|
||||
rows: list[Row]
|
||||
complete: bool
|
||||
ids: list[str] = field(default_factory=list)
|
||||
filtered: bool = False
|
||||
sweep: dict[str, Any] | None = None
|
||||
asks: str = ""
|
||||
|
||||
|
||||
# --------------------------------------------------------------- modules
|
||||
|
||||
def declared(module: ModuleType) -> bool:
|
||||
"""Whether `module` declares `READS`, and so takes `ask`."""
|
||||
return isinstance(getattr(module, "READS", None), Reads)
|
||||
|
||||
|
||||
def module_of(name: str) -> ModuleType | None:
|
||||
"""The reader named `name`, a board host first, else a search provider."""
|
||||
from plugins import readers
|
||||
|
||||
return readers.HOSTS.get(name) or readers.PROVIDERS.get(name)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- fetch
|
||||
|
||||
#: Host name suffixes that name a machine on a private network.
|
||||
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
||||
|
||||
|
||||
def local_host(name: str) -> bool:
|
||||
"""Whether a host name is an address or a local name rather than a public
|
||||
DNS name: an IP literal, anything whose last label is a number
|
||||
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1), `localhost`
|
||||
or a private suffix."""
|
||||
name = name.strip("[]").rstrip(".").lower()
|
||||
try:
|
||||
ipaddress.ip_address(name)
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
last = name.rsplit(".", 1)[-1]
|
||||
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
|
||||
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
|
||||
|
||||
|
||||
def fetch(client: httpx.Client, url: str, allow: Callable[[str], bool], *,
|
||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None, max_bytes: int | None = None,
|
||||
deadline: float | None = None, name: str = "") -> bytes:
|
||||
"""Fetch one URL for one target: https on port 443 only, never an address
|
||||
or a local name, only hosts `allow` accepts, at most three redirects each
|
||||
checked the same way, the body cut at `max_bytes` (`MAX_BYTES`) and the
|
||||
whole request ended within `deadline` (`DEADLINE`) seconds. A POST sends
|
||||
`payload` as JSON, `{}` when None; `headers` are added to the
|
||||
User-Agent and Accept and replace either where they name it.
|
||||
|
||||
The body is streamed and counted as it arrives, and the deadline checked
|
||||
between chunks, so a trickling answer cannot run on. Raises `Refused`,
|
||||
`TooLarge`, `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError`
|
||||
on a timeout or a connection failure, and ValueError on a broken
|
||||
redirect."""
|
||||
cap = MAX_BYTES if max_bytes is None else max_bytes
|
||||
limit = DEADLINE if deadline is None else deadline
|
||||
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
|
||||
ends = time.monotonic() + limit
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
parsed = httpx.URL(current)
|
||||
if (parsed.scheme != "https" or parsed.port not in (None, 443)
|
||||
or local_host(parsed.host) or not allow(parsed.host)):
|
||||
raise Refused(f"{current} is not an allowed host for {name or 'this reader'}")
|
||||
with client.stream(
|
||||
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
||||
json=(payload or {}) if method == "POST" else None,
|
||||
) as response:
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
if not location:
|
||||
raise ValueError("redirect with no location")
|
||||
current = str(httpx.URL(current).join(location))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
for chunk in response.iter_bytes():
|
||||
size += len(chunk)
|
||||
if size > cap:
|
||||
raise TooLarge(f"response too large (over {cap} bytes)")
|
||||
if time.monotonic() > ends:
|
||||
raise httpx.ReadTimeout(f"no whole answer within {limit:.0f} s",
|
||||
request=response.request)
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
raise ValueError("too many redirects")
|
||||
|
||||
|
||||
def getter(client: httpx.Client, allow: Callable[[str], bool], name: str = "") -> Get:
|
||||
"""The `get` a reader is handed: `fetch` bound to one target's allow-list."""
|
||||
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
return fetch(client, url, allow, method=method, payload=payload, headers=headers,
|
||||
name=name)
|
||||
return get
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- ask
|
||||
|
||||
#: A filter dimension of `Filters` -> the field of `Ask` it fills.
|
||||
_ASK_FIELD = {"country": "countries", "place": "places", "remote": "remote", "query": "query"}
|
||||
|
||||
#: Keys a subscription's params or a reach may use, -> the dimension.
|
||||
_PARAM_DIMENSION = {"countries": "country", "country": "country", "places": "place",
|
||||
"place": "place", "remote": "remote", "query": "query", "q": "query"}
|
||||
|
||||
|
||||
def _values(value: Any) -> tuple[str, ...]:
|
||||
"""A param's value as a tuple of non-empty strings."""
|
||||
items = value if isinstance(value, list | tuple | set) else [value]
|
||||
return tuple(str(v).strip() for v in items if v not in (None, "", 0) and str(v).strip())
|
||||
|
||||
|
||||
def _dimensions(module: ModuleType, given: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""`dimension -> value` from a params or reach mapping: the generic keys
|
||||
(`countries`, `places`, `remote`, `q`) and the reader's own `PARAMS`
|
||||
(saved param -> dimension, e.g. Jobindex's `area` -> `place`)."""
|
||||
names = _PARAM_DIMENSION | dict(getattr(module, "PARAMS", {}) or {})
|
||||
out: dict[str, Any] = {}
|
||||
for key, value in given.items():
|
||||
dimension = names.get(key)
|
||||
if dimension == "remote":
|
||||
if isinstance(value, bool):
|
||||
out["remote"] = value
|
||||
elif dimension == "query":
|
||||
if str(value or "").strip():
|
||||
out["query"] = str(value).strip()
|
||||
elif dimension in ("country", "place"):
|
||||
values = _values(value)
|
||||
if dimension == "country":
|
||||
values = tuple(v.upper() for v in values)
|
||||
if values:
|
||||
out[dimension] = values
|
||||
return out
|
||||
|
||||
|
||||
def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
|
||||
"""A reader's key: each of `key.fields` from the config entry
|
||||
`sources.search.<key.name>` (`keys` is `sources.search`), else from the
|
||||
environment variable at the same position in `key.env`. Raises
|
||||
`MissingKey` naming both places when a field has neither."""
|
||||
entry = (keys or {}).get(key.name) or {}
|
||||
out: dict[str, str] = {}
|
||||
for index, name in enumerate(key.fields):
|
||||
env = key.env[index] if index < len(key.env) else ""
|
||||
value = str((entry.get(name) if isinstance(entry, Mapping) else "") or "") \
|
||||
or (os.environ.get(env, "") if env else "")
|
||||
if not value:
|
||||
where = f"sources.search.{key.name}: {', '.join(key.fields)}"
|
||||
if key.env:
|
||||
where += f", or {' and '.join(key.env)}"
|
||||
raise MissingKey(f"{key.name} needs a key: {where}")
|
||||
out[name] = value
|
||||
return out
|
||||
|
||||
|
||||
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||
reach: Mapping[str, Any] | None = None, *,
|
||||
keys: Mapping[str, Any] | None = None) -> Ask:
|
||||
"""The `Ask` one target resolves to. `reach` (from the agents, package F)
|
||||
and then the subscription's own `params` (a board's override, e.g.
|
||||
`{"countries": ["DK"], "remote": true}`, or a search's saved params) set
|
||||
the dimensions, params winning per dimension, narrowed to the ones the
|
||||
reader declares in `READS.filters`. A search's params go whole into
|
||||
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
||||
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
||||
A module without `READS` gets `Ask()`."""
|
||||
if not declared(module):
|
||||
return Ask()
|
||||
reads: Reads = module.READS
|
||||
params = dict(subscription_params or {})
|
||||
wanted = _dimensions(module, reach or {}) | _dimensions(module, params)
|
||||
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()
|
||||
if name in reads.filters.declared()}
|
||||
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
|
||||
if reads.key is not None:
|
||||
extra["key"] = resolve_key(reads.key, keys)
|
||||
return Ask(**fields, params=extra)
|
||||
|
||||
|
||||
def filtered(ask: Ask) -> bool:
|
||||
"""Whether `ask` reads a part of a board: a country, a place or remote."""
|
||||
return bool(ask.countries or ask.places or ask.remote is not None)
|
||||
|
||||
|
||||
def series(module: ModuleType, ask: Ask,
|
||||
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
|
||||
"""The series of requests `ask` takes on this reader: one per value of a
|
||||
dimension declared `one` (the product when two are), one with every
|
||||
value for `many`, and host ids from `facets` for `ids` (values `facets`
|
||||
does not know are dropped; an empty list when none is left, since the
|
||||
host then has nothing in reach). `[ask]` for a module without `READS`."""
|
||||
if not declared(module):
|
||||
return [ask]
|
||||
reach = module.READS.filters
|
||||
countries, places = ask.countries, ask.places
|
||||
for dimension in ("country", "place"):
|
||||
values = countries if dimension == "country" else places
|
||||
if getattr(reach, dimension) != "ids" or not values:
|
||||
continue
|
||||
known = (facets or {}).get(dimension) or {}
|
||||
mapped = tuple(known[v] for v in values if v in known)
|
||||
if not mapped:
|
||||
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
|
||||
", ".join(values))
|
||||
return []
|
||||
if dimension == "country":
|
||||
countries = mapped
|
||||
else:
|
||||
places = mapped
|
||||
by_country = [(c,) for c in countries] if reach.country == "one" and countries \
|
||||
else [countries]
|
||||
by_place = [(p,) for p in places] if reach.place == "one" and places else [places]
|
||||
return [ask.model_copy(update={"countries": c, "places": p})
|
||||
for c, p in itertools.product(by_country, by_place)]
|
||||
|
||||
|
||||
def fingerprint(asks: Iterable[Ask]) -> str:
|
||||
"""What a read asked, without its params (which may hold a key): the same
|
||||
string for the same series, so a sweep or a count is only compared with
|
||||
a read that asked the same."""
|
||||
return json.dumps([ask.model_dump(exclude={"params"}, mode="json") for ask in asks],
|
||||
sort_keys=True)
|
||||
|
||||
|
||||
# --------------------------------------------------------------- listing
|
||||
|
||||
def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
|
||||
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||
|
||||
|
||||
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||
limits: Limits) -> dict[str, Row]:
|
||||
"""A whole listing, one request a series, unioned by `source_id`. A body
|
||||
over the cap gets one retry at the reader's `fallback_url`."""
|
||||
rows: dict[str, Row] = {}
|
||||
for index, ask in enumerate(asks):
|
||||
if index:
|
||||
time.sleep(limits.page_delay)
|
||||
url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug)
|
||||
try:
|
||||
body = get(url)
|
||||
except TooLarge:
|
||||
if not hasattr(module, "fallback_url"):
|
||||
raise
|
||||
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
||||
module.NAME, slug)
|
||||
body = get(module.fallback_url(slug))
|
||||
for row in module.parse(body, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
return rows
|
||||
|
||||
|
||||
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
|
||||
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
|
||||
can_sweep: bool = False,
|
||||
facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing:
|
||||
"""One target's listing, read and parsed. Raises on any failure of the
|
||||
first request, and on a reader that does not understand an answer.
|
||||
|
||||
A whole listing is complete whenever it parses. A paged one reads each
|
||||
series' pages, `limits.page_delay` apart, until the host has no next
|
||||
page, the stated total is reached, a page brings nothing new (a host
|
||||
that ignores the cursor, incomplete) or `limits.max_pages` pages in all.
|
||||
A series is complete when it read as many distinct result ids as its
|
||||
stated total, or, without a total, when it ended on the host's last
|
||||
page; never for `complete="never"`. The read is complete when every
|
||||
series is. A 429 after the first page ends the read, incomplete.
|
||||
|
||||
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
||||
the `sweep` to keep, and the next pull, handed it back, continues from
|
||||
its series and cursor; the sweep is complete when it reaches the end,
|
||||
and `ids` then hold every id it listed. A read at the reader's `ceiling`
|
||||
is not swept: the host answers nothing past it."""
|
||||
limits = limits or Limits()
|
||||
ask = ask or Ask()
|
||||
asks = series(module, ask, facets)
|
||||
is_filtered = filtered(ask)
|
||||
key = fingerprint(asks)
|
||||
if not asks:
|
||||
return Listing([], True, [], is_filtered, None, key)
|
||||
if not hasattr(module, "page"):
|
||||
rows = _whole(module, slug, get, asks, limits)
|
||||
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key)
|
||||
|
||||
reads: Reads | None = module.READS if declared(module) else None
|
||||
never = reads is not None and reads.complete == "never"
|
||||
ceiling = reads.ceiling if reads is not None else None
|
||||
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
||||
if sweep and sweep.get("asks") == key:
|
||||
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
|
||||
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
|
||||
whole = bool(sweep.get("whole", True))
|
||||
rows: dict[str, Row] = {}
|
||||
pages = 0
|
||||
stop: str = ""
|
||||
index = start
|
||||
for index in range(start, len(asks)):
|
||||
total: Any = None
|
||||
ended = stalled = False
|
||||
while True:
|
||||
if pages >= limits.max_pages:
|
||||
stop = "cap"
|
||||
break
|
||||
if pages:
|
||||
time.sleep(limits.page_delay)
|
||||
try:
|
||||
if reads is not None:
|
||||
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
||||
else:
|
||||
data, results, total, cursor = module.page(get, slug, cursor)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if not pages or exc.response.status_code != 429:
|
||||
raise
|
||||
log.warning("readers: %s:%s: 429 on page %d; kept %d, not complete",
|
||||
module.NAME, slug, pages + 1, len(rows))
|
||||
stop = "429"
|
||||
break
|
||||
pages += 1
|
||||
before = len(seen)
|
||||
seen |= _result_ids(results)
|
||||
for row in module.rows(data, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||
ended = True
|
||||
break
|
||||
if total is None and len(seen) == before:
|
||||
stalled = True
|
||||
break
|
||||
if stop:
|
||||
break
|
||||
done = (not never and not stalled
|
||||
and (len(seen) >= total if isinstance(total, int) else ended))
|
||||
if not done:
|
||||
log.warning("readers: %s:%s: read %d of %s; not complete", module.NAME, slug,
|
||||
len(seen), "an unstated total" if total is None else total)
|
||||
whole = whole and done
|
||||
cursor, seen = None, set()
|
||||
|
||||
listed = list(dict.fromkeys(earlier + list(rows)))
|
||||
if not stop:
|
||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key)
|
||||
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
||||
module.NAME, slug, pages, stop)
|
||||
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
||||
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key)
|
||||
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
||||
"rows": listed, "whole": whole}
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key)
|
||||
|
||||
|
||||
def _storable(cursor: Any) -> bool:
|
||||
"""Whether a cursor survives a trip through JSON in state."""
|
||||
try:
|
||||
return json.loads(json.dumps(cursor)) == cursor
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
# ------------------------------------------------------------ with state
|
||||
|
||||
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(state.get(key) or "{}")
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
||||
state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]:
|
||||
"""The reader's `facets` when `ask` sets a dimension it declares `ids`,
|
||||
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
|
||||
if not declared(module) or not hasattr(module, "facets"):
|
||||
return {}
|
||||
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
|
||||
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name in ("country", "place")):
|
||||
return {}
|
||||
now = datetime.now(UTC)
|
||||
if state is not None and key:
|
||||
kept = _load(state, key)
|
||||
try:
|
||||
if now - datetime.fromisoformat(kept["checked"]) < FACETS_TTL:
|
||||
return dict(kept.get("facets") or {})
|
||||
except (KeyError, TypeError, ValueError):
|
||||
pass
|
||||
found = dict(module.facets(get, slug) or {})
|
||||
if state is not None and key:
|
||||
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
||||
"facets": found})
|
||||
return found
|
||||
|
||||
|
||||
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
|
||||
slug: str, get: Get, ask: Ask | None = None, *,
|
||||
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
|
||||
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
||||
the facets it needs, the sweep it continues, and the 10% guard. A read
|
||||
that is filtered or whose reader declares `empty="suspect"`, and that
|
||||
listed under `SUSPECT_SHARE` of what the same read listed last time,
|
||||
raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
||||
the rest once the rows are stored."""
|
||||
ask = ask or Ask()
|
||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets)
|
||||
suspect = declared(module) and module.READS.empty == "suspect"
|
||||
last = _load(state, f"{prefix}:count:{target}")
|
||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||
if ((listing.filtered or suspect) and listing.sweep is None and count
|
||||
and len(listing.ids) < SUSPECT_SHARE * count):
|
||||
raise Suspect(f"listed {len(listing.ids)} roles, under "
|
||||
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
||||
return listing
|
||||
|
||||
|
||||
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
listing: Listing, ids: list[str]) -> None:
|
||||
"""Record a read whose rows are stored: `ok` (the run's marker),
|
||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||
`filtered`, the sweep to continue or none, and the count for the 10%
|
||||
guard once a read is whole."""
|
||||
state[f"{prefix}:ok:{target}"] = run
|
||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||
if listing.sweep is not None:
|
||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||
else:
|
||||
state.pop(f"{prefix}:sweep:{target}", None)
|
||||
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
|
||||
"count": len(listing.ids)})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- details
|
||||
|
||||
def id_set(state: Mapping[str, str], key: str) -> set[str]:
|
||||
"""A JSON list of ids kept in state, as a set; empty when there is none
|
||||
or it does not parse."""
|
||||
try:
|
||||
return {str(i) for i in json.loads(state.get(key) or "[]")}
|
||||
except (TypeError, ValueError):
|
||||
return set()
|
||||
|
||||
|
||||
def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[Row],
|
||||
detailed: set[str], failed: set[str], *, listed: set[str] | None = None,
|
||||
limits: Limits | None = None) -> tuple[list[Row], set[str], set[str]]:
|
||||
"""The rows with the roles not yet read in detail filled from their own
|
||||
pages, at most `limits.detail_cap`, `limits.page_delay` apart: first the
|
||||
roles never tried, then those in `failed`. Ids are compound,
|
||||
`<name>:<slug>:<source_id>`. Returns the rows and the ids read and
|
||||
failed, to keep as the caller's `detailed` and `failed` sets.
|
||||
|
||||
Only `DETAIL_FIELDS` are taken from what `detail` returns. A `location`
|
||||
without `places` becomes the one place. Every row not read now goes out
|
||||
with an empty description, so one read earlier stays, and a row read on
|
||||
an earlier pull without the listing's place and remote, so what its page
|
||||
said stays. A 404 or 410 counts as read, a 429 stops the reads for this
|
||||
pull, anything else keeps the role in `failed`. `listed` is every id of a
|
||||
complete listing: ids outside it leave both sets."""
|
||||
limits = limits or Limits()
|
||||
done, failing = set(detailed), set(failed) - set(detailed)
|
||||
|
||||
def compound(row: Row) -> str:
|
||||
return f"{name}:{slug}:{row['source_id']}"
|
||||
|
||||
unread = [row for row in rows if compound(row) not in done]
|
||||
queue = ([row for row in unread if compound(row) not in failing]
|
||||
+ [row for row in unread if compound(row) in failing])[:limits.detail_cap]
|
||||
filled: dict[str, Row] = {}
|
||||
for row in queue:
|
||||
key = compound(row)
|
||||
time.sleep(limits.page_delay)
|
||||
try:
|
||||
answer = module.detail(get, slug, row)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
code = exc.response.status_code
|
||||
if code == 429:
|
||||
log.warning("readers: %s:%s: 429 on a detail read; the rest wait", name, slug)
|
||||
break
|
||||
if code in (404, 410):
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
else:
|
||||
log.warning("readers: %s:%s: detail of %s answered %s", name, slug,
|
||||
row["source_id"], code)
|
||||
failing.add(key)
|
||||
continue
|
||||
except Exception as exc:
|
||||
log.warning("readers: %s:%s: detail of %s failed (%s)", name, slug,
|
||||
row["source_id"], exc)
|
||||
failing.add(key)
|
||||
continue
|
||||
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
||||
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
||||
name, slug, row["source_id"], ", ".join(ignored))
|
||||
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment]
|
||||
if taken.get("location") and "places" not in taken:
|
||||
taken["places"] = [{"text": taken["location"], "remote": None}]
|
||||
filled[key] = taken
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
out: list[Row] = []
|
||||
for row in rows:
|
||||
key = compound(row)
|
||||
if key in filled:
|
||||
out.append(row | {"description": ""} | filled[key]) # type: ignore[operator]
|
||||
elif key in detailed:
|
||||
out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator]
|
||||
"remote": None})
|
||||
else:
|
||||
out.append(row | {"description": ""}) # type: ignore[operator]
|
||||
if listed is not None:
|
||||
done &= listed
|
||||
failing &= listed
|
||||
return out, done, failing
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- raw
|
||||
|
||||
def places_of(row: Mapping[str, Any]) -> list[Place]:
|
||||
"""A row's places: its `places`, or, for a row without (an undeclared
|
||||
reader's), its `location` as the one place, split nothing."""
|
||||
places = row.get("places")
|
||||
if not isinstance(places, list):
|
||||
location = str(row.get("location") or "").strip()
|
||||
return [{"text": location, "remote": None}] if location else []
|
||||
out: list[Place] = []
|
||||
for place in places:
|
||||
if isinstance(place, Mapping) and str(place.get("text") or "").strip():
|
||||
out.append({"text": str(place["text"]).strip(), "remote": place.get("remote")})
|
||||
return out
|
||||
|
||||
|
||||
def raw_of(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""The `raw` of an `Item` for one row: every key the row states except
|
||||
those the `Item` carries itself (`ITEM_KEYS`), with `places` always a
|
||||
list, `place` the display line of the places joined with "; ", and the
|
||||
keys storage reads always present (`department`, `remote`,
|
||||
`salary_text`, `salary_min`, `salary_max`, `currency`, `teaser`)."""
|
||||
places = places_of(row)
|
||||
raw = {k: v for k, v in row.items() if k not in ITEM_KEYS}
|
||||
return raw | {
|
||||
"places": places,
|
||||
"place": "; ".join(place["text"] for place in places),
|
||||
"department": row.get("department") or "",
|
||||
"remote": row.get("remote"),
|
||||
"salary_text": row.get("salary_text") or "",
|
||||
"salary_min": row.get("salary_min"),
|
||||
"salary_max": row.get("salary_max"),
|
||||
"currency": row.get("currency") or "",
|
||||
"teaser": row.get("teaser") or "",
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------- failures
|
||||
|
||||
def failure_reason(exc: BaseException) -> str:
|
||||
"""One short line for why a target did not answer: the status of an HTTP
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
||||
return text[:200]
|
||||
|
||||
|
||||
def record_failure(state: MutableMapping[str, str], key: str, exc: BaseException) -> None:
|
||||
"""Keep a failure under `key` as `{"since", "reason"}`: the first failure
|
||||
in a row sets `since`, later ones only the reason. The pull closes a
|
||||
target's roles once `since` is old enough; its next answer removes the
|
||||
key."""
|
||||
since = _load(state, key).get("since")
|
||||
state[key] = json.dumps({"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"reason": failure_reason(exc)})
|
||||
+64
-381
@@ -5,421 +5,109 @@ roles at companies nobody has listed yet, by asking a cross-company board a
|
||||
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
|
||||
A subscription is one saved search; its `params` are the question.
|
||||
|
||||
Three boards, checked live on 2026-09-23 and, for Platsbanken, 2026-09-29:
|
||||
|
||||
jobindex Denmark's largest job board. Every search result page has an RSS
|
||||
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
|
||||
own syntax: words are OR, `"a b"` a phrase, `+word` required.
|
||||
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region
|
||||
Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus.
|
||||
20 results a page.
|
||||
adzuna An aggregator with a documented API for most of Europe, the UK
|
||||
and the US, but not Denmark (adzuna.dk does not exist). Needs a
|
||||
free key from developer.adzuna.com, as `sources.search.adzuna:
|
||||
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
|
||||
against Adzuna's documented response and not yet run live.
|
||||
platsbanken Sweden's public employment service, through the JobTech search
|
||||
API at jobsearch.api.jobtechdev.se, no key needed. `region` is
|
||||
the id of a län in the JobTech taxonomy (Skåne län is
|
||||
`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page,
|
||||
and an `offset` above 2000 is refused. It states `total`. It has
|
||||
no remote flag worth reading (`workplace_model` is mostly
|
||||
empty), so `remote` comes from the headline.
|
||||
|
||||
Considered and left out: Jobnet (its search endpoint answers 401 outside the
|
||||
browser session of its own site) and Jooble (a key needs a manual
|
||||
application, and its API host answered a Cloudflare challenge).
|
||||
The providers are the readers of `scope="search"` in `plugins/readers/`
|
||||
(`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other
|
||||
reader: the search's params go in `Ask.params`, a declared key is resolved
|
||||
from `sources.search.<name>` or the environment and refused when missing,
|
||||
and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most
|
||||
`MAX_PAGES`).
|
||||
|
||||
A search result list is never a board's complete list, so a role missing from
|
||||
one run is not closed here. A search that read everything that matches was
|
||||
complete, and the pull closes what it stopped listing; the rest close once no
|
||||
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna and
|
||||
Platsbanken state how many results match, so their searches are complete when
|
||||
they read that many; Jobindex states no count, so a Jobindex search is
|
||||
complete when a page within the page limit came back short.
|
||||
complete (`complete="stated"`, as many distinct results as the stated total),
|
||||
and the pull closes what it stopped listing; the rest close once no search
|
||||
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable, Iterator
|
||||
import sys
|
||||
from collections.abc import Iterator, Mapping
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import (
|
||||
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
|
||||
)
|
||||
|
||||
try:
|
||||
from plugins import readers
|
||||
except ModuleNotFoundError:
|
||||
# Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plugins import readers
|
||||
from plugins.readers import _runner
|
||||
from plugins.readers._shared import Row
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
TIMEOUT = 15.0
|
||||
|
||||
#: Pages fetched per search unless its params say otherwise, and the most a
|
||||
#: Pages read per search unless its params say otherwise, and the most a
|
||||
#: search may ask for. A search that needs more than ten pages is too broad to
|
||||
#: read anyway.
|
||||
DEFAULT_PAGES = 5
|
||||
MAX_PAGES = 10
|
||||
|
||||
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
|
||||
|
||||
#: What every provider turns a result into.
|
||||
Row = dict[str, Any]
|
||||
#: The pause between two pages of one search.
|
||||
PAGE_DELAY = _runner.PAGE_DELAY
|
||||
|
||||
|
||||
def _web_url(url: str) -> bool:
|
||||
parsed = httpx.URL((url or "").strip())
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
||||
|
||||
|
||||
def _pages(params: dict[str, Any]) -> int:
|
||||
def _pages(params: Mapping[str, Any]) -> int:
|
||||
try:
|
||||
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_PAGES
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes:
|
||||
response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True,
|
||||
headers={"User-Agent": USER_AGENT})
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- jobindex
|
||||
|
||||
JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss"
|
||||
JOBINDEX_PAGE = 20
|
||||
|
||||
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
|
||||
#: against the titles its feeds answer with. 0 is all of Denmark.
|
||||
JOBINDEX_AREAS: dict[int, str] = {
|
||||
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
|
||||
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
|
||||
8: "Aarhus", 7: "Aalborg",
|
||||
}
|
||||
|
||||
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
|
||||
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
|
||||
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
|
||||
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
|
||||
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
|
||||
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
||||
|
||||
|
||||
#: What a Danish ad's town line says about where the work is done:
|
||||
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
||||
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
|
||||
|
||||
|
||||
def _remote(text: str) -> str | None:
|
||||
if _REMOTE.search(text):
|
||||
return "remote"
|
||||
return "hybrid" if _HYBRID.search(text) else None
|
||||
|
||||
|
||||
def _text(fragment: str) -> str:
|
||||
"""HTML to plain text. Jobindex breaks titles after a slash with a
|
||||
zero-width space, which would otherwise end up in `title_key`."""
|
||||
return html.unescape(strip_html(fragment)).replace("\u200b", "").strip()
|
||||
|
||||
|
||||
def parse_jobindex(body: bytes) -> list[Row]:
|
||||
return _jobindex_page(body)[0]
|
||||
|
||||
|
||||
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
|
||||
"""One page of a Jobindex RSS search, and how many items it had before
|
||||
any was dropped. An item is `<title>` "Role, Company",
|
||||
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
|
||||
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
|
||||
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
|
||||
("robot job") has no logo, so its company is what follows the title's
|
||||
last comma, or '' when there is none. A body with no `<channel>` is not
|
||||
a search result and raises, since an empty list would read as a search
|
||||
that ran out and close everything it listed before."""
|
||||
root = ET.fromstring(body)
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
raise ValueError("jobindex answered without an RSS channel")
|
||||
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
||||
search_area = area_match.group(1).strip() if area_match else ""
|
||||
rows = []
|
||||
items = channel.findall("item")
|
||||
for item in items:
|
||||
link = (item.findtext("link") or "").strip()
|
||||
full_title = (item.findtext("title") or "").strip()
|
||||
if not full_title or not _web_url(link):
|
||||
continue
|
||||
description = item.findtext("description") or ""
|
||||
h4 = _H4.search(description)
|
||||
title = _text(h4.group(1)) if h4 else full_title
|
||||
alt = _ALT.search(description)
|
||||
company = html.unescape(alt.group(1)).strip() if alt else ""
|
||||
if not company and full_title.startswith(title + ", "):
|
||||
company = full_title[len(title) + 2:].strip()
|
||||
area = _AREA.search(description)
|
||||
posted = None
|
||||
if pub := item.findtext("pubDate"):
|
||||
try:
|
||||
posted = parsedate_to_datetime(pub)
|
||||
except (TypeError, ValueError):
|
||||
posted = None
|
||||
place = _text(area.group(1)) if area else search_area
|
||||
rows.append({
|
||||
"id": link.rstrip("/").rsplit("/", 1)[-1],
|
||||
"title": title or full_title,
|
||||
"company": company,
|
||||
"place": place,
|
||||
"country": "DK",
|
||||
"remote": _remote(f"{place} {title}"),
|
||||
"url": link,
|
||||
"posted": posted,
|
||||
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
||||
"salary_min": None, "salary_max": None, "currency": "",
|
||||
})
|
||||
return rows, len(items)
|
||||
|
||||
|
||||
def search_jobindex(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, newest first, until a short page or the page
|
||||
limit. Returns the rows and whether the results ran out: a page with
|
||||
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
|
||||
a dropped item does not pass for the end. Jobindex states no total, so a
|
||||
search that read the page limit full is never complete."""
|
||||
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
|
||||
if params.get("area"):
|
||||
query["geoareaid"] = params["area"]
|
||||
rows: list[Row] = []
|
||||
for page in range(1, _pages(params) + 1):
|
||||
got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
||||
rows += got
|
||||
if items < JOBINDEX_PAGE:
|
||||
return rows, True
|
||||
return rows, False
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ adzuna
|
||||
|
||||
ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
|
||||
ADZUNA_PAGE = 50
|
||||
|
||||
#: Adzuna's countries in Europe, and the currency its salaries are in.
|
||||
ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
|
||||
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
|
||||
|
||||
|
||||
def parse_adzuna(body: bytes, country: str) -> list[Row]:
|
||||
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
|
||||
and snippets carry `<strong>` highlights. A salary Adzuna estimated
|
||||
(`salary_is_predicted`) is not the posting's, so it is dropped."""
|
||||
return _adzuna_rows(json.loads(body), country)
|
||||
|
||||
|
||||
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
|
||||
rows = []
|
||||
for job in data.get("results") or []:
|
||||
title = _text(job.get("title") or "")
|
||||
url = (job.get("redirect_url") or "").strip()
|
||||
if not title or not _web_url(url):
|
||||
continue
|
||||
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
||||
rows.append({
|
||||
"id": str(job.get("id") or ""),
|
||||
"title": title,
|
||||
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
|
||||
"place": ((job.get("location") or {}).get("display_name") or "").strip(),
|
||||
"country": country.upper(),
|
||||
"remote": _remote(title),
|
||||
"url": url,
|
||||
"posted": _parse_iso(job.get("created")),
|
||||
"description": _text(job.get("description") or ""),
|
||||
"salary_min": None if predicted else job.get("salary_min"),
|
||||
"salary_max": None if predicted else job.get("salary_max"),
|
||||
"currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_iso(value: Any) -> datetime | None:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, until a short page, the stated `count` or
|
||||
the page limit. Returns the rows and whether they are all `count` says
|
||||
match, or, on an answer with no `count`, whether a page came back short."""
|
||||
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
|
||||
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
|
||||
if not (app_id and app_key):
|
||||
raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, "
|
||||
"or ADZUNA_APP_ID and ADZUNA_APP_KEY")
|
||||
country = str(params.get("country") or "").lower()
|
||||
if country not in ADZUNA_COUNTRIES:
|
||||
raise ValueError(f"adzuna country {country!r} is not one of "
|
||||
f"{', '.join(sorted(ADZUNA_COUNTRIES))}")
|
||||
query: dict[str, Any] = {"app_id": app_id, "app_key": app_key,
|
||||
"results_per_page": ADZUNA_PAGE, "sort_by": "date",
|
||||
"what": str(params.get("q") or "")}
|
||||
if params.get("where"):
|
||||
query["where"] = params["where"]
|
||||
rows: list[Row] = []
|
||||
seen: set[str] = set()
|
||||
count: Any = None
|
||||
ran_out = False
|
||||
for page in range(1, _pages(params) + 1):
|
||||
url = ADZUNA_SEARCH.format(country=country, page=page)
|
||||
data = json.loads(_get(client, url, query))
|
||||
results = data.get("results") or []
|
||||
count = data.get("count", count)
|
||||
seen |= {str(job.get("id") or "") for job in results}
|
||||
rows += _adzuna_rows(data, country)
|
||||
if len(results) < ADZUNA_PAGE:
|
||||
ran_out = True
|
||||
break
|
||||
if isinstance(count, int) and len(seen) >= count:
|
||||
break
|
||||
# Adzuna states how many results match; the search is complete when it
|
||||
# read that many distinct ones. Without a count, a short page is the end.
|
||||
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
|
||||
|
||||
|
||||
PLATSBANKEN_SEARCH = "https://jobsearch.api.jobtechdev.se/search"
|
||||
PLATSBANKEN_PAGE = 100
|
||||
#: The API refuses an `offset` above 2000, which no page limit reaches.
|
||||
PLATSBANKEN_MAX_OFFSET = 2000
|
||||
|
||||
#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region`
|
||||
#: concepts (which also list regions outside Sweden). "" is all of Sweden and
|
||||
#: is not a key here.
|
||||
PLATSBANKEN_REGIONS: dict[str, str] = {
|
||||
"DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län",
|
||||
"K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län",
|
||||
"wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län",
|
||||
"MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län",
|
||||
"tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län",
|
||||
"CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län",
|
||||
"s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län",
|
||||
"EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län",
|
||||
"NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län",
|
||||
"zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län",
|
||||
"oLT3_Q9p_3nn": "Östergötlands län",
|
||||
}
|
||||
|
||||
|
||||
def parse_platsbanken(body: bytes) -> list[Row]:
|
||||
"""One page of JobTech's `/search`: `{"total": {"value"}, "hits": [...]}`.
|
||||
A hit's `webpage_url` is Platsbanken's own page for the ad. `remote` is
|
||||
read from the headline and from `workplace_model`, which the ads mostly
|
||||
leave empty."""
|
||||
return _platsbanken_rows(json.loads(body))
|
||||
|
||||
|
||||
def _platsbanken_rows(data: dict[str, Any]) -> list[Row]:
|
||||
rows = []
|
||||
for hit in data.get("hits") or []:
|
||||
title = _text(hit.get("headline") or "")
|
||||
url = (hit.get("webpage_url") or "").strip()
|
||||
if not title or not _web_url(url):
|
||||
continue
|
||||
address = hit.get("workplace_address") or {}
|
||||
model = (hit.get("workplace_model") or {}).get("label") or ""
|
||||
rows.append({
|
||||
"id": str(hit.get("id") or ""),
|
||||
"title": title,
|
||||
"company": ((hit.get("employer") or {}).get("name") or "").strip(),
|
||||
"place": (address.get("municipality") or address.get("region") or "").strip(),
|
||||
"country": "SE",
|
||||
"remote": _remote(f"{title} {model}"),
|
||||
"url": url,
|
||||
"posted": _parse_iso(hit.get("publication_date")),
|
||||
"description": _text((hit.get("description") or {}).get("text") or ""),
|
||||
"salary_min": None, "salary_max": None, "currency": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def search_platsbanken(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, newest first, until a short page, the stated
|
||||
total or the page limit. Returns the rows and whether they are all the
|
||||
`total` says match, as for Adzuna. `region` is a key of
|
||||
`PLATSBANKEN_REGIONS`, or empty for all of Sweden."""
|
||||
region = str(params.get("region") or "")
|
||||
if region and region not in PLATSBANKEN_REGIONS:
|
||||
raise ValueError(f"platsbanken region {region!r} is not one of the ids in "
|
||||
"PLATSBANKEN_REGIONS")
|
||||
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "pubdate-desc",
|
||||
"limit": PLATSBANKEN_PAGE}
|
||||
if region:
|
||||
query["region"] = region
|
||||
rows: list[Row] = []
|
||||
seen: set[str] = set()
|
||||
total: Any = None
|
||||
ran_out = False
|
||||
for page in range(_pages(params)):
|
||||
offset = page * PLATSBANKEN_PAGE
|
||||
if offset > PLATSBANKEN_MAX_OFFSET:
|
||||
break
|
||||
data = json.loads(_get(client, PLATSBANKEN_SEARCH, query | {"offset": offset}))
|
||||
hits = data.get("hits") or []
|
||||
total = (data.get("total") or {}).get("value", total)
|
||||
seen |= {str(hit.get("id") or "") for hit in hits}
|
||||
rows += _platsbanken_rows(data)
|
||||
if len(hits) < PLATSBANKEN_PAGE:
|
||||
ran_out = True
|
||||
break
|
||||
if isinstance(total, int) and len(seen) >= total:
|
||||
break
|
||||
return rows, (len(seen) == total) if isinstance(total, int) else ran_out
|
||||
|
||||
|
||||
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
|
||||
tuple[list[Row], bool]]] = {
|
||||
"jobindex": search_jobindex,
|
||||
"adzuna": search_adzuna,
|
||||
"platsbanken": search_platsbanken,
|
||||
}
|
||||
def _limits(params: Mapping[str, Any]) -> _runner.Limits:
|
||||
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
|
||||
|
||||
|
||||
def provider_of(target: str) -> str:
|
||||
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
|
||||
provider, sep, name = target.partition(":")
|
||||
return provider if sep and name.strip() and provider in _SEARCHES else ""
|
||||
return provider if sep and name.strip() and provider in readers.PROVIDERS else ""
|
||||
|
||||
|
||||
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
||||
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
|
||||
"""One search read through the runner: its provider and its listing.
|
||||
Raises ValueError for an unknown provider or a missing key, before any
|
||||
request."""
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
||||
f"{', '.join(readers.PROVIDERS)}")
|
||||
module = readers.PROVIDERS[provider]
|
||||
slug = target.partition(":")[2]
|
||||
ask = _runner.ask_for(module, params, keys=keys)
|
||||
get = _runner.getter(client, module.allowed(slug), provider)
|
||||
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
|
||||
|
||||
|
||||
def run_search(target: str, params: dict[str, Any],
|
||||
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
|
||||
"""One search, outside any pull: for `app searches add`, which runs a
|
||||
search once before saving it."""
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
||||
f"{', '.join(PROVIDERS)}")
|
||||
search once before saving it. `keys` is the config's `sources.search`.
|
||||
Each row also carries `place`, the display line of its places."""
|
||||
with httpx.Client() as client:
|
||||
return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {})
|
||||
_module, listing = _read(client, target, params, keys)
|
||||
rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows]
|
||||
return rows, listing.complete
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
|
||||
class SearchSource(Source):
|
||||
name = "search"
|
||||
description = "Saved searches on cross-company job boards: Jobindex, Adzuna, Platsbanken."
|
||||
description = "Saved searches on cross-company job boards: " + ", ".join(
|
||||
module.LABEL for module in readers.PROVIDERS.values()) + "."
|
||||
#: Entries are `{"url": target, "label": name, "params": {...}}`.
|
||||
subscription_field = "searches"
|
||||
|
||||
@@ -447,37 +135,32 @@ class SearchSource(Source):
|
||||
log.warning("search: %r is not provider:name; skipped", target)
|
||||
continue
|
||||
try:
|
||||
rows, complete = _SEARCHES[provider](
|
||||
client, self.params_for(target), self.options.get(provider) or {})
|
||||
_module, listing = _read(client, target, self.params_for(target),
|
||||
self.options)
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
continue
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
source_id = f"{provider}:{row['id']}"
|
||||
for row in listing.rows:
|
||||
source_id = f"{provider}:{row['source_id']}"
|
||||
seen_ids.append(source_id)
|
||||
description = row.get("description") or ""
|
||||
yield Item(
|
||||
source_id=source_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted"),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": row["company"],
|
||||
"place": row["place"],
|
||||
"country": row["country"],
|
||||
"remote": row["remote"],
|
||||
"salary_min": row["salary_min"],
|
||||
"salary_max": row["salary_max"],
|
||||
"currency": row["currency"],
|
||||
raw=_runner.raw_of(row) | {
|
||||
"company": row.get("company") or "",
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
self.state[f"search:ok:{target}"] = run_marker
|
||||
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
|
||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
User-agent: *
|
||||
Disallow: /applybutton/
|
||||
Disallow: /talentcommunity/
|
||||
Disallow: /mobile/talentcommunity/
|
||||
Disallow: /emailsubscribe/
|
||||
Disallow: /email/image/
|
||||
Disallow: /services/
|
||||
Disallow: /preapply/
|
||||
Disallow: /error
|
||||
Disallow: /unsubscribe/
|
||||
Disallow: /reset/
|
||||
+6
-5
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import smartrecruiters # noqa: E402
|
||||
from plugins.readers import _runner, smartrecruiters # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -487,7 +487,7 @@ def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
|
||||
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
clock = iter(range(0, 1000, 10))
|
||||
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
|
||||
monkeypatch.setattr(_runner.time, "monotonic", lambda: float(next(clock)))
|
||||
monkeypatch.setattr(ats, "DEADLINE", 25.0)
|
||||
body = (b"x" for _ in range(100))
|
||||
with _client(lambda request: httpx.Response(200, content=body)) as client, \
|
||||
@@ -752,12 +752,13 @@ def test_teamtailor_is_read_through_the_streamed_get_and_a_redirect_elsewhere_is
|
||||
|
||||
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
||||
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
rows, complete = answer # type: ignore[misc]
|
||||
return _runner.Listing(rows, complete, [row["source_id"] for row in rows])
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
|
||||
"run": "run-1"})
|
||||
return source
|
||||
|
||||
@@ -155,19 +155,6 @@ def test_an_empty_board_is_complete_and_a_401_is_a_failure(web: dict[str, Any])
|
||||
ats._fetch_rows(client, "hibob:liquidia")
|
||||
|
||||
|
||||
def test_the_robots_txt_that_is_the_apps_own_page_allows(web: dict[str, Any]) -> None:
|
||||
"""The host answers /robots.txt with its single-page app's HTML, a 200
|
||||
with no rules."""
|
||||
web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \
|
||||
b'<!DOCTYPE html><html><head><title>Careers</title></head><body></body></html>'
|
||||
with httpx.Client() as client:
|
||||
ats._check_robots(client, "hibob:dixa")
|
||||
web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \
|
||||
b"User-agent: *\nDisallow: /\n"
|
||||
with pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "hibob:dixa")
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
def test_the_module_is_registered_as_a_paged_host() -> None:
|
||||
|
||||
@@ -133,25 +133,6 @@ def test_a_429_after_the_first_page_ends_the_read_incomplete(web: dict[str, Any]
|
||||
assert len(rows) == 5 and not complete
|
||||
|
||||
|
||||
def test_robots_are_read_from_the_api_host_whose_404_allows(web: dict[str, Any]) -> None:
|
||||
request = httpx.Request("GET", "https://api.hr-manager.net/robots.txt")
|
||||
web["answers"]["https://api.hr-manager.net/robots.txt"] = httpx.HTTPStatusError(
|
||||
"404", request=request, response=httpx.Response(404, request=request))
|
||||
state: dict[str, str] = {}
|
||||
with httpx.Client() as client:
|
||||
ats._check_robots(client, "hrmanager:stsgroup_tr", state)
|
||||
assert '"allowed": true' in state["ats:robots:hrmanager:stsgroup_tr"]
|
||||
assert web["asked"] == ["https://api.hr-manager.net/robots.txt"]
|
||||
|
||||
|
||||
def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide(
|
||||
web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://api.hr-manager.net/robots.txt"] = \
|
||||
b"User-agent: *\nDisallow: /JobPortal.svc/\n"
|
||||
with httpx.Client() as client, pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "hrmanager:stsgroup_tr")
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
def test_the_module_is_registered_as_a_paged_host() -> None:
|
||||
|
||||
+49
-6
@@ -20,7 +20,9 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from app.models import Application # noqa: E402
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.jobs import cascade, pull # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
@@ -202,14 +204,16 @@ def _row(job_id: str) -> dict[str, object]:
|
||||
"description": "", "remote": None, "salary_text": ""}
|
||||
|
||||
|
||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
|
||||
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None:
|
||||
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
|
||||
with `filtered`, a read the board's filter narrowed."""
|
||||
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
rows, complete = answer # type: ignore[misc]
|
||||
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered)
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, object]:
|
||||
@@ -275,6 +279,45 @@ def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
|
||||
assert pull.failing_boards(acme) == {}
|
||||
|
||||
|
||||
def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
|
||||
_pull(acme)
|
||||
uid = {r.source_id: r.uid for r in acme.roles()}
|
||||
agent = acme.save_agent("Hunt")
|
||||
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
|
||||
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
|
||||
|
||||
_answer(monkeypatch, ([_row("1")], True), filtered=True)
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
||||
assert result["closed"] == 1
|
||||
assert acme.state["ats:filtered:workable:acme"] == "1"
|
||||
|
||||
# Read without the filter again, the board closes as before.
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
|
||||
|
||||
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
||||
board="workable", params={"countries": ["dk"], "remote": True})
|
||||
asked: list[object] = []
|
||||
|
||||
def fake(module: object, slug: str, get: object, ask: object, **kwargs: object
|
||||
) -> _runner.Listing:
|
||||
asked.append(ask)
|
||||
return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type]
|
||||
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
_pull(store)
|
||||
[ask] = asked
|
||||
declared = readers.HOSTS["workable"].READS.filters.declared()
|
||||
assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ------------------------------------------------------ one pull at a time
|
||||
|
||||
def _ago(**delta: float) -> str:
|
||||
|
||||
+11
-193
@@ -1,6 +1,6 @@
|
||||
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
||||
every host: the registry and its contract check, the fetch a host is handed,
|
||||
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
||||
the per-role detail reads, and Lever's EU instance. No network —
|
||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,7 +11,7 @@ import subprocess
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -23,6 +23,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins.readers import lever # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -242,11 +243,9 @@ def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatc
|
||||
|
||||
# ------------------------------------------------------- a fake host
|
||||
|
||||
def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||
robots: bool = False) -> types.ModuleType:
|
||||
def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
|
||||
"""A host `fake` whose feed is a JSON list of ids, each role with a
|
||||
teaser for a description, and, with `detail`, a per-role page; with
|
||||
`robots`, its listing is the page robots.txt must allow."""
|
||||
teaser for a description, and, with `detail`, a per-role page."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
@@ -260,8 +259,6 @@ def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||
for i in json.loads(body)]
|
||||
if detail is not None:
|
||||
module.detail = detail
|
||||
if robots:
|
||||
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
return module
|
||||
|
||||
|
||||
@@ -411,8 +408,8 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
ids stay, and the next pull does not read them in detail again."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
||||
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
|
||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
@@ -511,7 +508,7 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
|
||||
"title": "Another title"}
|
||||
|
||||
caplog.set_level("DEBUG", logger=ats.log.name)
|
||||
caplog.set_level("DEBUG", logger=_runner.log.name)
|
||||
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
||||
assert (item.body, item.title, item.published_at) == (
|
||||
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
||||
@@ -519,8 +516,9 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
|
||||
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
|
||||
assert "department, salary_text, title left out" in caplog.text
|
||||
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
|
||||
"salary_min", "salary_max", "posted"}
|
||||
assert set(ats.DETAIL_FIELDS) == {"description", "location", "places", "teaser", "remote",
|
||||
"country", "employment", "currency", "salary_min",
|
||||
"salary_max", "posted"}
|
||||
|
||||
|
||||
def _paged_host() -> types.ModuleType:
|
||||
@@ -583,186 +581,6 @@ def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
||||
assert "ats:detailed:fake:acme" not in source.state
|
||||
|
||||
|
||||
# ------------------------------------------------------------ robots.txt
|
||||
|
||||
ROBOTS = "https://jobs.fake.example/robots.txt"
|
||||
|
||||
|
||||
def test_robots_that_allow_the_page_let_the_board_be_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("robots", [
|
||||
b"User-agent: *\nDisallow: /acme/\n",
|
||||
# A rule for this app by name counts as well as the one for everyone.
|
||||
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
|
||||
.encode(),
|
||||
])
|
||||
def test_robots_that_disallow_the_page_fail_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
|
||||
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == []
|
||||
assert web["asked"] == [ROBOTS] # the listing is never fetched
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
host = _fake_host(robots=True)
|
||||
items, _state = _run(monkeypatch, host)
|
||||
assert len(items) == 1 # paths are case-sensitive by default
|
||||
host.ROBOTS_ANY_CASE = True
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, host)
|
||||
assert items == []
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
||||
# The longest matching rule wins, whatever the order of the lines.
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
||||
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
|
||||
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
|
||||
# An allow wins a tie.
|
||||
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
|
||||
# `*` inside a path, and the query counts as part of it.
|
||||
("User-agent: *\nDisallow: /*company\n",
|
||||
"https://career5.successfactors.eu/career?company=x", False),
|
||||
# `$` anchors the end.
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
|
||||
# An empty Disallow allows.
|
||||
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
|
||||
# A byte order mark and CRLF line ends.
|
||||
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
|
||||
# Paths are case-sensitive.
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
|
||||
# A group for this app by name counts, a group for someone else does
|
||||
# not, and a token matches whole, not as a substring.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
|
||||
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
# Two agents share one group, and a comment is no rule.
|
||||
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
|
||||
"https://x.example/jobs/1", False),
|
||||
# A group for this app that allows does not undo the `*` group's refusal.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
# Rules before any user-agent line belong to no group.
|
||||
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
])
|
||||
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
|
||||
assert ats._robots_allows(robots.encode(), page) is allowed
|
||||
|
||||
|
||||
def test_a_robots_verdict_is_kept_for_a_day(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
module = _fake_host(robots=True)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
assert web["asked"] == [ROBOTS]
|
||||
|
||||
# Within the day the kept verdict holds, even though the site changed.
|
||||
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert (items, web["asked"]) == ([], [])
|
||||
|
||||
# A day on, robots.txt is read again.
|
||||
kept = json.loads(state["ats:robots:fake:acme"])
|
||||
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
||||
state["ats:robots:fake:acme"] = json.dumps(kept)
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
|
||||
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [404, 410, 401, 403])
|
||||
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
|
||||
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [
|
||||
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
|
||||
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
|
||||
])
|
||||
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == [] and web["asked"] == [ROBOTS]
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
|
||||
"robots.txt could not be read"
|
||||
assert "ats:robots:fake:acme" not in state
|
||||
|
||||
|
||||
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
|
||||
where a robots.txt that disallows everything is served."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if str(request.url) == ROBOTS:
|
||||
return httpx.Response(301, headers={"location": location})
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
|
||||
return httpx.Response(200, text='["1"]')
|
||||
return handler
|
||||
|
||||
|
||||
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", [
|
||||
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
|
||||
"http://www.jobs.fake.example/robots.txt",
|
||||
])
|
||||
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
||||
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Refused):
|
||||
ats._get(client, ROBOTS, "fake", "acme")
|
||||
|
||||
|
||||
def test_fetch_board_honours_robots_txt(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with pytest.raises(ats._Disallowed) as caught:
|
||||
ats.fetch_board("fake:acme")
|
||||
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- Lever EU
|
||||
|
||||
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
||||
subscription resolves to, series and their union, whole and paged listings,
|
||||
completeness from the declaration, the sweep across pulls, the 10% guard,
|
||||
the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||
network: `get` is a dict of URL -> body or exception."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, Filters, Key, Reads, _runner # noqa: E402
|
||||
|
||||
NO_DELAY = _runner.Limits(page_delay=0)
|
||||
|
||||
|
||||
def _status(code: int) -> httpx.HTTPStatusError:
|
||||
request = httpx.Request("GET", "https://jobs.fake.example/")
|
||||
return httpx.HTTPStatusError(str(code), request=request,
|
||||
response=httpx.Response(code, request=request))
|
||||
|
||||
|
||||
class Web:
|
||||
"""A `get` answering from a dict of URL -> body or exception."""
|
||||
|
||||
def __init__(self, answers: dict[str, Any] | None = None) -> None:
|
||||
self.answers = answers or {}
|
||||
self.asked: list[str] = []
|
||||
|
||||
def __call__(self, url: str, **kwargs: Any) -> bytes:
|
||||
self.asked.append(url)
|
||||
answer = self.answers.get(url, _status(404))
|
||||
if isinstance(answer, BaseException):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
|
||||
def _row(i: str, **extra: Any) -> dict[str, Any]:
|
||||
return {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
|
||||
"places": [{"text": "Berlin", "remote": None}]} | extra
|
||||
|
||||
|
||||
def _whole(**reads: Any) -> Any:
|
||||
"""A declared whole-listing board: the feed is a JSON list of ids, one
|
||||
feed per `ask.countries` value."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
|
||||
"complete": "whole", "places": "many", "robots": "test"} | reads))
|
||||
module.feed_url = lambda slug, ask: (f"https://jobs.fake.example/{slug}/list"
|
||||
+ "".join(f"?c={c}" for c in ask.countries))
|
||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
|
||||
return module
|
||||
|
||||
|
||||
def _paged(pages: list[list[str]], total: int | None = None, **reads: Any) -> Any:
|
||||
"""A declared paged board over `pages`, the cursor being the page index."""
|
||||
module = _whole(**({"listing": "paged", "complete": "stated" if total is not None
|
||||
else "end"} | reads))
|
||||
del module.feed_url, module.parse
|
||||
module.asked = []
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
index = int(cursor or 0)
|
||||
module.asked.append((index, ask.countries))
|
||||
body = get(f"https://jobs.fake.example/{slug}/page/{index}")
|
||||
ids = json.loads(body) if body else pages[index]
|
||||
more = index + 1 < len(pages) and bool(ids)
|
||||
return ids, [{"id": i} for i in ids], total, index + 1 if more else None
|
||||
|
||||
module.page = page
|
||||
module.rows = lambda ids, slug: [_row(i) for i in ids]
|
||||
return module
|
||||
|
||||
|
||||
def _pages_web(pages: list[list[str]]) -> Web:
|
||||
return Web({f"https://jobs.fake.example/acme/page/{n}": json.dumps(ids).encode()
|
||||
for n, ids in enumerate(pages)})
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- ask
|
||||
|
||||
def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
|
||||
board = _whole(filters=Filters(country="one"))
|
||||
ask = _runner.ask_for(board, {"countries": ["dk", "se"], "remote": True, "places": ["X"]})
|
||||
assert (ask.countries, ask.remote, ask.places, ask.params) == (("DK", "SE"), None, (), {})
|
||||
# The params override the reach per dimension; the reach fills the rest.
|
||||
both = _whole(filters=Filters(country="many", remote="many"))
|
||||
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
|
||||
assert (ask.countries, ask.remote) == (("DE",), True)
|
||||
# A module without READS reads the whole board.
|
||||
assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
|
||||
|
||||
|
||||
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
module = _whole(scope="search", listing="paged", complete="stated", text="excerpt",
|
||||
filters=Filters(query="one", place="one"),
|
||||
key=Key(name="fake", fields=("id", "secret"), env=("FAKE_ID", "FAKE_SECRET")))
|
||||
module.PARAMS = {"q": "query", "area": "place"}
|
||||
monkeypatch.delenv("FAKE_ID", raising=False)
|
||||
monkeypatch.setenv("FAKE_SECRET", "s")
|
||||
ask = _runner.ask_for(module, {"q": "python", "area": 1, "pages": 2},
|
||||
keys={"fake": {"id": "i"}})
|
||||
assert (ask.query, ask.places) == ("python", ("1",))
|
||||
assert ask.params == {"q": "python", "area": 1, "pages": 2,
|
||||
"key": {"id": "i", "secret": "s"}}
|
||||
with pytest.raises(ValueError, match=r"sources\.search\.fake: id, secret, or FAKE_ID"):
|
||||
_runner.ask_for(module, {"q": "x"}, keys={})
|
||||
|
||||
|
||||
def test_series_are_one_a_value_for_one_and_one_for_many() -> None:
|
||||
ask = Ask(countries=("DK", "SE"), places=("Aarhus",))
|
||||
one = _whole(filters=Filters(country="one", place="many"))
|
||||
assert [(a.countries, a.places) for a in _runner.series(one, ask)] == [
|
||||
(("DK",), ("Aarhus",)), (("SE",), ("Aarhus",))]
|
||||
many = _whole(filters=Filters(country="many"))
|
||||
assert _runner.series(many, ask) == [ask]
|
||||
|
||||
|
||||
def test_ids_filters_are_mapped_through_the_facets() -> None:
|
||||
module = _whole(filters=Filters(country="ids"))
|
||||
module.facets = lambda get, slug: {}
|
||||
ask = Ask(countries=("DK", "XX"))
|
||||
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
|
||||
assert mapped.countries == ("h-1",)
|
||||
# None known: the host has nothing in reach, and the read is empty.
|
||||
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
|
||||
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
|
||||
facets={"country": {}})
|
||||
assert (listing.rows, listing.complete, listing.filtered) == ([], True, True)
|
||||
|
||||
|
||||
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
|
||||
module = _whole(filters=Filters(country="ids"))
|
||||
calls: list[str] = []
|
||||
module.facets = lambda get, slug: calls.append(slug) or {"country": {"DK": "7"}}
|
||||
state: dict[str, str] = {}
|
||||
assert _runner.facets_for(module, "acme", Web(), Ask(), state, "k") == {}
|
||||
for _ in range(2):
|
||||
found = _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k")
|
||||
assert found == {"country": {"DK": "7"}}
|
||||
assert calls == ["acme"]
|
||||
kept = json.loads(state["k"])
|
||||
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
||||
state["k"] = json.dumps(kept)
|
||||
_runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k")
|
||||
assert calls == ["acme", "acme"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- listing
|
||||
|
||||
def test_a_whole_listing_reads_one_feed_a_series_and_unions_by_id() -> None:
|
||||
module = _whole(filters=Filters(country="one"))
|
||||
web = Web({"https://jobs.fake.example/acme/list?c=DK": b'["1", "2"]',
|
||||
"https://jobs.fake.example/acme/list?c=SE": b'["2", "3"]'})
|
||||
listing = _runner.read_listing(module, "acme", web, Ask(countries=("DK", "SE")),
|
||||
limits=NO_DELAY)
|
||||
assert [r["source_id"] for r in listing.rows] == ["1", "2", "3"]
|
||||
assert (listing.complete, listing.filtered, listing.ids) == (True, True, ["1", "2", "3"])
|
||||
|
||||
|
||||
def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
|
||||
"""Greenhouse's shape: the feed with the text, and one without it."""
|
||||
module = _whole()
|
||||
module.fallback_url = lambda slug: f"https://jobs.fake.example/{slug}/small"
|
||||
web = Web({"https://jobs.fake.example/acme/list": _runner.TooLarge("over"),
|
||||
"https://jobs.fake.example/acme/small": b'["1"]'})
|
||||
listing = _runner.read_listing(module, "acme", web)
|
||||
assert ([r["source_id"] for r in listing.rows], listing.complete) == (["1"], True)
|
||||
# A reader without a fallback fails.
|
||||
del module.fallback_url
|
||||
with pytest.raises(_runner.TooLarge):
|
||||
_runner.read_listing(module, "acme", web)
|
||||
|
||||
|
||||
def test_a_module_without_reads_gets_the_older_signatures() -> None:
|
||||
module: Any = types.ModuleType("old")
|
||||
module.NAME = "old"
|
||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
|
||||
listing = _runner.read_listing(module, "acme",
|
||||
Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
|
||||
Ask(countries=("DK",)))
|
||||
assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
|
||||
|
||||
|
||||
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
|
||||
pages = [["1", "2"], ["3"]]
|
||||
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
|
||||
limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2", "3"], True)
|
||||
short = _runner.read_listing(_paged(pages, total=4), "acme", _pages_web(pages),
|
||||
limits=NO_DELAY)
|
||||
assert not short.complete
|
||||
|
||||
|
||||
def test_the_page_cap_leaves_a_board_incomplete() -> None:
|
||||
pages = [["1"], ["2"], ["3"]]
|
||||
listing = _runner.read_listing(_paged(pages), "acme", _pages_web(pages),
|
||||
limits=_runner.Limits(max_pages=2, page_delay=0))
|
||||
assert (listing.ids, listing.complete, listing.sweep) == (["1", "2"], False, None)
|
||||
|
||||
|
||||
def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
|
||||
"""A host that ignores the cursor answers page one again; without a total
|
||||
the read would never end, so a page with nothing new stops it."""
|
||||
pages = [["1", "2"], ["1", "2"], ["3"]]
|
||||
module = _paged(pages)
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], False)
|
||||
assert [index for index, _ in module.asked] == [0, 1]
|
||||
|
||||
|
||||
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
|
||||
pages = [["1"], ["2"]]
|
||||
web = _pages_web(pages)
|
||||
web.answers["https://jobs.fake.example/acme/page/1"] = _status(429)
|
||||
listing = _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1"], False)
|
||||
web.answers["https://jobs.fake.example/acme/page/0"] = _status(429)
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
_runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY)
|
||||
|
||||
|
||||
def test_a_search_is_never_complete_when_it_says_so() -> None:
|
||||
pages = [["1"]]
|
||||
module = _paged(pages, scope="search", complete="never", text="excerpt")
|
||||
assert not _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY).complete
|
||||
|
||||
|
||||
def test_paged_series_each_read_to_the_end_and_union() -> None:
|
||||
pages = [["1", "2"]]
|
||||
module = _paged(pages, filters=Filters(country="one"))
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages),
|
||||
Ask(countries=("DK", "SE")), limits=NO_DELAY)
|
||||
assert module.asked == [(0, ("DK",)), (0, ("SE",))]
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ sweep
|
||||
|
||||
def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
|
||||
pages = [["1"], ["2"], ["3"], ["4"], ["5"]]
|
||||
module = _paged(pages, total=5)
|
||||
web, state = _pages_web(pages), {}
|
||||
limits = _runner.Limits(max_pages=2, page_delay=0)
|
||||
seen: list[tuple[list[str], bool]] = []
|
||||
for _pull in range(3):
|
||||
listing = _runner.read(state, "ats", "fake:acme", module, "acme", web, limits=limits)
|
||||
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||
seen.append((listing.ids, listing.complete))
|
||||
assert seen == [(["1", "2"], False), (["3", "4"], False), (["1", "2", "3", "4", "5"], True)]
|
||||
assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4]
|
||||
assert "ats:sweep:fake:acme" not in state
|
||||
assert json.loads(state["ats:count:fake:acme"])["count"] == 5
|
||||
|
||||
|
||||
def test_a_sweep_starts_again_when_the_ask_changed() -> None:
|
||||
pages = [["1"], ["2"], ["3"]]
|
||||
module = _paged(pages, filters=Filters(country="many"))
|
||||
web, state = _pages_web(pages), {}
|
||||
limits = _runner.Limits(max_pages=2, page_delay=0)
|
||||
first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits)
|
||||
_runner.stamp(state, "ats", "t", "r", first, first.ids)
|
||||
module.asked.clear()
|
||||
_runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), limits=limits)
|
||||
assert module.asked[0][0] == 0
|
||||
|
||||
|
||||
def test_a_board_at_its_ceiling_is_not_swept() -> None:
|
||||
pages = [["1", "2"], ["3", "4"], ["5"]]
|
||||
module = _paged(pages, total=5, page_size=2, ceiling=4)
|
||||
listing = _runner.read({}, "ats", "t", module, "acme", _pages_web(pages),
|
||||
limits=_runner.Limits(max_pages=2, page_delay=0))
|
||||
assert (listing.complete, listing.sweep) == (False, None)
|
||||
|
||||
|
||||
def test_a_search_is_not_swept() -> None:
|
||||
pages = [["1"], ["2"]]
|
||||
module = _paged(pages, total=2)
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), can_sweep=False,
|
||||
limits=_runner.Limits(max_pages=1, page_delay=0))
|
||||
assert listing.sweep is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------- the 10% guard
|
||||
|
||||
def test_a_suspect_read_far_under_the_last_count_fails() -> None:
|
||||
module = _whole(empty="suspect")
|
||||
web = Web({"https://jobs.fake.example/acme/list": json.dumps(
|
||||
[str(n) for n in range(20)]).encode()})
|
||||
state: dict[str, str] = {}
|
||||
listing = _runner.read(state, "ats", "t", module, "acme", web)
|
||||
_runner.stamp(state, "ats", "t", "r", listing, listing.ids)
|
||||
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
|
||||
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
|
||||
_runner.read(state, "ats", "t", module, "acme", web)
|
||||
# A reader whose empty answer is trusted reads it as it is.
|
||||
module.READS = module.READS.model_copy(update={"empty": "trusted"})
|
||||
assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"]
|
||||
|
||||
|
||||
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
||||
"""A board read whole last time and with a filter now lists far fewer
|
||||
roles; that is the filter, not a failure."""
|
||||
module = _whole(filters=Filters(country="one"))
|
||||
web = Web({"https://jobs.fake.example/acme/list": json.dumps(
|
||||
[str(n) for n in range(50)]).encode(),
|
||||
"https://jobs.fake.example/acme/list?c=DK": b'["1"]'})
|
||||
state: dict[str, str] = {}
|
||||
listing = _runner.read(state, "ats", "t", module, "acme", web)
|
||||
_runner.stamp(state, "ats", "t", "r", listing, listing.ids)
|
||||
filtered = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)))
|
||||
assert filtered.ids == ["1"] and filtered.filtered
|
||||
|
||||
|
||||
# ------------------------------------------------------ through ATSSource
|
||||
|
||||
@pytest.fixture
|
||||
def board(monkeypatch: pytest.MonkeyPatch) -> Any:
|
||||
pages = [["1"], ["2"], ["3"]]
|
||||
module = _paged(pages, total=3, empty="suspect")
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
||||
web = _pages_web(pages)
|
||||
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: web(url))
|
||||
module.web = web
|
||||
return module
|
||||
|
||||
|
||||
def _source(state: dict[str, str], params: dict[str, Any] | None = None) -> ats.ATSSource:
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
|
||||
"params": params or {}}], "run": "r"})
|
||||
source.state = state
|
||||
return source
|
||||
|
||||
|
||||
def test_ats_source_sweeps_and_stamps_every_id_once_the_sweep_ends(board: Any) -> None:
|
||||
state: dict[str, str] = {}
|
||||
items = list(_source(state).fetch())
|
||||
assert [i.source_id for i in items] == ["fake:acme:1", "fake:acme:2"]
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == 2
|
||||
|
||||
items = list(_source(state).fetch())
|
||||
assert [i.source_id for i in items] == ["fake:acme:3"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
assert json.loads(state["ats:ids:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
|
||||
|
||||
def test_ats_source_records_a_suspect_read_as_failing(board: Any) -> None:
|
||||
state = {"ats:count:fake:acme": json.dumps(
|
||||
{"asks": _runner.fingerprint([Ask()]), "count": 100})}
|
||||
board.web.answers = {"https://jobs.fake.example/acme/page/0": b"[]"}
|
||||
assert list(_source(state).fetch()) == []
|
||||
assert "under 10%" in json.loads(state["ats:failing:fake:acme"])["reason"]
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
|
||||
board.READS = board.READS.model_copy(update={"filters": Filters(country="many")})
|
||||
state: dict[str, str] = {}
|
||||
list(_source(state, {"countries": ["dk"]}).fetch())
|
||||
assert board.asked[0] == (0, ("DK",))
|
||||
assert state["ats:filtered:fake:acme"] == "1"
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- raw
|
||||
|
||||
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
||||
row = _row("1", places=[{"text": "Berlin", "remote": False},
|
||||
{"text": "Remote (DE)", "remote": True}],
|
||||
teaser="Short.", seniority="senior", employment="full_time", country="DE",
|
||||
description="Long.", posted=None, location="Berlin; Remote (DE)")
|
||||
raw = _runner.raw_of(row)
|
||||
assert raw["places"] == row["places"]
|
||||
assert raw["place"] == "Berlin; Remote (DE)"
|
||||
assert (raw["teaser"], raw["seniority"], raw["employment"], raw["country"]) == (
|
||||
"Short.", "senior", "full_time", "DE")
|
||||
assert raw["location"] == "Berlin; Remote (DE)"
|
||||
assert not {"description", "title", "url", "posted", "source_id"} & set(raw)
|
||||
assert (raw["department"], raw["salary_text"], raw["currency"], raw["salary_min"]) == (
|
||||
"", "", "", None)
|
||||
|
||||
|
||||
def test_a_row_without_places_has_its_location_as_the_one_place() -> None:
|
||||
raw = _runner.raw_of({"source_id": "1", "title": "t", "url": "u",
|
||||
"location": "Berlin, Munich"})
|
||||
assert raw["places"] == [{"text": "Berlin, Munich", "remote": None}]
|
||||
assert raw["place"] == "Berlin, Munich"
|
||||
assert _runner.raw_of({"source_id": "1", "title": "t", "url": "u"})["places"] == []
|
||||
|
||||
|
||||
def test_a_detail_location_without_places_becomes_the_one_place() -> None:
|
||||
module = _whole(text="detail")
|
||||
module.detail = lambda get, slug, row: {"location": "Aarhus", "employment": "contract",
|
||||
"description": "Full."}
|
||||
rows, done, failed = _runner.read_details(module, "fake", "acme", Web(), [_row("1")],
|
||||
set(), set(), limits=NO_DELAY)
|
||||
assert rows[0]["places"] == [{"text": "Aarhus", "remote": None}]
|
||||
assert (rows[0]["employment"], rows[0]["description"]) == ("contract", "Full.")
|
||||
assert (done, failed) == ({"fake:acme:1"}, set())
|
||||
+160
-254
@@ -1,15 +1,17 @@
|
||||
"""The `search` plugin, `country_of`, and what the pull and the CLI do with
|
||||
searches. No network: Jobindex's feed is a trimmed copy of a real one
|
||||
(2026-09-23), Adzuna's is written from its documented response, Platsbanken's
|
||||
is trimmed from a live answer (2026-09-29)."""
|
||||
"""The `search` plugin over the runner and the provider readers, `country_of`,
|
||||
and what the pull does with searches. No network: `_runner.fetch` is a stub,
|
||||
and a fake provider stands in where the pull is what is tested. The readers'
|
||||
own parsing is tested in `test_readers_<provider>.py`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -18,7 +20,8 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from plugins import search # noqa: E402
|
||||
from plugins import readers, search # noqa: E402
|
||||
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
||||
from plugins.search import SearchSource # noqa: E402
|
||||
from app.config import SearchSeed # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
@@ -111,246 +114,129 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) ->
|
||||
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- parsers
|
||||
# ----------------------------------------------------- read through the runner
|
||||
|
||||
def test_jobindex_reads_title_company_town_and_date() -> None:
|
||||
rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes())
|
||||
assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"]
|
||||
paid, bare, robot = rows
|
||||
# A paid ad: the company is its logo's alt text, the title its <h4>.
|
||||
assert paid["company"] == "DSV"
|
||||
assert paid["title"].startswith("Dangerous Goods Compliance Manager")
|
||||
assert paid["place"] == "Hedehusene"
|
||||
assert paid["country"] == "DK"
|
||||
assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23"
|
||||
assert paid["description"].startswith("Do you want to minimise")
|
||||
# A crawled ad with no company after the title's comma has none.
|
||||
assert bare["company"] == ""
|
||||
assert bare["place"] == "København"
|
||||
# A crawled ad names its company after the title.
|
||||
assert robot["title"] == "Marketing Content Creator"
|
||||
assert robot["company"] == "Clerk.IO ApS"
|
||||
def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]:
|
||||
"""`_runner.fetch` answering `answer(url)`, remembering the URLs asked."""
|
||||
asked: list[str] = []
|
||||
|
||||
def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes:
|
||||
assert allow(urlsplit(url).hostname), url
|
||||
asked.append(url)
|
||||
return answer(url)
|
||||
|
||||
monkeypatch.setattr(_runner, "fetch", fake_fetch)
|
||||
monkeypatch.setattr(search, "PAGE_DELAY", 0)
|
||||
return asked
|
||||
|
||||
|
||||
def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None:
|
||||
assert search._remote("Birkerød, Hybrid position") == "hybrid"
|
||||
assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
|
||||
assert search._remote("Remote, Danmark") == "remote"
|
||||
assert search._remote("Hedehusene") is None
|
||||
def _jobindex_page(url: str) -> bytes:
|
||||
"""A full Jobindex page (20 items) whatever page is asked."""
|
||||
page = int(parse_qs(urlsplit(url).query)["page"][0])
|
||||
items = "".join(f"<item><title>Role {page}-{n}, Acme</title>"
|
||||
f"<link>https://www.jobindex.dk/vis-job/r{page}x{n}</link>"
|
||||
f"<description><p>Teaser {n}</p></description></item>"
|
||||
for n in range(20))
|
||||
return f"<rss><channel><title>Jobindex</title>{items}</channel></rss>".encode()
|
||||
|
||||
|
||||
def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None:
|
||||
rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de")
|
||||
assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused
|
||||
first, predicted = rows
|
||||
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
||||
assert first["description"] == "Build our backend services in Python & Go."
|
||||
assert first["company"] == "Acme GmbH"
|
||||
assert first["place"] == "Berlin, Deutschland"
|
||||
assert first["country"] == "DE"
|
||||
assert first["remote"] == "remote"
|
||||
assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR")
|
||||
assert (predicted["salary_min"], predicted["currency"]) == (None, "")
|
||||
@pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)])
|
||||
def test_the_saved_pages_param_bounds_the_pages_read(
|
||||
monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None:
|
||||
urls = _web(monkeypatch, _jobindex_page)
|
||||
params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else
|
||||
{"pages": pages})
|
||||
rows, complete = search.run_search("jobindex:cph", params)
|
||||
assert len(urls) == asked and len(rows) == 20 * asked
|
||||
assert not complete # Jobindex states no total: never complete
|
||||
query = parse_qs(urlsplit(urls[0]).query)
|
||||
assert (query["q"], query["geoareaid"]) == (["python"], ["1"])
|
||||
|
||||
|
||||
def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
||||
with pytest.raises(ValueError, match="app_id"):
|
||||
search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
pages: list[int] = []
|
||||
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
pages.append(params["page"])
|
||||
return b"short" if params["page"] == 3 else b"full"
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setattr(search, "_jobindex_page",
|
||||
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
|
||||
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
|
||||
|
||||
# Stopped by the page limit with every page full: not complete.
|
||||
pages.clear()
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert (pages, complete) == ([1, 2], False)
|
||||
|
||||
|
||||
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
|
||||
def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A full page whose parser dropped one item still has 20 items, so the
|
||||
search reads on instead of calling itself complete at 19 rows."""
|
||||
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
||||
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
|
||||
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
|
||||
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert not complete
|
||||
_web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes())
|
||||
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
|
||||
"params": {"q": "x", "pages": 1}}]})
|
||||
items = list(source.fetch())
|
||||
paid = items[0]
|
||||
assert paid.source_id == "jobindex:h1701165"
|
||||
assert (paid.body, paid.excerpt) == ("", "")
|
||||
assert paid.raw["teaser"].startswith("Do you want to minimise")
|
||||
assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK"
|
||||
assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}]
|
||||
assert paid.raw["place"] == "Hedehusene"
|
||||
assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph")
|
||||
assert source.state["search:complete:jobindex:cph"] == ""
|
||||
|
||||
|
||||
def test_a_jobindex_answer_without_a_channel_raises() -> None:
|
||||
with pytest.raises(ValueError, match="channel"):
|
||||
search.parse_jobindex(b"<rss version='2.0'></rss>")
|
||||
|
||||
|
||||
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
|
||||
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
|
||||
asked: list[int] = []
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
page = int(url.rsplit("/", 1)[1])
|
||||
asked.append(page)
|
||||
start = (page - 1) * search.ADZUNA_PAGE
|
||||
def _adzuna(count: int | None, stock: int = 60) -> Any:
|
||||
def answer(url: str) -> bytes:
|
||||
page = int(urlsplit(url).path.rsplit("/", 1)[1])
|
||||
start = (page - 1) * 50
|
||||
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
|
||||
for n in range(start, min(start + 50, stock))]
|
||||
body: dict[str, Any] = {"results": results}
|
||||
if count is not None:
|
||||
body["count"] = count
|
||||
return json.dumps(body).encode()
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setenv("ADZUNA_APP_ID", "id")
|
||||
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
|
||||
return asked
|
||||
return answer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
|
||||
def test_a_search_without_its_key_is_refused_before_any_request(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
||||
urls = _web(monkeypatch, _adzuna(60))
|
||||
with pytest.raises(ValueError, match="app_id"):
|
||||
search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {})
|
||||
assert urls == []
|
||||
|
||||
|
||||
def test_the_key_comes_from_the_config_before_the_environment(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("ADZUNA_APP_ID", "env-id")
|
||||
monkeypatch.setenv("ADZUNA_APP_KEY", "env-key")
|
||||
urls = _web(monkeypatch, _adzuna(60))
|
||||
search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
|
||||
{"adzuna": {"app_id": "cfg-id"}})
|
||||
query = parse_qs(urlsplit(urls[0]).query)
|
||||
assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"])
|
||||
assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)])
|
||||
def test_an_adzuna_search_is_complete_when_it_read_its_count(
|
||||
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
|
||||
_adzuna(monkeypatch, count)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None:
|
||||
_web(monkeypatch, _adzuna(count))
|
||||
rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
|
||||
{"adzuna": {"app_id": "i", "app_key": "k"}})
|
||||
assert (len(rows), done) == (60, complete)
|
||||
assert rows[0]["place"] == ""
|
||||
|
||||
|
||||
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
|
||||
|
||||
|
||||
#: Two hits of a live `/search?q=data science®ion=CaRE_1nn_cSU` (2026-09-29),
|
||||
#: trimmed to the fields the parser reads, and a third the way an ad with no
|
||||
#: municipality and a remote headline comes.
|
||||
PLATSBANKEN = {
|
||||
"total": {"value": 32},
|
||||
"hits": [
|
||||
{"id": "31457942", "headline": "Postdoctoral Researcher",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942",
|
||||
"publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59",
|
||||
"description": {"text": "The European Spallation Source (ESS) is one of the largest.",
|
||||
"text_formatted": "<p>The European Spallation Source (ESS)</p>"},
|
||||
"employer": {"name": "European Spallation Source Eric"},
|
||||
"application_details": {"url": "https://europeanspallationsource.se/careers"},
|
||||
"workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"},
|
||||
"workplace_address": {"municipality": "Lund", "region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
|
||||
{"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504",
|
||||
"publication_date": "2026-09-03T15:46:06",
|
||||
"description": {"text": "Robotics at ESS."},
|
||||
"employer": {"name": "European Spallation Source Eric"},
|
||||
"workplace_model": {"concept_id": None, "label": None},
|
||||
"workplace_address": {"municipality": None, "region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
|
||||
{"id": "31400001", "headline": "Data scientist, distansarbete möjligt",
|
||||
"webpage_url": "javascript:alert(1)", "workplace_address": {}},
|
||||
{"id": "31400002", "headline": "Data scientist, distansarbete möjligt",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002",
|
||||
"employer": {"name": None}, "workplace_address": {}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_platsbanken_reads_title_company_place_and_date() -> None:
|
||||
rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode())
|
||||
assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
||||
first, no_town, remote = rows
|
||||
assert first["title"] == "Postdoctoral Researcher"
|
||||
assert first["company"] == "European Spallation Source Eric"
|
||||
assert first["place"] == "Lund"
|
||||
assert first["country"] == "SE"
|
||||
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
|
||||
assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09"
|
||||
assert first["description"].startswith("The European Spallation Source")
|
||||
assert first["remote"] is None
|
||||
assert no_town["place"] == "Skåne län" # the region when there is no municipality
|
||||
assert remote["remote"] == "remote"
|
||||
assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None)
|
||||
|
||||
|
||||
def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None,
|
||||
stock: int = 250) -> list[dict[str, Any]]:
|
||||
"""Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request,
|
||||
stating `total`. Returns the params of each request."""
|
||||
asked: list[dict[str, Any]] = []
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
asked.append(params)
|
||||
start = params["offset"]
|
||||
hits = [{"id": str(n), "headline": f"Role {n}",
|
||||
"webpage_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + params["limit"], stock))]
|
||||
body: dict[str, Any] = {"hits": hits}
|
||||
if total is not None:
|
||||
body["total"] = {"value": total}
|
||||
return json.dumps(body).encode()
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
return asked
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("total", "pages", "found", "complete"), [
|
||||
(250, 5, 250, True), # a short third page ends it
|
||||
(300, 5, 250, False), # the stated total is more than there was
|
||||
(250, 2, 200, False), # stopped by the page limit
|
||||
(None, 5, 250, True), # no total: a short page is the end
|
||||
])
|
||||
def test_a_platsbanken_search_is_complete_when_it_read_its_total(
|
||||
monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int,
|
||||
complete: bool) -> None:
|
||||
_platsbanken(monkeypatch, total)
|
||||
rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type]
|
||||
assert (len(rows), done) == (found, complete)
|
||||
|
||||
|
||||
def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset(
|
||||
def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 200, stock=200)
|
||||
rows, done = search.search_platsbanken(
|
||||
None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type]
|
||||
def answer(url: str) -> bytes:
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
start = int(query["offset"][0])
|
||||
hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}",
|
||||
"workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]}
|
||||
for n in range(start, min(start + 100, 200))]
|
||||
return json.dumps({"total": {"value": 200}, "hits": hits}).encode()
|
||||
|
||||
urls = _web(monkeypatch, answer)
|
||||
rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"})
|
||||
assert (len(rows), done) == (200, True)
|
||||
assert [p["offset"] for p in asked] == [0, 100]
|
||||
assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU"
|
||||
assert asked[0]["limit"] == search.PLATSBANKEN_PAGE
|
||||
offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls]
|
||||
assert offsets == [["0"], ["100"]]
|
||||
assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"]
|
||||
assert rows[0]["place"] == "Lund; Malmö"
|
||||
|
||||
|
||||
def test_a_platsbanken_search_without_a_region_sends_none(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 5, stock=5)
|
||||
search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type]
|
||||
assert "region" not in asked[0]
|
||||
|
||||
|
||||
def test_a_platsbanken_search_never_asks_past_the_offset_limit(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 5000, stock=5000)
|
||||
monkeypatch.setattr(search, "MAX_PAGES", 30)
|
||||
_rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type]
|
||||
assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET
|
||||
assert not done
|
||||
|
||||
|
||||
def test_a_platsbanken_search_refuses_an_unknown_region() -> None:
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_platsbanken_regions_are_the_21_lan() -> None:
|
||||
assert len(search.PLATSBANKEN_REGIONS) == 21
|
||||
assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län"
|
||||
def test_an_unknown_provider_is_refused() -> None:
|
||||
with pytest.raises(ValueError, match="provider"):
|
||||
search.run_search("indeed:x", {"q": "x"})
|
||||
|
||||
|
||||
def test_provider_of() -> None:
|
||||
@@ -369,15 +255,27 @@ class FakeConfig:
|
||||
path = None
|
||||
|
||||
|
||||
def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
|
||||
def fake(client: object, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
|
||||
return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København",
|
||||
"country": "DK", "remote": None, "url": f"https://x.example/{i}",
|
||||
"posted": None, "description": "", "salary_min": None,
|
||||
"salary_max": None, "currency": ""} for i in ids], complete
|
||||
def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
|
||||
"""A provider `fake` that lists `ids` in one page and states a total of
|
||||
as many, or of one more when the search is to be incomplete."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL = "fake", "Fake"
|
||||
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
|
||||
places="one", fields=Fields(country="stated"), robots="test")
|
||||
module.allowed = lambda slug: lambda host: host == "search.fake.example"
|
||||
|
||||
monkeypatch.setitem(search._SEARCHES, "jobindex", fake)
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
assert ask.params["q"]
|
||||
return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None
|
||||
|
||||
module.page = page
|
||||
module.rows = lambda decoded, slug: [
|
||||
{"source_id": i, "title": f"Engineer {i}", "company": "Acme",
|
||||
"url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}],
|
||||
"country": "DK", "remote": None, "posted": None, "description": ""}
|
||||
for i in decoded]
|
||||
assert not readers.problems(module, "fake")
|
||||
monkeypatch.setitem(readers.PROVIDERS, "fake", module)
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, Any]:
|
||||
@@ -388,36 +286,36 @@ def _pull(store: Store) -> dict[str, Any]:
|
||||
|
||||
def test_a_complete_search_closes_what_it_stopped_listing(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
|
||||
params={"q": "backend", "area": 1})
|
||||
_stub_search(monkeypatch, ["h1", "h2"], complete=True)
|
||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||
params={"q": "backend"})
|
||||
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
||||
_pull(store)
|
||||
roles = {r.source_id: r for r in store.roles()}
|
||||
assert set(roles) == {"jobindex:h1", "jobindex:h2"}
|
||||
assert roles["jobindex:h1"].country == "DK"
|
||||
assert roles["jobindex:h1"].subscription_id is not None
|
||||
assert set(roles) == {"fake:h1", "fake:h2"}
|
||||
assert roles["fake:h1"].country == "DK"
|
||||
assert roles["fake:h1"].subscription_id is not None
|
||||
|
||||
_stub_search(monkeypatch, ["h2"], complete=True)
|
||||
_fake_provider(monkeypatch, ["h2"], complete=True)
|
||||
result = _pull(store)
|
||||
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
||||
assert (result["searches"], result["closed"]) == (1, 1)
|
||||
|
||||
|
||||
def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale(
|
||||
def test_an_incomplete_search_closes_nothing_until_stale(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
|
||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||
params={"q": "backend"})
|
||||
_stub_search(monkeypatch, ["h1", "h2"], complete=False)
|
||||
_fake_provider(monkeypatch, ["h1", "h2"], complete=False)
|
||||
_pull(store)
|
||||
_stub_search(monkeypatch, ["h2"], complete=False)
|
||||
_fake_provider(monkeypatch, ["h2"], complete=False)
|
||||
_pull(store)
|
||||
assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"}
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
|
||||
|
||||
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,))
|
||||
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
|
||||
_pull(store)
|
||||
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
||||
|
||||
|
||||
def test_seed_searches_imports_once(store: Store) -> None:
|
||||
@@ -434,24 +332,32 @@ def test_seed_searches_imports_once(store: Store) -> None:
|
||||
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- cli
|
||||
|
||||
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_stub_search(monkeypatch, ["h1"], complete=True)
|
||||
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
|
||||
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
|
||||
"params": {"q": "x"}}]})
|
||||
items = list(source.fetch())
|
||||
assert [i.source_id for i in items] == ["jobindex:h1"]
|
||||
assert items[0].raw["search"] == "jobindex:cph"
|
||||
assert source.state["search:complete:jobindex:cph"] == "1"
|
||||
assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'
|
||||
assert [i.source_id for i in items] == ["fake:h1"]
|
||||
assert items[0].raw["search"] == "fake:cph"
|
||||
assert source.state["search:complete:fake:cph"] == "1"
|
||||
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
||||
|
||||
|
||||
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
||||
source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin",
|
||||
"params": {"q": "x", "country": "de"}}]})
|
||||
assert list(source.fetch()) == []
|
||||
assert "search:ok:adzuna:berlin" not in source.state
|
||||
|
||||
|
||||
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen",
|
||||
board="jobindex", params={"q": "backend"})
|
||||
_stub_search(monkeypatch, ["h1"], complete=False)
|
||||
sub = store.add_subscription("search", "fake:cph", label="Copenhagen",
|
||||
board="fake", params={"q": "backend"})
|
||||
_fake_provider(monkeypatch, ["h1"], complete=False)
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
||||
store.update_subscription(sub.id, subscribed=False, fetch=False)
|
||||
|
||||
Reference in new issue
Block a user