Merge text for search readers with detail (wp-src-searchdetail)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
04f4c09a21
5 files changed
+346
-40
No files matched your search
+51
-31
@@ -2,18 +2,22 @@
|
||||
|
||||
Some hosts list a role without its text and state it on the role's own page
|
||||
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
|
||||
with a `detail`). The pull reads listings only; this stage reads those pages
|
||||
after scoring, so the calls go where the ranking is:
|
||||
with a `detail`, and a search provider such as Jobbank). The pull reads
|
||||
listings only; this stage reads those pages after scoring, so the calls go
|
||||
where the ranking is. A saved search whose reader has `detail` is treated as
|
||||
a board under its target `provider:name`: the same bar, budgets and state
|
||||
keys, and its failing record is `search:failing:<target>`.
|
||||
|
||||
1. `enrich_waiting` hands over the open roles without text that an enabled
|
||||
agent with a prompt ranks ungated among its best `enrich.top` and at
|
||||
`enrich.min_score` or above, best first by the highest such score. A role
|
||||
below that bar for every agent gets no call.
|
||||
2. They are taken in that order, those whose read failed last, on boards whose reader has `detail`, at most
|
||||
`enrich.per_board` from one board and `enrich.per_run` in all, leaving
|
||||
out the roles already read, unless their read failed or gave no text
|
||||
`enrich.retry_days` ago (`plugins.ats.detail_due`).
|
||||
3. One batch per board, on the `http` pool, reads them
|
||||
2. They are taken in that order, those whose read failed last, on boards and
|
||||
searches whose reader has `detail`, at most `enrich.per_board` from one
|
||||
board and `enrich.per_run` in all, leaving out the roles already read,
|
||||
unless their read failed or gave no text `enrich.retry_days` ago
|
||||
(`plugins.ats.detail_due`).
|
||||
3. One batch per board or search, on the `http` pool, reads them
|
||||
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
|
||||
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
|
||||
what each page filled. `Store.upsert_role` drops the vector of a role
|
||||
@@ -37,6 +41,7 @@ import httpx
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs import stages
|
||||
from app.models import Subscription
|
||||
from app.storage import Store
|
||||
from pipeline import NULL, Batch, RunContext, Tally, Task, job
|
||||
|
||||
@@ -53,7 +58,14 @@ def _ats() -> Any:
|
||||
return ats
|
||||
|
||||
|
||||
def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
|
||||
def _native(found: dict[str, Any], sub: Subscription) -> str:
|
||||
"""The host's own id of a stored role: a board's roles are stored as
|
||||
`host:slug:id`, a search's as `provider:id`."""
|
||||
prefix = sub.target if sub.source == "ats" else sub.target.partition(":")[0]
|
||||
return str(found["source_id"]).removeprefix(f"{prefix}:")
|
||||
|
||||
|
||||
def _row(found: dict[str, Any], sub: Subscription) -> dict[str, Any]:
|
||||
"""A stored role back as the row its reader listed: the stored `raw`
|
||||
with the host's own id, the title, the link and the date."""
|
||||
try:
|
||||
@@ -65,29 +77,36 @@ def _row(found: dict[str, Any], target: str) -> dict[str, Any]:
|
||||
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
|
||||
except ValueError:
|
||||
posted = None
|
||||
return raw | {"source_id": str(found["source_id"]).removeprefix(f"{target}:"),
|
||||
return raw | {"source_id": _native(found, sub),
|
||||
"title": found["title"], "url": found["url"], "description": "",
|
||||
"posted": posted}
|
||||
|
||||
|
||||
def _readable(store: Store) -> dict[int, Subscription]:
|
||||
"""The fetchable boards and searches whose reader has `detail`, by
|
||||
subscription id."""
|
||||
ats = _ats()
|
||||
return {sub.id: sub for source in ("ats", "search")
|
||||
for sub in store.subscriptions(source=source, fetchable=True)
|
||||
if hasattr(ats.reader_of(sub.target), "detail")}
|
||||
|
||||
|
||||
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
|
||||
"""`(target, uids)` per board, best board first, within the budgets: the
|
||||
roles `enrich_waiting` ranks, on fetchable boards whose reader has
|
||||
`detail` and that are not failing, and due for a read."""
|
||||
"""`(target, uids)` per board or search, best first, within the budgets:
|
||||
the roles `enrich_waiting` ranks, on fetchable boards and searches whose
|
||||
reader has `detail` and that are not failing, and due for a read."""
|
||||
settings = config.enrich
|
||||
ats = _ats()
|
||||
boards = {sub.id: sub.target for sub in store.subscriptions(source="ats", fetchable=True)}
|
||||
subs = _readable(store)
|
||||
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for found in store.enrich_waiting(settings.top, settings.min_score):
|
||||
target = boards.get(found["subscription_id"])
|
||||
if target is None:
|
||||
sub = subs.get(found["subscription_id"])
|
||||
if sub is None or f"{sub.source}:failing:{sub.target}" in store.state:
|
||||
continue
|
||||
module = ats.readers.HOSTS.get(target.partition(":")[0])
|
||||
if module is None or not hasattr(module, "detail") \
|
||||
or f"ats:failing:{target}" in store.state:
|
||||
continue
|
||||
by_board[target].append(found)
|
||||
due = {target: set(ats.detail_due(store.state, target, [f["source_id"] for f in found],
|
||||
# The id the board's per-role state keeps: `host:slug:id`.
|
||||
found["compound"] = f"{sub.target}:{_native(found, sub)}"
|
||||
by_board[sub.target].append(found)
|
||||
due = {target: set(ats.detail_due(store.state, target, [f["compound"] for f in found],
|
||||
retry_days=settings.retry_days))
|
||||
for target, found in by_board.items()}
|
||||
# Best first; roles whose read failed after every other, so a page that
|
||||
@@ -95,13 +114,13 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
|
||||
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
|
||||
for target in by_board}
|
||||
ranked = sorted((f for found in by_board.values() for f in found),
|
||||
key=lambda f: (f["source_id"] in failed[boards[f["subscription_id"]]],
|
||||
key=lambda f: (f["compound"] in failed[subs[f["subscription_id"]].target],
|
||||
-float(f["score"] or 0), f["uid"]))
|
||||
picked: dict[str, list[str]] = {}
|
||||
total = 0
|
||||
for found in ranked:
|
||||
target = boards[found["subscription_id"]]
|
||||
if found["source_id"] not in due[target]:
|
||||
target = subs[found["subscription_id"]].target
|
||||
if found["compound"] not in due[target]:
|
||||
continue
|
||||
if total >= settings.per_run:
|
||||
break
|
||||
@@ -114,7 +133,8 @@ def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
|
||||
|
||||
|
||||
def batches(config: Config, store: Store) -> list[Batch]:
|
||||
"""The finder: one batch per board, nothing while the stage is off."""
|
||||
"""The finder: one batch per board or search, nothing while the stage is
|
||||
off."""
|
||||
if not stages.stage_on(config, store, "enrich"):
|
||||
return []
|
||||
return [Batch({"target": target, "uids": uids},
|
||||
@@ -125,16 +145,16 @@ def batches(config: Config, store: Store) -> list[Batch]:
|
||||
@job(KIND, "Reading roles on their own pages", resource="http")
|
||||
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Read one board's roles in `uids` on their own pages, in that order,
|
||||
and store what each page filled. `filled` is how many roles changed,
|
||||
`texts` how many of them now have text."""
|
||||
"""Read one board's or search's roles in `uids` on their own pages, in
|
||||
that order, and store what each page filled. `filled` is how many roles
|
||||
changed, `texts` how many of them now have text."""
|
||||
ats = _ats()
|
||||
subs = [s for s in store.subscriptions(source="ats", fetchable=True) if s.target == target]
|
||||
subs = [s for s in _readable(store).values() if s.target == target]
|
||||
if not subs or not uids:
|
||||
return {"target": target, "filled": 0, "texts": 0}
|
||||
sub = subs[0]
|
||||
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
|
||||
rows = [_row(found[uid], target) for uid in uids if uid in found]
|
||||
rows = [_row(found[uid], sub) for uid in uids if uid in found]
|
||||
context.check()
|
||||
with httpx.Client() as client:
|
||||
filled = ats.read_board_details(store.state, target, rows, client=client,
|
||||
@@ -143,7 +163,7 @@ def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
|
||||
texts = 0
|
||||
for row in filled:
|
||||
item = ats.item_of(row, target, label)
|
||||
item.source = "ats"
|
||||
item.source = sub.source
|
||||
store.upsert_role(item, sub)
|
||||
texts += bool(row.get("description"))
|
||||
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
|
||||
|
||||
+51
-2
@@ -5,6 +5,11 @@ link, keep the text as `article_text`. It is on demand, never in the pull,
|
||||
because a fetch of a Jobindex page counts as a click in Jobindex's statistics
|
||||
and one click for a posting somebody opened is fair.
|
||||
|
||||
A role of a search whose reader has `detail` (Jobbank) is read through that
|
||||
`detail`, which reads the ad's own page and states its text, rather than
|
||||
scraping the link; a failure of that read is an `Unreadable` with the
|
||||
category its HTTP status or exception names.
|
||||
|
||||
A Jobindex role links to Jobindex's own `vis-job` page, which carries a short
|
||||
ad and a "Se jobbet" button to the posting itself: the employer's site, The
|
||||
Hub or Jobindex's `jobannonce`. That button's link is found, kept as
|
||||
@@ -19,9 +24,14 @@ message the route answers with.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.models import Role
|
||||
from app.storage import Store
|
||||
from desk.tools import scrape
|
||||
from desk.tools import failures, scrape
|
||||
from desk.tools.checks import Page
|
||||
from plugins.readers import jobindex
|
||||
|
||||
@@ -113,6 +123,43 @@ def _jobindex_text(store: Store, role: Role) -> str:
|
||||
raise Unreadable(failure or own.failure)
|
||||
|
||||
|
||||
def _detail_target(role: Role) -> str:
|
||||
"""The search target of a role whose reader has `detail`, else ''."""
|
||||
from plugins import ats
|
||||
|
||||
target = str(role.raw.get("search") or "")
|
||||
if role.source == "search" and hasattr(ats.reader_of(target), "detail"):
|
||||
return target
|
||||
return ""
|
||||
|
||||
|
||||
def _detail_text(role: Role, target: str) -> str:
|
||||
"""The text of a search role, read by its reader's `detail` on the ad's
|
||||
own page as the pull's enrich stage reads it. Raises `Unreadable` when
|
||||
the read fails or the page states no text."""
|
||||
from plugins import ats
|
||||
|
||||
provider = target.partition(":")[0]
|
||||
try:
|
||||
posted: datetime | None = datetime.fromisoformat(role.published_at)
|
||||
except ValueError:
|
||||
posted = None
|
||||
row: dict[str, Any] = role.raw | {
|
||||
"source_id": role.source_id.removeprefix(f"{provider}:"), "title": role.title,
|
||||
"url": role.original_url, "description": "", "posted": posted}
|
||||
try:
|
||||
with httpx.Client() as client:
|
||||
found = ats.read_detail(client, target, row)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise Unreadable(failures.of_status(exc.response.status_code)) from exc
|
||||
except Exception as exc:
|
||||
category = failures.of_exception(exc)
|
||||
raise Unreadable("" if category == failures.UNKNOWN else category) from exc
|
||||
if not (text := str(found.get("description") or "").strip()):
|
||||
raise Unreadable("empty")
|
||||
return text
|
||||
|
||||
|
||||
def fetch_posting(store: Store, uid: str) -> Role:
|
||||
"""Fetch the posting page once and keep it as plain text, for a role
|
||||
whose description is a teaser. Synchronous: one request the person asked
|
||||
@@ -124,7 +171,9 @@ def fetch_posting(store: Store, uid: str) -> Role:
|
||||
raise AlreadyFetched("The posting text is already fetched.")
|
||||
if not role.original_url:
|
||||
raise NoLink("This role has no link to fetch the posting from.")
|
||||
if _is_jobindex(role):
|
||||
if target := _detail_target(role):
|
||||
text = _detail_text(role, target)
|
||||
elif _is_jobindex(role):
|
||||
text = _jobindex_text(store, role)
|
||||
else:
|
||||
got = scrape.scrape(role.original_url)
|
||||
|
||||
+46
-7
@@ -38,7 +38,9 @@ that read once it is complete, rather than by a second request.
|
||||
|
||||
A pull reads listings only. A host that states a role's text on the role's
|
||||
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
||||
scoring, best roles first, through `read_board_details`.
|
||||
scoring, best roles first, through `read_board_details`. A search provider
|
||||
whose reader has `detail` (Jobbank) is read the same way, under its search
|
||||
target `provider:name`; `read_detail` is the one read the Fetch button makes.
|
||||
|
||||
The limits below are read when a board is read, so a test that replaces one
|
||||
on this module reaches the runner.
|
||||
@@ -53,6 +55,7 @@ import sys
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -114,11 +117,18 @@ def _limits(stale_days: int | None = None) -> _runner.Limits:
|
||||
stale_days=stale_days)
|
||||
|
||||
|
||||
def reader_of(target: str) -> ModuleType | None:
|
||||
"""The reader of a board (`host:slug`) or of a search (`provider:name`),
|
||||
None for a name that is neither. The two never share a name."""
|
||||
host = target.partition(":")[0]
|
||||
return readers.HOSTS.get(host) or readers.PROVIDERS.get(host)
|
||||
|
||||
|
||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
"""The hosts a request or a redirect may land on for one board: what its
|
||||
host module's `allowed` says. Anything else is refused, at the first hop
|
||||
and at every redirect."""
|
||||
module = readers.HOSTS.get(host)
|
||||
"""The hosts a request or a redirect may land on for one board or search:
|
||||
what its module's `allowed` says. Anything else is refused, at the first
|
||||
hop and at every redirect."""
|
||||
module = reader_of(host)
|
||||
return module.allowed(slug) if module is not None else lambda h: False
|
||||
|
||||
|
||||
@@ -540,7 +550,9 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
|
||||
are written and changed by what this read did, so a pull that dropped ids
|
||||
meanwhile keeps its drop."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
module = reader_of(target)
|
||||
if module is None or not hasattr(module, "detail"):
|
||||
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
|
||||
keys = detail_keys(target)
|
||||
now = now or datetime.now(UTC)
|
||||
|
||||
@@ -582,12 +594,39 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
|
||||
return filled
|
||||
|
||||
|
||||
def read_detail(client: httpx.Client, target: str, row: Row) -> dict[str, Any]:
|
||||
"""One role of board or search `target` read on its own page: what the
|
||||
reader's `detail` returns, for the Fetch button. No state is kept and no
|
||||
field is filtered. Raises what `detail` raises, and ValueError for a
|
||||
target whose reader has none."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = reader_of(target)
|
||||
if module is None or not hasattr(module, "detail"):
|
||||
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
|
||||
return module.detail(_getter(client, host, slug), slug, row)
|
||||
|
||||
|
||||
def item_of(row: Row, target: str, label: str) -> Item:
|
||||
"""One row of board `target` as the `Item` the store takes: its id
|
||||
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
|
||||
date is dated now, as `Item` itself would date it."""
|
||||
date is dated now, as `Item` itself would date it. A search's row is
|
||||
stored as `plugins/search.py` stores it: `provider:id`, the employer from
|
||||
`company` and `label` the search's own."""
|
||||
host, _, slug = target.partition(":")
|
||||
description = row.get("description") or ""
|
||||
if host not in readers.HOSTS:
|
||||
return Item(
|
||||
source_id=f"{host}:{row['source_id']}",
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted") or datetime.now(UTC),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {"company": row.get("company") or "", "provider": host,
|
||||
"search": target},
|
||||
)
|
||||
return Item(
|
||||
source_id=f"{host}:{slug}:{row['source_id']}",
|
||||
location=row["url"],
|
||||
|
||||
@@ -239,6 +239,122 @@ def test_a_board_switched_off_since_the_plan_reads_nothing(
|
||||
assert web["asked"] == []
|
||||
|
||||
|
||||
# ----------------------------------------------------------- search roles
|
||||
|
||||
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
|
||||
AD_URL = "https://jobbank.dk/job/1234/acme/role"
|
||||
|
||||
|
||||
#: `Store.enrich_waiting` hands over `source = 'ats'` roles only, and a search's
|
||||
#: are `search`: until it takes both, the tests that need its answer fail.
|
||||
needs_waiting = pytest.mark.xfail(
|
||||
reason="Store.enrich_waiting reads source 'ats' only", strict=False)
|
||||
|
||||
|
||||
def search(store: Store, target: str = "jobbank:cph") -> Any:
|
||||
return store.add_subscription("search", target, label=target, params={"q": "x"})
|
||||
|
||||
|
||||
def search_role(store: Store, target: str, native: str, *, provider: str = "") -> str:
|
||||
"""A role of search `target` as a pull stores it: `provider:id`, the
|
||||
employer in `raw` and no text."""
|
||||
provider = provider or target.partition(":")[0]
|
||||
sub = next(s for s in store.subscriptions(source="search") if s.target == target)
|
||||
row = {"source_id": native, "title": "Researcher", "url": AD_URL.replace("1234", native),
|
||||
"description": "", "company": "Acme A/S", "places": [{"text": "Fyn", "remote": None}]}
|
||||
item = ats.item_of(row, target, sub.label)
|
||||
item.source = "search"
|
||||
assert item.source_id == f"{provider}:{native}"
|
||||
store.upsert_role(item, sub)
|
||||
return item.uid
|
||||
|
||||
|
||||
@needs_waiting
|
||||
def test_a_search_whose_reader_has_detail_is_planned_like_a_board(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
search(store, "jobindex:cph") # no `detail`: left out
|
||||
board(store, "fake:acme")
|
||||
a, b = search_role(store, "jobbank:cph", "1234"), search_role(store, "jobbank:cph", "99")
|
||||
j = search_role(store, "jobindex:cph", "7")
|
||||
f = role(store, "fake:acme", "1")
|
||||
scored(store, "One", {a: 0.9, b: 0.6, j: 0.95, f: 0.7})
|
||||
|
||||
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [a, b]), ("fake:acme", [f])]
|
||||
# The budgets are a board's: `per_board` and `per_run` count the search as one.
|
||||
assert enrich.plan(config(tmp_path, per_board=1), store) == [("jobbank:cph", [a]),
|
||||
("fake:acme", [f])]
|
||||
assert enrich.plan(config(tmp_path, per_run=2), store) == [("jobbank:cph", [a]),
|
||||
("fake:acme", [f])]
|
||||
# A failing search is left out, as a failing board is.
|
||||
store.state["search:failing:jobbank:cph"] = "{}"
|
||||
assert enrich.plan(config(tmp_path), store) == [("fake:acme", [f])]
|
||||
|
||||
|
||||
@needs_waiting
|
||||
def test_a_search_role_read_waits_for_the_retry_and_a_failed_one_goes_last(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
one, two, three = (search_role(store, "jobbank:cph", n) for n in ("1", "2", "3"))
|
||||
scored(store, "One", {one: 0.9, two: 0.8, three: 0.7})
|
||||
store.state["ats:detailed:jobbank:cph"] = json.dumps(["jobbank:cph:1", "jobbank:cph:2"])
|
||||
store.state["ats:detailfailed:jobbank:cph"] = json.dumps(["jobbank:cph:2"])
|
||||
store.state["ats:detailempty:jobbank:cph"] = json.dumps(
|
||||
{"jobbank:cph:1": datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"jobbank:cph:2": "2020-01-01T00:00:00+00:00"})
|
||||
|
||||
# 1 gave no text a moment ago and waits; 2 failed and is due, after 3.
|
||||
assert enrich.plan(config(tmp_path), store) == [("jobbank:cph", [three, two])]
|
||||
|
||||
|
||||
@needs_waiting
|
||||
def test_a_batch_reads_a_search_roles_page_and_stores_its_text(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
uid = search_role(store, "jobbank:cph", "1234")
|
||||
store.put_vectors("fake", [(uid, np.ones(3, dtype=np.float32))])
|
||||
web["answers"][AD_URL] = AD
|
||||
|
||||
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[uid])
|
||||
|
||||
assert out == {"target": "jobbank:cph", "read": 1, "filled": 1, "texts": 1}
|
||||
assert web["asked"] == [AD_URL]
|
||||
got = store.role(uid)
|
||||
assert got is not None and got.description
|
||||
assert (got.source, got.source_id, got.company) == ("search", "jobbank:1234", "Acme A/S")
|
||||
assert got.raw["search"] == "jobbank:cph" and got.raw["provider"] == "jobbank"
|
||||
assert store.vectors("fake") == {}
|
||||
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == ["jobbank:cph:1234"]
|
||||
# Read, it is not planned again, and has no text left to wait for.
|
||||
scored(store, "One", {uid: 0.9})
|
||||
assert enrich.plan(config(tmp_path), store) == []
|
||||
|
||||
|
||||
@needs_waiting
|
||||
def test_a_search_batch_counts_a_404_as_read_as_a_board_does(
|
||||
store: Store, tmp_path: Path, web: dict[str, Any]) -> None:
|
||||
search(store)
|
||||
one, two = search_role(store, "jobbank:cph", "1"), search_role(store, "jobbank:cph", "2")
|
||||
out = enrich.run_batch(config(tmp_path), store, target="jobbank:cph", uids=[one, two])
|
||||
assert out["filled"] == 0 and out["read"] == 2
|
||||
assert json.loads(store.state["ats:detailed:jobbank:cph"]) == [
|
||||
"jobbank:cph:1", "jobbank:cph:2"]
|
||||
|
||||
|
||||
def test_a_reader_is_found_for_a_board_and_a_search_and_one_without_detail_is_refused(
|
||||
web: dict[str, Any]) -> None:
|
||||
assert ats.reader_of("fake:acme") is readers.HOSTS["fake"]
|
||||
assert ats.reader_of("jobbank:cph") is readers.PROVIDERS["jobbank"]
|
||||
assert ats.reader_of("nowhere:x") is None
|
||||
with httpx.Client() as client:
|
||||
for target in ("jobindex:cph", "plain:acme", "nowhere:x"):
|
||||
with pytest.raises(ValueError, match="no reader"):
|
||||
ats.read_detail(client, target, {"source_id": "1", "url": BASE})
|
||||
with pytest.raises(ValueError, match="no reader"):
|
||||
ats.read_board_details({}, target, [{"source_id": "1", "url": BASE}],
|
||||
client=client)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cascade
|
||||
|
||||
class _Context:
|
||||
|
||||
@@ -8,12 +8,15 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import httpx
|
||||
|
||||
from app.jobs import posting
|
||||
from app.storage import Store
|
||||
from desk.models import Item
|
||||
from desk.tools import scrape
|
||||
from desk.tools.article import ArticleContent
|
||||
from desk.tools.checks import Page
|
||||
from plugins import ats
|
||||
from prepped import upsert
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -164,3 +167,82 @@ def test_a_kept_link_is_followed_without_fetching_the_ad_page_again(
|
||||
web = Web(monkeypatch, {}, {LINK: "The real posting."})
|
||||
assert posting.fetch_posting(store, uid).article_text == "The real posting."
|
||||
assert web.asked == [LINK]
|
||||
|
||||
|
||||
AD = (ROOT / "tests" / "fixtures" / "search" / "jobbank-detail.html").read_bytes()
|
||||
AD_URL = "https://jobbank.dk/job/1234/acme/role"
|
||||
|
||||
|
||||
def search_role(store: Store, provider: str = "jobbank", url: str = AD_URL) -> str:
|
||||
"""A role of a saved search as a pull stores it: `provider:id`, source
|
||||
`search`, the search's target in `raw`."""
|
||||
target = f"{provider}:cph"
|
||||
sub = store.add_subscription("search", target, label=target, params={"q": "x"})
|
||||
row = {"source_id": "1234", "title": "Researcher", "url": url, "description": "",
|
||||
"company": "Acme A/S"}
|
||||
item = ats.item_of(row, target, target)
|
||||
item.source = "search"
|
||||
upsert(store, item, sub)
|
||||
return item.uid
|
||||
|
||||
|
||||
class Pages:
|
||||
"""`ats._get` answering from `answers` (URL -> body or the exception to
|
||||
raise), remembering what it was asked."""
|
||||
|
||||
def __init__(self, monkeypatch: pytest.MonkeyPatch, answers: dict[str, object]) -> None:
|
||||
self.asked: list[str] = []
|
||||
self.answers = answers
|
||||
monkeypatch.setattr(ats, "_get", self.get)
|
||||
|
||||
def get(self, client: object, url: str, host: str, slug: str, **kw: object) -> bytes:
|
||||
self.asked.append(url)
|
||||
answer = self.answers[url]
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer # type: ignore[return-value]
|
||||
|
||||
|
||||
def test_a_search_role_whose_reader_has_detail_is_read_by_its_detail(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = search_role(store)
|
||||
pages = Pages(monkeypatch, {AD_URL: AD})
|
||||
web = Web(monkeypatch, {}, {})
|
||||
|
||||
got = posting.fetch_posting(store, uid)
|
||||
|
||||
assert got.article_text and got.article_at and got.posting_url == ""
|
||||
assert pages.asked == [AD_URL] and web.asked == [] # no scrape of the link
|
||||
with pytest.raises(posting.AlreadyFetched):
|
||||
posting.fetch_posting(store, uid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("answer", "message"), [
|
||||
(httpx.HTTPStatusError("404", request=httpx.Request("GET", AD_URL),
|
||||
response=httpx.Response(404, request=httpx.Request("GET", AD_URL))),
|
||||
"gone"),
|
||||
(httpx.HTTPStatusError("429", request=httpx.Request("GET", AD_URL),
|
||||
response=httpx.Response(429, request=httpx.Request("GET", AD_URL))),
|
||||
"refusing"),
|
||||
(httpx.ConnectError("down"), "could not be reached"),
|
||||
(b"<html>Just a moment...</html>", "Nothing readable"),
|
||||
(b'<script type="application/ld+json">{"@type": "JobPosting"}</script>', "no readable text"),
|
||||
])
|
||||
def test_a_failed_detail_read_says_why(store: Store, monkeypatch: pytest.MonkeyPatch,
|
||||
answer: object, message: str) -> None:
|
||||
uid = search_role(store)
|
||||
Pages(monkeypatch, {AD_URL: answer})
|
||||
with pytest.raises(posting.Unreadable, match=message) as failure:
|
||||
posting.fetch_posting(store, uid)
|
||||
assert failure.value.status == 422
|
||||
stored = store.role(uid)
|
||||
assert stored is not None and stored.article_text == ""
|
||||
|
||||
|
||||
def test_a_search_role_without_detail_is_scraped_from_its_link(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = search_role(store, "adzuna", "https://www.adzuna.dk/details/1")
|
||||
pages = Pages(monkeypatch, {})
|
||||
Web(monkeypatch, {}, {"https://www.adzuna.dk/details/1": "The posting."})
|
||||
assert posting.fetch_posting(store, uid).article_text == "The posting."
|
||||
assert pages.asked == []
|
||||
Reference in new issue
Block a user