Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
779b136757
commit
902089a5e2
14 files changed
+1608
-1492
No files matched your search
-11
@@ -1,11 +0,0 @@
|
||||
User-agent: *
|
||||
Disallow: /applybutton/
|
||||
Disallow: /talentcommunity/
|
||||
Disallow: /mobile/talentcommunity/
|
||||
Disallow: /emailsubscribe/
|
||||
Disallow: /email/image/
|
||||
Disallow: /services/
|
||||
Disallow: /preapply/
|
||||
Disallow: /error
|
||||
Disallow: /unsubscribe/
|
||||
Disallow: /reset/
|
||||
+6
-5
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import smartrecruiters # noqa: E402
|
||||
from plugins.readers import _runner, smartrecruiters # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -487,7 +487,7 @@ def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
|
||||
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
clock = iter(range(0, 1000, 10))
|
||||
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
|
||||
monkeypatch.setattr(_runner.time, "monotonic", lambda: float(next(clock)))
|
||||
monkeypatch.setattr(ats, "DEADLINE", 25.0)
|
||||
body = (b"x" for _ in range(100))
|
||||
with _client(lambda request: httpx.Response(200, content=body)) as client, \
|
||||
@@ -752,12 +752,13 @@ def test_teamtailor_is_read_through_the_streamed_get_and_a_redirect_elsewhere_is
|
||||
|
||||
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
||||
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
rows, complete = answer # type: ignore[misc]
|
||||
return _runner.Listing(rows, complete, [row["source_id"] for row in rows])
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
|
||||
"run": "run-1"})
|
||||
return source
|
||||
|
||||
@@ -155,19 +155,6 @@ def test_an_empty_board_is_complete_and_a_401_is_a_failure(web: dict[str, Any])
|
||||
ats._fetch_rows(client, "hibob:liquidia")
|
||||
|
||||
|
||||
def test_the_robots_txt_that_is_the_apps_own_page_allows(web: dict[str, Any]) -> None:
|
||||
"""The host answers /robots.txt with its single-page app's HTML, a 200
|
||||
with no rules."""
|
||||
web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \
|
||||
b'<!DOCTYPE html><html><head><title>Careers</title></head><body></body></html>'
|
||||
with httpx.Client() as client:
|
||||
ats._check_robots(client, "hibob:dixa")
|
||||
web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \
|
||||
b"User-agent: *\nDisallow: /\n"
|
||||
with pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "hibob:dixa")
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
def test_the_module_is_registered_as_a_paged_host() -> None:
|
||||
|
||||
@@ -133,25 +133,6 @@ def test_a_429_after_the_first_page_ends_the_read_incomplete(web: dict[str, Any]
|
||||
assert len(rows) == 5 and not complete
|
||||
|
||||
|
||||
def test_robots_are_read_from_the_api_host_whose_404_allows(web: dict[str, Any]) -> None:
|
||||
request = httpx.Request("GET", "https://api.hr-manager.net/robots.txt")
|
||||
web["answers"]["https://api.hr-manager.net/robots.txt"] = httpx.HTTPStatusError(
|
||||
"404", request=request, response=httpx.Response(404, request=request))
|
||||
state: dict[str, str] = {}
|
||||
with httpx.Client() as client:
|
||||
ats._check_robots(client, "hrmanager:stsgroup_tr", state)
|
||||
assert '"allowed": true' in state["ats:robots:hrmanager:stsgroup_tr"]
|
||||
assert web["asked"] == ["https://api.hr-manager.net/robots.txt"]
|
||||
|
||||
|
||||
def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide(
|
||||
web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://api.hr-manager.net/robots.txt"] = \
|
||||
b"User-agent: *\nDisallow: /JobPortal.svc/\n"
|
||||
with httpx.Client() as client, pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "hrmanager:stsgroup_tr")
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
def test_the_module_is_registered_as_a_paged_host() -> None:
|
||||
|
||||
+49
-6
@@ -20,7 +20,9 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from app.models import Application # noqa: E402
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.jobs import cascade, pull # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
@@ -202,14 +204,16 @@ def _row(job_id: str) -> dict[str, object]:
|
||||
"description": "", "remote": None, "salary_text": ""}
|
||||
|
||||
|
||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
|
||||
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None:
|
||||
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
|
||||
with `filtered`, a read the board's filter narrowed."""
|
||||
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
rows, complete = answer # type: ignore[misc]
|
||||
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered)
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, object]:
|
||||
@@ -275,6 +279,45 @@ def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
|
||||
assert pull.failing_boards(acme) == {}
|
||||
|
||||
|
||||
def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
|
||||
_pull(acme)
|
||||
uid = {r.source_id: r.uid for r in acme.roles()}
|
||||
agent = acme.save_agent("Hunt")
|
||||
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
|
||||
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
|
||||
|
||||
_answer(monkeypatch, ([_row("1")], True), filtered=True)
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
||||
assert result["closed"] == 1
|
||||
assert acme.state["ats:filtered:workable:acme"] == "1"
|
||||
|
||||
# Read without the filter again, the board closes as before.
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
|
||||
|
||||
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
||||
board="workable", params={"countries": ["dk"], "remote": True})
|
||||
asked: list[object] = []
|
||||
|
||||
def fake(module: object, slug: str, get: object, ask: object, **kwargs: object
|
||||
) -> _runner.Listing:
|
||||
asked.append(ask)
|
||||
return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type]
|
||||
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
_pull(store)
|
||||
[ask] = asked
|
||||
declared = readers.HOSTS["workable"].READS.filters.declared()
|
||||
assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ------------------------------------------------------ one pull at a time
|
||||
|
||||
def _ago(**delta: float) -> str:
|
||||
|
||||
+11
-193
@@ -1,6 +1,6 @@
|
||||
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
||||
every host: the registry and its contract check, the fetch a host is handed,
|
||||
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
||||
the per-role detail reads, and Lever's EU instance. No network —
|
||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,7 +11,7 @@ import subprocess
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -23,6 +23,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins.readers import lever # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -242,11 +243,9 @@ def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatc
|
||||
|
||||
# ------------------------------------------------------- a fake host
|
||||
|
||||
def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||
robots: bool = False) -> types.ModuleType:
|
||||
def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
|
||||
"""A host `fake` whose feed is a JSON list of ids, each role with a
|
||||
teaser for a description, and, with `detail`, a per-role page; with
|
||||
`robots`, its listing is the page robots.txt must allow."""
|
||||
teaser for a description, and, with `detail`, a per-role page."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
@@ -260,8 +259,6 @@ def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||
for i in json.loads(body)]
|
||||
if detail is not None:
|
||||
module.detail = detail
|
||||
if robots:
|
||||
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
return module
|
||||
|
||||
|
||||
@@ -411,8 +408,8 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
ids stay, and the next pull does not read them in detail again."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
||||
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
|
||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
@@ -511,7 +508,7 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
|
||||
"title": "Another title"}
|
||||
|
||||
caplog.set_level("DEBUG", logger=ats.log.name)
|
||||
caplog.set_level("DEBUG", logger=_runner.log.name)
|
||||
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
||||
assert (item.body, item.title, item.published_at) == (
|
||||
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
||||
@@ -519,8 +516,9 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
|
||||
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
|
||||
assert "department, salary_text, title left out" in caplog.text
|
||||
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
|
||||
"salary_min", "salary_max", "posted"}
|
||||
assert set(ats.DETAIL_FIELDS) == {"description", "location", "places", "teaser", "remote",
|
||||
"country", "employment", "currency", "salary_min",
|
||||
"salary_max", "posted"}
|
||||
|
||||
|
||||
def _paged_host() -> types.ModuleType:
|
||||
@@ -583,186 +581,6 @@ def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
||||
assert "ats:detailed:fake:acme" not in source.state
|
||||
|
||||
|
||||
# ------------------------------------------------------------ robots.txt
|
||||
|
||||
ROBOTS = "https://jobs.fake.example/robots.txt"
|
||||
|
||||
|
||||
def test_robots_that_allow_the_page_let_the_board_be_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("robots", [
|
||||
b"User-agent: *\nDisallow: /acme/\n",
|
||||
# A rule for this app by name counts as well as the one for everyone.
|
||||
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
|
||||
.encode(),
|
||||
])
|
||||
def test_robots_that_disallow_the_page_fail_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
|
||||
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == []
|
||||
assert web["asked"] == [ROBOTS] # the listing is never fetched
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
host = _fake_host(robots=True)
|
||||
items, _state = _run(monkeypatch, host)
|
||||
assert len(items) == 1 # paths are case-sensitive by default
|
||||
host.ROBOTS_ANY_CASE = True
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, host)
|
||||
assert items == []
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
||||
# The longest matching rule wins, whatever the order of the lines.
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
||||
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
|
||||
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
|
||||
# An allow wins a tie.
|
||||
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
|
||||
# `*` inside a path, and the query counts as part of it.
|
||||
("User-agent: *\nDisallow: /*company\n",
|
||||
"https://career5.successfactors.eu/career?company=x", False),
|
||||
# `$` anchors the end.
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
|
||||
# An empty Disallow allows.
|
||||
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
|
||||
# A byte order mark and CRLF line ends.
|
||||
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
|
||||
# Paths are case-sensitive.
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
|
||||
# A group for this app by name counts, a group for someone else does
|
||||
# not, and a token matches whole, not as a substring.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
|
||||
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
# Two agents share one group, and a comment is no rule.
|
||||
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
|
||||
"https://x.example/jobs/1", False),
|
||||
# A group for this app that allows does not undo the `*` group's refusal.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
# Rules before any user-agent line belong to no group.
|
||||
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
])
|
||||
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
|
||||
assert ats._robots_allows(robots.encode(), page) is allowed
|
||||
|
||||
|
||||
def test_a_robots_verdict_is_kept_for_a_day(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
module = _fake_host(robots=True)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
assert web["asked"] == [ROBOTS]
|
||||
|
||||
# Within the day the kept verdict holds, even though the site changed.
|
||||
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert (items, web["asked"]) == ([], [])
|
||||
|
||||
# A day on, robots.txt is read again.
|
||||
kept = json.loads(state["ats:robots:fake:acme"])
|
||||
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
||||
state["ats:robots:fake:acme"] = json.dumps(kept)
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
|
||||
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [404, 410, 401, 403])
|
||||
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
|
||||
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [
|
||||
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
|
||||
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
|
||||
])
|
||||
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == [] and web["asked"] == [ROBOTS]
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
|
||||
"robots.txt could not be read"
|
||||
assert "ats:robots:fake:acme" not in state
|
||||
|
||||
|
||||
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
|
||||
where a robots.txt that disallows everything is served."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if str(request.url) == ROBOTS:
|
||||
return httpx.Response(301, headers={"location": location})
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
|
||||
return httpx.Response(200, text='["1"]')
|
||||
return handler
|
||||
|
||||
|
||||
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", [
|
||||
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
|
||||
"http://www.jobs.fake.example/robots.txt",
|
||||
])
|
||||
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
||||
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Refused):
|
||||
ats._get(client, ROBOTS, "fake", "acme")
|
||||
|
||||
|
||||
def test_fetch_board_honours_robots_txt(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with pytest.raises(ats._Disallowed) as caught:
|
||||
ats.fetch_board("fake:acme")
|
||||
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- Lever EU
|
||||
|
||||
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
||||
subscription resolves to, series and their union, whole and paged listings,
|
||||
completeness from the declaration, the sweep across pulls, the 10% guard,
|
||||
the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||
network: `get` is a dict of URL -> body or exception."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, Filters, Key, Reads, _runner # noqa: E402
|
||||
|
||||
NO_DELAY = _runner.Limits(page_delay=0)
|
||||
|
||||
|
||||
def _status(code: int) -> httpx.HTTPStatusError:
|
||||
request = httpx.Request("GET", "https://jobs.fake.example/")
|
||||
return httpx.HTTPStatusError(str(code), request=request,
|
||||
response=httpx.Response(code, request=request))
|
||||
|
||||
|
||||
class Web:
|
||||
"""A `get` answering from a dict of URL -> body or exception."""
|
||||
|
||||
def __init__(self, answers: dict[str, Any] | None = None) -> None:
|
||||
self.answers = answers or {}
|
||||
self.asked: list[str] = []
|
||||
|
||||
def __call__(self, url: str, **kwargs: Any) -> bytes:
|
||||
self.asked.append(url)
|
||||
answer = self.answers.get(url, _status(404))
|
||||
if isinstance(answer, BaseException):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
|
||||
def _row(i: str, **extra: Any) -> dict[str, Any]:
|
||||
return {"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
|
||||
"places": [{"text": "Berlin", "remote": None}]} | extra
|
||||
|
||||
|
||||
def _whole(**reads: Any) -> Any:
|
||||
"""A declared whole-listing board: the feed is a JSON list of ids, one
|
||||
feed per `ask.countries` value."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
|
||||
"complete": "whole", "places": "many", "robots": "test"} | reads))
|
||||
module.feed_url = lambda slug, ask: (f"https://jobs.fake.example/{slug}/list"
|
||||
+ "".join(f"?c={c}" for c in ask.countries))
|
||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
|
||||
return module
|
||||
|
||||
|
||||
def _paged(pages: list[list[str]], total: int | None = None, **reads: Any) -> Any:
|
||||
"""A declared paged board over `pages`, the cursor being the page index."""
|
||||
module = _whole(**({"listing": "paged", "complete": "stated" if total is not None
|
||||
else "end"} | reads))
|
||||
del module.feed_url, module.parse
|
||||
module.asked = []
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
index = int(cursor or 0)
|
||||
module.asked.append((index, ask.countries))
|
||||
body = get(f"https://jobs.fake.example/{slug}/page/{index}")
|
||||
ids = json.loads(body) if body else pages[index]
|
||||
more = index + 1 < len(pages) and bool(ids)
|
||||
return ids, [{"id": i} for i in ids], total, index + 1 if more else None
|
||||
|
||||
module.page = page
|
||||
module.rows = lambda ids, slug: [_row(i) for i in ids]
|
||||
return module
|
||||
|
||||
|
||||
def _pages_web(pages: list[list[str]]) -> Web:
|
||||
return Web({f"https://jobs.fake.example/acme/page/{n}": json.dumps(ids).encode()
|
||||
for n, ids in enumerate(pages)})
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- ask
|
||||
|
||||
def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
|
||||
board = _whole(filters=Filters(country="one"))
|
||||
ask = _runner.ask_for(board, {"countries": ["dk", "se"], "remote": True, "places": ["X"]})
|
||||
assert (ask.countries, ask.remote, ask.places, ask.params) == (("DK", "SE"), None, (), {})
|
||||
# The params override the reach per dimension; the reach fills the rest.
|
||||
both = _whole(filters=Filters(country="many", remote="many"))
|
||||
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
|
||||
assert (ask.countries, ask.remote) == (("DE",), True)
|
||||
# A module without READS reads the whole board.
|
||||
assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
|
||||
|
||||
|
||||
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
module = _whole(scope="search", listing="paged", complete="stated", text="excerpt",
|
||||
filters=Filters(query="one", place="one"),
|
||||
key=Key(name="fake", fields=("id", "secret"), env=("FAKE_ID", "FAKE_SECRET")))
|
||||
module.PARAMS = {"q": "query", "area": "place"}
|
||||
monkeypatch.delenv("FAKE_ID", raising=False)
|
||||
monkeypatch.setenv("FAKE_SECRET", "s")
|
||||
ask = _runner.ask_for(module, {"q": "python", "area": 1, "pages": 2},
|
||||
keys={"fake": {"id": "i"}})
|
||||
assert (ask.query, ask.places) == ("python", ("1",))
|
||||
assert ask.params == {"q": "python", "area": 1, "pages": 2,
|
||||
"key": {"id": "i", "secret": "s"}}
|
||||
with pytest.raises(ValueError, match=r"sources\.search\.fake: id, secret, or FAKE_ID"):
|
||||
_runner.ask_for(module, {"q": "x"}, keys={})
|
||||
|
||||
|
||||
def test_series_are_one_a_value_for_one_and_one_for_many() -> None:
|
||||
ask = Ask(countries=("DK", "SE"), places=("Aarhus",))
|
||||
one = _whole(filters=Filters(country="one", place="many"))
|
||||
assert [(a.countries, a.places) for a in _runner.series(one, ask)] == [
|
||||
(("DK",), ("Aarhus",)), (("SE",), ("Aarhus",))]
|
||||
many = _whole(filters=Filters(country="many"))
|
||||
assert _runner.series(many, ask) == [ask]
|
||||
|
||||
|
||||
def test_ids_filters_are_mapped_through_the_facets() -> None:
|
||||
module = _whole(filters=Filters(country="ids"))
|
||||
module.facets = lambda get, slug: {}
|
||||
ask = Ask(countries=("DK", "XX"))
|
||||
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
|
||||
assert mapped.countries == ("h-1",)
|
||||
# None known: the host has nothing in reach, and the read is empty.
|
||||
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
|
||||
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
|
||||
facets={"country": {}})
|
||||
assert (listing.rows, listing.complete, listing.filtered) == ([], True, True)
|
||||
|
||||
|
||||
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
|
||||
module = _whole(filters=Filters(country="ids"))
|
||||
calls: list[str] = []
|
||||
module.facets = lambda get, slug: calls.append(slug) or {"country": {"DK": "7"}}
|
||||
state: dict[str, str] = {}
|
||||
assert _runner.facets_for(module, "acme", Web(), Ask(), state, "k") == {}
|
||||
for _ in range(2):
|
||||
found = _runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k")
|
||||
assert found == {"country": {"DK": "7"}}
|
||||
assert calls == ["acme"]
|
||||
kept = json.loads(state["k"])
|
||||
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
||||
state["k"] = json.dumps(kept)
|
||||
_runner.facets_for(module, "acme", Web(), Ask(countries=("DK",)), state, "k")
|
||||
assert calls == ["acme", "acme"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- listing
|
||||
|
||||
def test_a_whole_listing_reads_one_feed_a_series_and_unions_by_id() -> None:
|
||||
module = _whole(filters=Filters(country="one"))
|
||||
web = Web({"https://jobs.fake.example/acme/list?c=DK": b'["1", "2"]',
|
||||
"https://jobs.fake.example/acme/list?c=SE": b'["2", "3"]'})
|
||||
listing = _runner.read_listing(module, "acme", web, Ask(countries=("DK", "SE")),
|
||||
limits=NO_DELAY)
|
||||
assert [r["source_id"] for r in listing.rows] == ["1", "2", "3"]
|
||||
assert (listing.complete, listing.filtered, listing.ids) == (True, True, ["1", "2", "3"])
|
||||
|
||||
|
||||
def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
|
||||
"""Greenhouse's shape: the feed with the text, and one without it."""
|
||||
module = _whole()
|
||||
module.fallback_url = lambda slug: f"https://jobs.fake.example/{slug}/small"
|
||||
web = Web({"https://jobs.fake.example/acme/list": _runner.TooLarge("over"),
|
||||
"https://jobs.fake.example/acme/small": b'["1"]'})
|
||||
listing = _runner.read_listing(module, "acme", web)
|
||||
assert ([r["source_id"] for r in listing.rows], listing.complete) == (["1"], True)
|
||||
# A reader without a fallback fails.
|
||||
del module.fallback_url
|
||||
with pytest.raises(_runner.TooLarge):
|
||||
_runner.read_listing(module, "acme", web)
|
||||
|
||||
|
||||
def test_a_module_without_reads_gets_the_older_signatures() -> None:
|
||||
module: Any = types.ModuleType("old")
|
||||
module.NAME = "old"
|
||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
|
||||
listing = _runner.read_listing(module, "acme",
|
||||
Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
|
||||
Ask(countries=("DK",)))
|
||||
assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
|
||||
|
||||
|
||||
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
|
||||
pages = [["1", "2"], ["3"]]
|
||||
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
|
||||
limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2", "3"], True)
|
||||
short = _runner.read_listing(_paged(pages, total=4), "acme", _pages_web(pages),
|
||||
limits=NO_DELAY)
|
||||
assert not short.complete
|
||||
|
||||
|
||||
def test_the_page_cap_leaves_a_board_incomplete() -> None:
|
||||
pages = [["1"], ["2"], ["3"]]
|
||||
listing = _runner.read_listing(_paged(pages), "acme", _pages_web(pages),
|
||||
limits=_runner.Limits(max_pages=2, page_delay=0))
|
||||
assert (listing.ids, listing.complete, listing.sweep) == (["1", "2"], False, None)
|
||||
|
||||
|
||||
def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
|
||||
"""A host that ignores the cursor answers page one again; without a total
|
||||
the read would never end, so a page with nothing new stops it."""
|
||||
pages = [["1", "2"], ["1", "2"], ["3"]]
|
||||
module = _paged(pages)
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], False)
|
||||
assert [index for index, _ in module.asked] == [0, 1]
|
||||
|
||||
|
||||
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
|
||||
pages = [["1"], ["2"]]
|
||||
web = _pages_web(pages)
|
||||
web.answers["https://jobs.fake.example/acme/page/1"] = _status(429)
|
||||
listing = _runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1"], False)
|
||||
web.answers["https://jobs.fake.example/acme/page/0"] = _status(429)
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
_runner.read_listing(_paged(pages), "acme", web, limits=NO_DELAY)
|
||||
|
||||
|
||||
def test_a_search_is_never_complete_when_it_says_so() -> None:
|
||||
pages = [["1"]]
|
||||
module = _paged(pages, scope="search", complete="never", text="excerpt")
|
||||
assert not _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY).complete
|
||||
|
||||
|
||||
def test_paged_series_each_read_to_the_end_and_union() -> None:
|
||||
pages = [["1", "2"]]
|
||||
module = _paged(pages, filters=Filters(country="one"))
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages),
|
||||
Ask(countries=("DK", "SE")), limits=NO_DELAY)
|
||||
assert module.asked == [(0, ("DK",)), (0, ("SE",))]
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ sweep
|
||||
|
||||
def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
|
||||
pages = [["1"], ["2"], ["3"], ["4"], ["5"]]
|
||||
module = _paged(pages, total=5)
|
||||
web, state = _pages_web(pages), {}
|
||||
limits = _runner.Limits(max_pages=2, page_delay=0)
|
||||
seen: list[tuple[list[str], bool]] = []
|
||||
for _pull in range(3):
|
||||
listing = _runner.read(state, "ats", "fake:acme", module, "acme", web, limits=limits)
|
||||
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||
seen.append((listing.ids, listing.complete))
|
||||
assert seen == [(["1", "2"], False), (["3", "4"], False), (["1", "2", "3", "4", "5"], True)]
|
||||
assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4]
|
||||
assert "ats:sweep:fake:acme" not in state
|
||||
assert json.loads(state["ats:count:fake:acme"])["count"] == 5
|
||||
|
||||
|
||||
def test_a_sweep_starts_again_when_the_ask_changed() -> None:
|
||||
pages = [["1"], ["2"], ["3"]]
|
||||
module = _paged(pages, filters=Filters(country="many"))
|
||||
web, state = _pages_web(pages), {}
|
||||
limits = _runner.Limits(max_pages=2, page_delay=0)
|
||||
first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits)
|
||||
_runner.stamp(state, "ats", "t", "r", first, first.ids)
|
||||
module.asked.clear()
|
||||
_runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)), limits=limits)
|
||||
assert module.asked[0][0] == 0
|
||||
|
||||
|
||||
def test_a_board_at_its_ceiling_is_not_swept() -> None:
|
||||
pages = [["1", "2"], ["3", "4"], ["5"]]
|
||||
module = _paged(pages, total=5, page_size=2, ceiling=4)
|
||||
listing = _runner.read({}, "ats", "t", module, "acme", _pages_web(pages),
|
||||
limits=_runner.Limits(max_pages=2, page_delay=0))
|
||||
assert (listing.complete, listing.sweep) == (False, None)
|
||||
|
||||
|
||||
def test_a_search_is_not_swept() -> None:
|
||||
pages = [["1"], ["2"]]
|
||||
module = _paged(pages, total=2)
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), can_sweep=False,
|
||||
limits=_runner.Limits(max_pages=1, page_delay=0))
|
||||
assert listing.sweep is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------- the 10% guard
|
||||
|
||||
def test_a_suspect_read_far_under_the_last_count_fails() -> None:
|
||||
module = _whole(empty="suspect")
|
||||
web = Web({"https://jobs.fake.example/acme/list": json.dumps(
|
||||
[str(n) for n in range(20)]).encode()})
|
||||
state: dict[str, str] = {}
|
||||
listing = _runner.read(state, "ats", "t", module, "acme", web)
|
||||
_runner.stamp(state, "ats", "t", "r", listing, listing.ids)
|
||||
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
|
||||
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
|
||||
_runner.read(state, "ats", "t", module, "acme", web)
|
||||
# A reader whose empty answer is trusted reads it as it is.
|
||||
module.READS = module.READS.model_copy(update={"empty": "trusted"})
|
||||
assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"]
|
||||
|
||||
|
||||
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
||||
"""A board read whole last time and with a filter now lists far fewer
|
||||
roles; that is the filter, not a failure."""
|
||||
module = _whole(filters=Filters(country="one"))
|
||||
web = Web({"https://jobs.fake.example/acme/list": json.dumps(
|
||||
[str(n) for n in range(50)]).encode(),
|
||||
"https://jobs.fake.example/acme/list?c=DK": b'["1"]'})
|
||||
state: dict[str, str] = {}
|
||||
listing = _runner.read(state, "ats", "t", module, "acme", web)
|
||||
_runner.stamp(state, "ats", "t", "r", listing, listing.ids)
|
||||
filtered = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)))
|
||||
assert filtered.ids == ["1"] and filtered.filtered
|
||||
|
||||
|
||||
# ------------------------------------------------------ through ATSSource
|
||||
|
||||
@pytest.fixture
|
||||
def board(monkeypatch: pytest.MonkeyPatch) -> Any:
|
||||
pages = [["1"], ["2"], ["3"]]
|
||||
module = _paged(pages, total=3, empty="suspect")
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
||||
web = _pages_web(pages)
|
||||
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: web(url))
|
||||
module.web = web
|
||||
return module
|
||||
|
||||
|
||||
def _source(state: dict[str, str], params: dict[str, Any] | None = None) -> ats.ATSSource:
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
|
||||
"params": params or {}}], "run": "r"})
|
||||
source.state = state
|
||||
return source
|
||||
|
||||
|
||||
def test_ats_source_sweeps_and_stamps_every_id_once_the_sweep_ends(board: Any) -> None:
|
||||
state: dict[str, str] = {}
|
||||
items = list(_source(state).fetch())
|
||||
assert [i.source_id for i in items] == ["fake:acme:1", "fake:acme:2"]
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == 2
|
||||
|
||||
items = list(_source(state).fetch())
|
||||
assert [i.source_id for i in items] == ["fake:acme:3"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
assert json.loads(state["ats:ids:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
|
||||
|
||||
def test_ats_source_records_a_suspect_read_as_failing(board: Any) -> None:
|
||||
state = {"ats:count:fake:acme": json.dumps(
|
||||
{"asks": _runner.fingerprint([Ask()]), "count": 100})}
|
||||
board.web.answers = {"https://jobs.fake.example/acme/page/0": b"[]"}
|
||||
assert list(_source(state).fetch()) == []
|
||||
assert "under 10%" in json.loads(state["ats:failing:fake:acme"])["reason"]
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
|
||||
board.READS = board.READS.model_copy(update={"filters": Filters(country="many")})
|
||||
state: dict[str, str] = {}
|
||||
list(_source(state, {"countries": ["dk"]}).fetch())
|
||||
assert board.asked[0] == (0, ("DK",))
|
||||
assert state["ats:filtered:fake:acme"] == "1"
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- raw
|
||||
|
||||
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
||||
row = _row("1", places=[{"text": "Berlin", "remote": False},
|
||||
{"text": "Remote (DE)", "remote": True}],
|
||||
teaser="Short.", seniority="senior", employment="full_time", country="DE",
|
||||
description="Long.", posted=None, location="Berlin; Remote (DE)")
|
||||
raw = _runner.raw_of(row)
|
||||
assert raw["places"] == row["places"]
|
||||
assert raw["place"] == "Berlin; Remote (DE)"
|
||||
assert (raw["teaser"], raw["seniority"], raw["employment"], raw["country"]) == (
|
||||
"Short.", "senior", "full_time", "DE")
|
||||
assert raw["location"] == "Berlin; Remote (DE)"
|
||||
assert not {"description", "title", "url", "posted", "source_id"} & set(raw)
|
||||
assert (raw["department"], raw["salary_text"], raw["currency"], raw["salary_min"]) == (
|
||||
"", "", "", None)
|
||||
|
||||
|
||||
def test_a_row_without_places_has_its_location_as_the_one_place() -> None:
|
||||
raw = _runner.raw_of({"source_id": "1", "title": "t", "url": "u",
|
||||
"location": "Berlin, Munich"})
|
||||
assert raw["places"] == [{"text": "Berlin, Munich", "remote": None}]
|
||||
assert raw["place"] == "Berlin, Munich"
|
||||
assert _runner.raw_of({"source_id": "1", "title": "t", "url": "u"})["places"] == []
|
||||
|
||||
|
||||
def test_a_detail_location_without_places_becomes_the_one_place() -> None:
|
||||
module = _whole(text="detail")
|
||||
module.detail = lambda get, slug, row: {"location": "Aarhus", "employment": "contract",
|
||||
"description": "Full."}
|
||||
rows, done, failed = _runner.read_details(module, "fake", "acme", Web(), [_row("1")],
|
||||
set(), set(), limits=NO_DELAY)
|
||||
assert rows[0]["places"] == [{"text": "Aarhus", "remote": None}]
|
||||
assert (rows[0]["employment"], rows[0]["description"]) == ("contract", "Full.")
|
||||
assert (done, failed) == ({"fake:acme:1"}, set())
|
||||
+160
-254
@@ -1,15 +1,17 @@
|
||||
"""The `search` plugin, `country_of`, and what the pull and the CLI do with
|
||||
searches. No network: Jobindex's feed is a trimmed copy of a real one
|
||||
(2026-09-23), Adzuna's is written from its documented response, Platsbanken's
|
||||
is trimmed from a live answer (2026-09-29)."""
|
||||
"""The `search` plugin over the runner and the provider readers, `country_of`,
|
||||
and what the pull does with searches. No network: `_runner.fetch` is a stub,
|
||||
and a fake provider stands in where the pull is what is tested. The readers'
|
||||
own parsing is tested in `test_readers_<provider>.py`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -18,7 +20,8 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from plugins import search # noqa: E402
|
||||
from plugins import readers, search # noqa: E402
|
||||
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
||||
from plugins.search import SearchSource # noqa: E402
|
||||
from app.config import SearchSeed # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
@@ -111,246 +114,129 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) ->
|
||||
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- parsers
|
||||
# ----------------------------------------------------- read through the runner
|
||||
|
||||
def test_jobindex_reads_title_company_town_and_date() -> None:
|
||||
rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes())
|
||||
assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"]
|
||||
paid, bare, robot = rows
|
||||
# A paid ad: the company is its logo's alt text, the title its <h4>.
|
||||
assert paid["company"] == "DSV"
|
||||
assert paid["title"].startswith("Dangerous Goods Compliance Manager")
|
||||
assert paid["place"] == "Hedehusene"
|
||||
assert paid["country"] == "DK"
|
||||
assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23"
|
||||
assert paid["description"].startswith("Do you want to minimise")
|
||||
# A crawled ad with no company after the title's comma has none.
|
||||
assert bare["company"] == ""
|
||||
assert bare["place"] == "København"
|
||||
# A crawled ad names its company after the title.
|
||||
assert robot["title"] == "Marketing Content Creator"
|
||||
assert robot["company"] == "Clerk.IO ApS"
|
||||
def _web(monkeypatch: pytest.MonkeyPatch, answer: Any) -> list[str]:
|
||||
"""`_runner.fetch` answering `answer(url)`, remembering the URLs asked."""
|
||||
asked: list[str] = []
|
||||
|
||||
def fake_fetch(client: object, url: str, allow: Any, **kwargs: Any) -> bytes:
|
||||
assert allow(urlsplit(url).hostname), url
|
||||
asked.append(url)
|
||||
return answer(url)
|
||||
|
||||
monkeypatch.setattr(_runner, "fetch", fake_fetch)
|
||||
monkeypatch.setattr(search, "PAGE_DELAY", 0)
|
||||
return asked
|
||||
|
||||
|
||||
def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None:
|
||||
assert search._remote("Birkerød, Hybrid position") == "hybrid"
|
||||
assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
|
||||
assert search._remote("Remote, Danmark") == "remote"
|
||||
assert search._remote("Hedehusene") is None
|
||||
def _jobindex_page(url: str) -> bytes:
|
||||
"""A full Jobindex page (20 items) whatever page is asked."""
|
||||
page = int(parse_qs(urlsplit(url).query)["page"][0])
|
||||
items = "".join(f"<item><title>Role {page}-{n}, Acme</title>"
|
||||
f"<link>https://www.jobindex.dk/vis-job/r{page}x{n}</link>"
|
||||
f"<description><p>Teaser {n}</p></description></item>"
|
||||
for n in range(20))
|
||||
return f"<rss><channel><title>Jobindex</title>{items}</channel></rss>".encode()
|
||||
|
||||
|
||||
def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None:
|
||||
rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de")
|
||||
assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused
|
||||
first, predicted = rows
|
||||
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
||||
assert first["description"] == "Build our backend services in Python & Go."
|
||||
assert first["company"] == "Acme GmbH"
|
||||
assert first["place"] == "Berlin, Deutschland"
|
||||
assert first["country"] == "DE"
|
||||
assert first["remote"] == "remote"
|
||||
assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR")
|
||||
assert (predicted["salary_min"], predicted["currency"]) == (None, "")
|
||||
@pytest.mark.parametrize(("pages", "asked"), [(None, 5), (2, 2), (30, 10), ("x", 5)])
|
||||
def test_the_saved_pages_param_bounds_the_pages_read(
|
||||
monkeypatch: pytest.MonkeyPatch, pages: Any, asked: int) -> None:
|
||||
urls = _web(monkeypatch, _jobindex_page)
|
||||
params: dict[str, Any] = {"q": "python", "area": 1} | ({} if pages is None else
|
||||
{"pages": pages})
|
||||
rows, complete = search.run_search("jobindex:cph", params)
|
||||
assert len(urls) == asked and len(rows) == 20 * asked
|
||||
assert not complete # Jobindex states no total: never complete
|
||||
query = parse_qs(urlsplit(urls[0]).query)
|
||||
assert (query["q"], query["geoareaid"]) == (["python"], ["1"])
|
||||
|
||||
|
||||
def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
||||
with pytest.raises(ValueError, match="app_id"):
|
||||
search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
pages: list[int] = []
|
||||
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
pages.append(params["page"])
|
||||
return b"short" if params["page"] == 3 else b"full"
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setattr(search, "_jobindex_page",
|
||||
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
|
||||
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
|
||||
|
||||
# Stopped by the page limit with every page full: not complete.
|
||||
pages.clear()
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert (pages, complete) == ([1, 2], False)
|
||||
|
||||
|
||||
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
|
||||
def test_a_jobindex_role_hands_its_teaser_over_in_raw_not_in_the_body(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A full page whose parser dropped one item still has 20 items, so the
|
||||
search reads on instead of calling itself complete at 19 rows."""
|
||||
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
||||
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
|
||||
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
|
||||
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert not complete
|
||||
_web(monkeypatch, lambda url: (FIXTURES / "jobindex.xml").read_bytes())
|
||||
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
|
||||
"params": {"q": "x", "pages": 1}}]})
|
||||
items = list(source.fetch())
|
||||
paid = items[0]
|
||||
assert paid.source_id == "jobindex:h1701165"
|
||||
assert (paid.body, paid.excerpt) == ("", "")
|
||||
assert paid.raw["teaser"].startswith("Do you want to minimise")
|
||||
assert paid.raw["company"] == "DSV" and paid.raw["country"] == "DK"
|
||||
assert paid.raw["places"] == [{"text": "Hedehusene", "remote": None}]
|
||||
assert paid.raw["place"] == "Hedehusene"
|
||||
assert (paid.raw["provider"], paid.raw["search"]) == ("jobindex", "jobindex:cph")
|
||||
assert source.state["search:complete:jobindex:cph"] == ""
|
||||
|
||||
|
||||
def test_a_jobindex_answer_without_a_channel_raises() -> None:
|
||||
with pytest.raises(ValueError, match="channel"):
|
||||
search.parse_jobindex(b"<rss version='2.0'></rss>")
|
||||
|
||||
|
||||
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
|
||||
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
|
||||
asked: list[int] = []
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
page = int(url.rsplit("/", 1)[1])
|
||||
asked.append(page)
|
||||
start = (page - 1) * search.ADZUNA_PAGE
|
||||
def _adzuna(count: int | None, stock: int = 60) -> Any:
|
||||
def answer(url: str) -> bytes:
|
||||
page = int(urlsplit(url).path.rsplit("/", 1)[1])
|
||||
start = (page - 1) * 50
|
||||
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
|
||||
for n in range(start, min(start + 50, stock))]
|
||||
body: dict[str, Any] = {"results": results}
|
||||
if count is not None:
|
||||
body["count"] = count
|
||||
return json.dumps(body).encode()
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setenv("ADZUNA_APP_ID", "id")
|
||||
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
|
||||
return asked
|
||||
return answer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
|
||||
def test_a_search_without_its_key_is_refused_before_any_request(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
||||
urls = _web(monkeypatch, _adzuna(60))
|
||||
with pytest.raises(ValueError, match="app_id"):
|
||||
search.run_search("adzuna:berlin", {"q": "x", "country": "de"}, {})
|
||||
assert urls == []
|
||||
|
||||
|
||||
def test_the_key_comes_from_the_config_before_the_environment(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("ADZUNA_APP_ID", "env-id")
|
||||
monkeypatch.setenv("ADZUNA_APP_KEY", "env-key")
|
||||
urls = _web(monkeypatch, _adzuna(60))
|
||||
search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
|
||||
{"adzuna": {"app_id": "cfg-id"}})
|
||||
query = parse_qs(urlsplit(urls[0]).query)
|
||||
assert (query["app_id"], query["app_key"]) == (["cfg-id"], ["env-key"])
|
||||
assert urlsplit(urls[0]).path == "/v1/api/jobs/de/search/1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False)])
|
||||
def test_an_adzuna_search_is_complete_when_it_read_its_count(
|
||||
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
|
||||
_adzuna(monkeypatch, count)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
monkeypatch: pytest.MonkeyPatch, count: int, complete: bool) -> None:
|
||||
_web(monkeypatch, _adzuna(count))
|
||||
rows, done = search.run_search("adzuna:berlin", {"q": "x", "country": "de"},
|
||||
{"adzuna": {"app_id": "i", "app_key": "k"}})
|
||||
assert (len(rows), done) == (60, complete)
|
||||
assert rows[0]["place"] == ""
|
||||
|
||||
|
||||
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
|
||||
|
||||
|
||||
#: Two hits of a live `/search?q=data science®ion=CaRE_1nn_cSU` (2026-09-29),
|
||||
#: trimmed to the fields the parser reads, and a third the way an ad with no
|
||||
#: municipality and a remote headline comes.
|
||||
PLATSBANKEN = {
|
||||
"total": {"value": 32},
|
||||
"hits": [
|
||||
{"id": "31457942", "headline": "Postdoctoral Researcher",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942",
|
||||
"publication_date": "2026-09-09T15:47:27", "application_deadline": "2026-10-07T23:59:59",
|
||||
"description": {"text": "The European Spallation Source (ESS) is one of the largest.",
|
||||
"text_formatted": "<p>The European Spallation Source (ESS)</p>"},
|
||||
"employer": {"name": "European Spallation Source Eric"},
|
||||
"application_details": {"url": "https://europeanspallationsource.se/careers"},
|
||||
"workplace_model": {"concept_id": "ZxwM_gtz_n61", "label": "Arbete på plats"},
|
||||
"workplace_address": {"municipality": "Lund", "region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
|
||||
{"id": "31435504", "headline": "Postdoctoral Fellow in Robotics and Computer Vision",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504",
|
||||
"publication_date": "2026-09-03T15:46:06",
|
||||
"description": {"text": "Robotics at ESS."},
|
||||
"employer": {"name": "European Spallation Source Eric"},
|
||||
"workplace_model": {"concept_id": None, "label": None},
|
||||
"workplace_address": {"municipality": None, "region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU", "country": "Sverige"}},
|
||||
{"id": "31400001", "headline": "Data scientist, distansarbete möjligt",
|
||||
"webpage_url": "javascript:alert(1)", "workplace_address": {}},
|
||||
{"id": "31400002", "headline": "Data scientist, distansarbete möjligt",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002",
|
||||
"employer": {"name": None}, "workplace_address": {}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_platsbanken_reads_title_company_place_and_date() -> None:
|
||||
rows = search.parse_platsbanken(json.dumps(PLATSBANKEN).encode())
|
||||
assert [r["id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
||||
first, no_town, remote = rows
|
||||
assert first["title"] == "Postdoctoral Researcher"
|
||||
assert first["company"] == "European Spallation Source Eric"
|
||||
assert first["place"] == "Lund"
|
||||
assert first["country"] == "SE"
|
||||
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
|
||||
assert first["posted"] is not None and first["posted"].date().isoformat() == "2026-09-09"
|
||||
assert first["description"].startswith("The European Spallation Source")
|
||||
assert first["remote"] is None
|
||||
assert no_town["place"] == "Skåne län" # the region when there is no municipality
|
||||
assert remote["remote"] == "remote"
|
||||
assert (remote["company"], remote["place"], remote["posted"]) == ("", "", None)
|
||||
|
||||
|
||||
def _platsbanken(monkeypatch: pytest.MonkeyPatch, total: int | None,
|
||||
stock: int = 250) -> list[dict[str, Any]]:
|
||||
"""Platsbanken stubbed to `stock` results, `PLATSBANKEN_PAGE` a request,
|
||||
stating `total`. Returns the params of each request."""
|
||||
asked: list[dict[str, Any]] = []
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
asked.append(params)
|
||||
start = params["offset"]
|
||||
hits = [{"id": str(n), "headline": f"Role {n}",
|
||||
"webpage_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + params["limit"], stock))]
|
||||
body: dict[str, Any] = {"hits": hits}
|
||||
if total is not None:
|
||||
body["total"] = {"value": total}
|
||||
return json.dumps(body).encode()
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
return asked
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("total", "pages", "found", "complete"), [
|
||||
(250, 5, 250, True), # a short third page ends it
|
||||
(300, 5, 250, False), # the stated total is more than there was
|
||||
(250, 2, 200, False), # stopped by the page limit
|
||||
(None, 5, 250, True), # no total: a short page is the end
|
||||
])
|
||||
def test_a_platsbanken_search_is_complete_when_it_read_its_total(
|
||||
monkeypatch: pytest.MonkeyPatch, total: int | None, pages: int, found: int,
|
||||
complete: bool) -> None:
|
||||
_platsbanken(monkeypatch, total)
|
||||
rows, done = search.search_platsbanken(None, {"q": "x", "pages": pages}, {}) # type: ignore[arg-type]
|
||||
assert (len(rows), done) == (found, complete)
|
||||
|
||||
|
||||
def test_a_platsbanken_search_stops_at_its_total_and_sends_region_and_offset(
|
||||
def test_a_platsbanken_search_sends_region_and_offset_and_stops_at_its_total(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 200, stock=200)
|
||||
rows, done = search.search_platsbanken(
|
||||
None, {"q": "data", "region": "CaRE_1nn_cSU"}, {}) # type: ignore[arg-type]
|
||||
def answer(url: str) -> bytes:
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
start = int(query["offset"][0])
|
||||
hits = [{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}",
|
||||
"workplace_addresses": [{"municipality": "Lund"}, {"municipality": "Malmö"}]}
|
||||
for n in range(start, min(start + 100, 200))]
|
||||
return json.dumps({"total": {"value": 200}, "hits": hits}).encode()
|
||||
|
||||
urls = _web(monkeypatch, answer)
|
||||
rows, done = search.run_search("platsbanken:skane", {"q": "data", "region": "CaRE_1nn_cSU"})
|
||||
assert (len(rows), done) == (200, True)
|
||||
assert [p["offset"] for p in asked] == [0, 100]
|
||||
assert asked[0]["q"] == "data" and asked[0]["region"] == "CaRE_1nn_cSU"
|
||||
assert asked[0]["limit"] == search.PLATSBANKEN_PAGE
|
||||
offsets = [parse_qs(urlsplit(u).query)["offset"] for u in urls]
|
||||
assert offsets == [["0"], ["100"]]
|
||||
assert parse_qs(urlsplit(urls[0]).query)["region"] == ["CaRE_1nn_cSU"]
|
||||
assert rows[0]["place"] == "Lund; Malmö"
|
||||
|
||||
|
||||
def test_a_platsbanken_search_without_a_region_sends_none(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 5, stock=5)
|
||||
search.search_platsbanken(None, {"q": "x"}, {}) # type: ignore[arg-type]
|
||||
assert "region" not in asked[0]
|
||||
|
||||
|
||||
def test_a_platsbanken_search_never_asks_past_the_offset_limit(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _platsbanken(monkeypatch, 5000, stock=5000)
|
||||
monkeypatch.setattr(search, "MAX_PAGES", 30)
|
||||
_rows, done = search.search_platsbanken(None, {"q": "x", "pages": 30}, {}) # type: ignore[arg-type]
|
||||
assert max(p["offset"] for p in asked) == search.PLATSBANKEN_MAX_OFFSET
|
||||
assert not done
|
||||
|
||||
|
||||
def test_a_platsbanken_search_refuses_an_unknown_region() -> None:
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
search.search_platsbanken(None, {"q": "x", "region": "nope"}, {}) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_platsbanken_regions_are_the_21_lan() -> None:
|
||||
assert len(search.PLATSBANKEN_REGIONS) == 21
|
||||
assert search.PLATSBANKEN_REGIONS["CaRE_1nn_cSU"] == "Skåne län"
|
||||
def test_an_unknown_provider_is_refused() -> None:
|
||||
with pytest.raises(ValueError, match="provider"):
|
||||
search.run_search("indeed:x", {"q": "x"})
|
||||
|
||||
|
||||
def test_provider_of() -> None:
|
||||
@@ -369,15 +255,27 @@ class FakeConfig:
|
||||
path = None
|
||||
|
||||
|
||||
def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
|
||||
def fake(client: object, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
|
||||
return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København",
|
||||
"country": "DK", "remote": None, "url": f"https://x.example/{i}",
|
||||
"posted": None, "description": "", "salary_min": None,
|
||||
"salary_max": None, "currency": ""} for i in ids], complete
|
||||
def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
|
||||
"""A provider `fake` that lists `ids` in one page and states a total of
|
||||
as many, or of one more when the search is to be incomplete."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL = "fake", "Fake"
|
||||
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
|
||||
places="one", fields=Fields(country="stated"), robots="test")
|
||||
module.allowed = lambda slug: lambda host: host == "search.fake.example"
|
||||
|
||||
monkeypatch.setitem(search._SEARCHES, "jobindex", fake)
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
assert ask.params["q"]
|
||||
return ids, [{"id": i} for i in ids], len(ids) + (0 if complete else 1), None
|
||||
|
||||
module.page = page
|
||||
module.rows = lambda decoded, slug: [
|
||||
{"source_id": i, "title": f"Engineer {i}", "company": "Acme",
|
||||
"url": f"https://x.example/{i}", "places": [{"text": "København", "remote": None}],
|
||||
"country": "DK", "remote": None, "posted": None, "description": ""}
|
||||
for i in decoded]
|
||||
assert not readers.problems(module, "fake")
|
||||
monkeypatch.setitem(readers.PROVIDERS, "fake", module)
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, Any]:
|
||||
@@ -388,36 +286,36 @@ def _pull(store: Store) -> dict[str, Any]:
|
||||
|
||||
def test_a_complete_search_closes_what_it_stopped_listing(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
|
||||
params={"q": "backend", "area": 1})
|
||||
_stub_search(monkeypatch, ["h1", "h2"], complete=True)
|
||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||
params={"q": "backend"})
|
||||
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
||||
_pull(store)
|
||||
roles = {r.source_id: r for r in store.roles()}
|
||||
assert set(roles) == {"jobindex:h1", "jobindex:h2"}
|
||||
assert roles["jobindex:h1"].country == "DK"
|
||||
assert roles["jobindex:h1"].subscription_id is not None
|
||||
assert set(roles) == {"fake:h1", "fake:h2"}
|
||||
assert roles["fake:h1"].country == "DK"
|
||||
assert roles["fake:h1"].subscription_id is not None
|
||||
|
||||
_stub_search(monkeypatch, ["h2"], complete=True)
|
||||
_fake_provider(monkeypatch, ["h2"], complete=True)
|
||||
result = _pull(store)
|
||||
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
||||
assert (result["searches"], result["closed"]) == (1, 1)
|
||||
|
||||
|
||||
def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale(
|
||||
def test_an_incomplete_search_closes_nothing_until_stale(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
|
||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||
params={"q": "backend"})
|
||||
_stub_search(monkeypatch, ["h1", "h2"], complete=False)
|
||||
_fake_provider(monkeypatch, ["h1", "h2"], complete=False)
|
||||
_pull(store)
|
||||
_stub_search(monkeypatch, ["h2"], complete=False)
|
||||
_fake_provider(monkeypatch, ["h2"], complete=False)
|
||||
_pull(store)
|
||||
assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"}
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
|
||||
|
||||
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,))
|
||||
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
|
||||
_pull(store)
|
||||
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
||||
|
||||
|
||||
def test_seed_searches_imports_once(store: Store) -> None:
|
||||
@@ -434,24 +332,32 @@ def test_seed_searches_imports_once(store: Store) -> None:
|
||||
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- cli
|
||||
|
||||
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_stub_search(monkeypatch, ["h1"], complete=True)
|
||||
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
|
||||
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
|
||||
"params": {"q": "x"}}]})
|
||||
items = list(source.fetch())
|
||||
assert [i.source_id for i in items] == ["jobindex:h1"]
|
||||
assert items[0].raw["search"] == "jobindex:cph"
|
||||
assert source.state["search:complete:jobindex:cph"] == "1"
|
||||
assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'
|
||||
assert [i.source_id for i in items] == ["fake:h1"]
|
||||
assert items[0].raw["search"] == "fake:cph"
|
||||
assert source.state["search:complete:fake:cph"] == "1"
|
||||
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
||||
|
||||
|
||||
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
|
||||
source = SearchSource({"searches": [{"url": "adzuna:berlin", "label": "Berlin",
|
||||
"params": {"q": "x", "country": "de"}}]})
|
||||
assert list(source.fetch()) == []
|
||||
assert "search:ok:adzuna:berlin" not in source.state
|
||||
|
||||
|
||||
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen",
|
||||
board="jobindex", params={"q": "backend"})
|
||||
_stub_search(monkeypatch, ["h1"], complete=False)
|
||||
sub = store.add_subscription("search", "fake:cph", label="Copenhagen",
|
||||
board="fake", params={"q": "backend"})
|
||||
_fake_provider(monkeypatch, ["h1"], complete=False)
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
|
||||
store.update_subscription(sub.id, subscribed=False, fetch=False)
|
||||
|
||||
Reference in new issue
Block a user