Files
jobboard/tests/test_fetch_terms.py
T
WolfandClaude Opus 5.5 57cac5f896 Add migration 27 and read a remote role's area in the chips, the gates and the rows
roles.remote_model, roles.remote_where_model and role_remote, with a stored remote no host word gives kept as the host's word. The remote tokens filter over role_remote, remote_from and a require gate on country read a remote role through its area, extract waits while remote_where_model is NULL and writes the model's remote outright, and rows carry remote_where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 16:00:06 +02:00

477 lines
22 KiB
Python

"""Fetch terms (see "Fetch terms" in `docs/decisions.md`): an agent's words that
narrow what a big board read with the wide reach sends. The terms of a pull,
where they apply, how the runner makes series of them, the request each host
sends (the parameters `docs/filter-survey.md` verified), the guards on a
term read, and the agent editor's API. No network."""
from __future__ import annotations
import json
import sqlite3
import sys
from collections.abc import Iterator
from pathlib import Path
from typing import Any, ClassVar
from urllib.parse import parse_qs, unquote, urlsplit
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from pipeline import Tally # noqa: E402
from app.config import Config, ServerConfig # noqa: E402
from app.jobs import pull, reach # noqa: E402
from app.main import create_app # noqa: E402
from app.models import Agent, Gate # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import ( # noqa: E402
Ask, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken,
smartrecruiters, successfactors, workday,
)
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
EUROPE = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS",
"LI", "NO", "GB", "CH", "EU"]
TERMS = ["engineer", "data"]
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[object]] = []
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def _wolf(store: Store, terms: list[str] | None = None, name: str = "Wolf") -> int:
"""Agent 2 of 2026-09-30: Europe required; Denmark, Sweden, remote or a
town in Skåne or Denmark."""
gates = [Gate(field="country", values=EUROPE, mode="require"),
Gate(field="country", values=["DK", "SE"], mode="any"),
Gate(field="remote", values=["remote"], mode="any"),
Gate(field="place", values=["Malmö", "copenhagen"], mode="any")]
return store.save_agent(name, gates=gates, fetch_terms=terms).id
def _board(store: Store, target: str, company: str, size: int | None = 1000,
params: dict[str, object] | None = None, tagged: bool = True) -> None:
store.add_subscription("ats", target, label=company, company=company,
board=target.partition(":")[0], params=params or {})
store.add_company(company)
if size is not None:
store.state[f"ats:size:{target}"] = str(size)
if tagged:
from app.models import company_key
store.set_all_countries(company_key(company), True)
def _sub(store: Store, target: str) -> Any:
return next(s for s in store.subscriptions(source="ats") if s.target == target)
# ----------------------------------------------------------- the terms of a pull
def _agent(terms: list[str]) -> Agent:
return Agent(id=1, name="a", created_at="", fetch_terms=terms)
def test_the_terms_are_the_union_over_the_agents_one_of_each() -> None:
found = reach.fetch_terms([_agent(["engineer", "Data"]), _agent(["data", "head of"])])
assert found == ("Data", "engineer", "head of")
def test_an_agent_without_terms_means_there_are_none() -> None:
assert reach.fetch_terms([_agent(TERMS), _agent([])]) == ()
assert reach.fetch_terms([]) == ()
def test_only_enabled_agents_count(store: Store) -> None:
_wolf(store, TERMS)
off = store.save_agent("Off", fetch_terms=[])
store.save_agent(off.name, agent_id=off.id, enabled=False)
assert reach.plan(store).terms == ("data", "engineer")
store.save_agent("On", fetch_terms=[])
assert reach.plan(store).terms == ()
# ------------------------------------------------------------ where they apply
def test_a_tagged_board_carries_the_terms_and_the_wide_mark(store: Store) -> None:
_wolf(store, TERMS)
_board(store, "phenom:www.careers.philips.com/global/en", "Philips")
found = reach.plan(store).for_subscription(
_sub(store, "phenom:www.careers.philips.com/global/en"))
assert found is not None
assert found["terms"] == ["data", "engineer"] and found["wide"] is True
assert {"DE", "DK", "FR"} <= set(found["countries"])
def test_the_narrow_reach_takes_no_terms(store: Store) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub))
assert ask == Ask(countries=("DK", "SE"))
assert reach.line(plan, sub) == "follows agents: DK, SE · all roles"
@pytest.mark.parametrize("size, params", [
(299, {}), (5000, {"countries": ["DE"]}),
], ids=["small", "own reach"])
def test_small_unknown_overridden_and_switched_off_boards_take_no_terms(
store: Store, size: int | None, params: dict[str, object]) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:BoschGroup", "Bosch", size, params)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
assert _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)).terms == ()
def test_the_agents_reach_takes_terms_once_it_is_too_many_countries_to_ask(
store: Store) -> None:
_wolf(store, TERMS)
store.save_agent("Nordics and more", fetch_terms=["product"], gates=[Gate(
field="country", values=["DE", "FR", "NL", "BE", "AT", "PL", "NO", "FI"], mode="any")])
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub))
assert ask == Ask(terms=("data", "engineer", "product"))
assert reach.line(plan, sub) == \
"all places, too many countries to ask · follows agents' terms: data, engineer, product"
# All roles drops them there too; the countries stay dropped.
store.set_all_roles("bosch", True)
plan = reach.plan(store)
assert _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)) == Ask()
assert reach.line(plan, sub) == "too many countries to ask: read whole"
# ------------------------------------------------------ terms times the reach
def test_terms_and_a_list_of_countries_combine_under_the_cap() -> None:
wide = {"countries": EUROPE[:30], "remote": True, "terms": TERMS, "wide": True}
ask = _runner.ask_for(phenom, {}, wide)
assert ask.countries == tuple(EUROPE[:30]) and ask.terms == ("engineer", "data")
asks = _runner.series(phenom, ask)
assert [(a.query, len(a.countries)) for a in asks] == [("engineer", 30), ("data", 30)]
def test_terms_and_countries_one_a_request_combine_as_their_product() -> None:
wide = {"countries": ["DK", "SE"], "terms": TERMS, "wide": True}
ask = _runner.ask_for(smartrecruiters, {}, wide)
asks = _runner.series(smartrecruiters, ask)
assert [(a.query, a.countries) for a in asks] == [
("engineer", ("DK",)), ("engineer", ("SE",)), ("data", ("DK",)), ("data", ("SE",))]
def test_countries_and_remote_are_two_series_a_term() -> None:
ask = _runner.ask_for(workday, {}, {"countries": ["DK"], "remote": True, "terms": TERMS,
"wide": True})
facets = {"country": {"DK": "Location=dk"}, "remote": {"remote": "remoteType=r"}}
asks = _runner.series(workday, ask, facets)
assert [(a.query, a.countries, a.remote) for a in asks] == [
("engineer", ("Location=dk",), None), ("engineer", (), True),
("data", ("Location=dk",), None), ("data", (), True)]
def test_over_the_cap_the_places_go_first_and_the_terms_stay() -> None:
five = ["engineer", "data", "product", "head of", "architect"]
# Five terms times (Europe + the remote series) is ten series: over eight.
ask = _runner.ask_for(workday, {}, {"countries": EUROPE, "remote": True, "terms": five,
"wide": True})
assert ask == Ask(terms=tuple(five))
assert [a.query for a in _runner.series(workday, ask)] == five
# Three are six series: the places stay.
ask = _runner.ask_for(workday, {}, {"countries": EUROPE, "remote": True,
"terms": five[:3], "wide": True})
assert ask.countries and ask.remote and ask.terms == tuple(five[:3])
def test_more_terms_than_the_cap_read_the_board_unfiltered() -> None:
nine = [f"word{n}" for n in range(9)]
assert _runner.MAX_SERIES == 8
assert _runner.ask_for(phenom, {}, {"countries": ["DK"], "terms": nine, "wide": True}) == Ask()
def test_a_one_host_drops_europe_and_keeps_the_terms() -> None:
ask = _runner.ask_for(smartrecruiters, {}, {"countries": EUROPE, "remote": True,
"terms": TERMS, "wide": True})
assert ask == Ask(terms=("engineer", "data"))
def test_terms_need_the_wide_mark_or_a_dropped_reach() -> None:
assert _runner.ask_for(phenom, {}, {"countries": ["DK", "SE"], "terms": TERMS}).terms == ()
assert _runner.ask_for(phenom, {}, {"terms": TERMS}).terms == ()
assert _runner.ask_for(phenom, {}, {"terms": TERMS, "wide": True}).terms == ("engineer", "data")
def test_hosts_without_a_query_and_the_search_providers_take_no_terms() -> None:
wide = {"countries": ["DK"], "terms": TERMS, "wide": True}
assert _runner.ask_for(greenhouse, {}, wide) == Ask()
for provider in (jobindex, platsbanken):
ask = _runner.ask_for(provider, {"q": "python"}, wide)
assert ask.terms == () and ask.query == "python"
assert adzuna.READS.scope == "search"
def test_a_boards_own_query_wins_over_the_terms() -> None:
ask = _runner.ask_for(smartrecruiters, {"q": "sap"}, {"terms": TERMS, "wide": True})
assert (ask.query, ask.terms) == ("sap", ())
assert _runner.narrowed(smartrecruiters, ask)
# ------------------------------------------------------------ the line
def test_the_reach_line_counts_the_terms(store: Store) -> None:
_wolf(store, ["engineer", "data", "head of"])
_board(store, "phenom:www.careers.philips.com/global/en", "Philips")
_board(store, "smartrecruiters:BoschGroup", "Bosch")
plan = reach.plan(store)
philips = _sub(store, "phenom:www.careers.philips.com/global/en")
assert reach.line(plan, philips) == \
"all countries (Europe) · follows agents' terms: data, engineer, head of"
assert reach.line(plan, _sub(store, "smartrecruiters:BoschGroup")) == \
"all places, too many countries to ask · follows agents' terms: data, engineer, head of"
def test_one_term_is_one_term(store: Store) -> None:
_wolf(store, ["engineer"])
_board(store, "smartrecruiters:BoschGroup", "Bosch")
assert reach.line(reach.plan(store), _sub(store, "smartrecruiters:BoschGroup")) == \
"all places, too many countries to ask · follows agents' terms: engineer"
# ------------------------------------------------------ each host's request
class Recorder:
"""A `get` that answers `body` to every request and keeps what was asked."""
def __init__(self, body: bytes) -> None:
self.body = body
self.asked: list[tuple[str, dict[str, Any] | None]] = []
def __call__(self, url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
self.asked.append((url, payload))
return self.body
def _query(url: str) -> dict[str, list[str]]:
return parse_qs(urlsplit(url).query)
def test_smartrecruiters_sends_the_term_as_q_beside_the_country() -> None:
get = Recorder(b'{"content": [], "totalFound": 0}')
smartrecruiters.page(get, "BoschGroup", None, Ask(countries=("DK",), query="engineer"))
smartrecruiters.page(get, "BoschGroup", None, Ask(query="head of"))
smartrecruiters.page(get, "BoschGroup", None, Ask())
assert _query(get.asked[0][0]) == {"offset": ["0"], "limit": ["100"], "country": ["dk"],
"q": ["engineer"]}
assert "q=head%20of" in get.asked[1][0]
assert "q" not in _query(get.asked[2][0])
def test_successfactors_sends_the_term_as_the_forms_q() -> None:
get = Recorder((FIXTURES / "successfactors-table.html").read_bytes())
successfactors.page(get, "careers.vestas.com", None, Ask(countries=("DK",), query="engineer"))
successfactors.page(get, "careers.vestas.com", None, Ask())
assert get.asked[0][0] == ("https://careers.vestas.com/search/?q=engineer&locationsearch="
"&optionsFacetsDD_country=DK&startrow=0")
assert get.asked[1][0].startswith("https://careers.vestas.com/search/?q=&locationsearch=")
def test_workday_sends_the_term_as_search_text() -> None:
get = Recorder(b'{"total": 0, "jobPostings": []}')
workday.page(get, "lego.wd103/LEGO_External", None,
Ask(countries=("Location=dk",), query="engineer"))
workday.page(get, "lego.wd103/LEGO_External", None, Ask())
assert get.asked[0][0] == "https://lego.wd103.myworkdayjobs.com/wday/cxs/lego/LEGO_External/jobs"
assert get.asked[0][1] == {"appliedFacets": {"Location": ["dk"]}, "limit": 20, "offset": 0,
"searchText": "engineer"}
assert get.asked[1][1]["searchText"] == ""
def test_phenom_sends_the_term_as_keywords() -> None:
get = Recorder((FIXTURES / "phenom.json").read_bytes())
phenom.page(get, "www.careers.philips.com/global/en", None,
Ask(countries=("NL",), query="engineer"))
phenom.page(get, "www.careers.philips.com/global/en", None, Ask())
assert get.asked[0][0] == "https://www.careers.philips.com/widgets"
body = get.asked[0][1]
assert body["keywords"] == "engineer"
assert body["selected_fields"] == {"country": ["Netherlands"]}
assert get.asked[1][1]["keywords"] == ""
def test_eightfold_sends_the_term_as_query() -> None:
get = Recorder(b'{"status": 200, "data": {"positions": [], "count": 0}}')
eightfold.page(get, "jobs.ericsson.com/ericsson.com", None,
Ask(countries=("SE",), query="software"))
eightfold.page(get, "jobs.ericsson.com/ericsson.com", None, Ask())
assert _query(get.asked[0][0]) == {"domain": ["ericsson.com"], "sort_by": ["timestamp"],
"start": ["0"], "location": ["Sweden"],
"query": ["software"]}
assert "query" not in _query(get.asked[1][0])
def test_oracle_sends_the_term_as_the_finders_keyword() -> None:
pod = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com/CX_1"
get = Recorder(b'{"items": [{"requisitionList": [], "TotalJobsCount": 0}]}')
oracle.page(get, pod, None, Ask(countries=("300000000471640",), query="engineer"))
oracle.page(get, pod, None, Ask(query="r&d, lab"))
finders = [unquote(_query(url)["finder"][0]) for url, _ in get.asked]
assert finders[0] == ("findReqs;siteNumber=CX_1,keyword=engineer,selectedLocationsFacet="
"300000000471640,limit=200,offset=0")
# A comma in a term stays inside the keyword.
assert "keyword=r%26d%2C%20lab," in get.asked[1][0]
# ------------------------------------------------------------- the guards
def test_a_term_read_is_narrowed_and_its_count_is_its_own(tmp_path: Path) -> None:
def get(url: str, **kwargs: Any) -> bytes:
q = _query(url).get("q", [""])[0]
ids = {"": [str(n) for n in range(40)], "engineer": ["1", "2"], "data": ["2", "3"]}[q]
return json.dumps({"totalFound": len(ids), "content": [
{"id": i, "name": f"Role {i}", "location": {"city": "Berlin", "country": "de"}}
for i in ids]}).encode()
state: dict[str, str] = {}
whole = _runner.read(state, "ats", "t", smartrecruiters, "Acme", get)
_runner.stamp(state, "ats", "t", "r", whole, whole.ids)
assert not whole.narrowed and len(whole.ids) == 40
# Three of forty is under a tenth, but a term read is not held to a whole read.
termed = _runner.read(state, "ats", "t", smartrecruiters, "Acme", get,
Ask(terms=("engineer", "data")))
assert termed.narrowed and not termed.filtered and termed.complete
assert termed.ids == ["1", "2", "3"]
_runner.stamp(state, "ats", "t", "r", termed, termed.ids)
assert state["ats:narrowed:t"] == "1"
# The same terms again, far fewer: that is a failure, held to its own count.
state["ats:count:t"] = json.dumps({"asks": termed.asks, "count": 100})
with pytest.raises(_runner.Suspect):
_runner.read(state, "ats", "t", smartrecruiters, "Acme", get,
Ask(terms=("engineer", "data")))
def _posting(job_id: str) -> dict[str, Any]:
return {"id": job_id, "name": f"Engineer {job_id}",
"location": {"city": "Berlin", "country": "de"}}
def test_a_term_read_writes_no_size_and_closes_nothing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, "smartrecruiters:bfl", "Black Forest Labs")
asked: list[str] = []
def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
asked.append(url)
q = _query(url).get("q", [""])[0]
ids = {"": ["1", "2", "9"], "engineer": ["1"], "data": ["2"]}[q]
return json.dumps({"totalFound": len(ids),
"content": [_posting(i) for i in ids]}).encode()
monkeypatch.setattr(ats, "_get", get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def run() -> dict[str, Any]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type]
for b in batches]
return pull.close_boards(FakeConfig(), store, # type: ignore[arg-type]
Tally(results=tuple(results)))
# No terms: Europe is too many countries for SmartRecruiters, so whole.
# A whole read writes no size either: the board keeps the one it has.
run()
assert [_query(u).get("q") for u in asked] == [None]
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
open_before = {r.source_id for r in store.roles(limit=50)}
assert open_before == {f"smartrecruiters:bfl:{n}" for n in ("1", "2", "9")}
agent = store.agents()[0]
store.save_agent(agent.name, gates=agent.gates, agent_id=agent.id, fetch_terms=TERMS)
asked.clear()
settled = run()
assert sorted(_query(u)["q"][0] for u in asked) == ["data", "engineer"]
assert all("country" not in _query(u) for u in asked)
assert store.state["ats:narrowed:smartrecruiters:bfl"] == "1"
assert store.state["ats:complete:smartrecruiters:bfl"] == "1"
# Role 9 is outside the terms, not gone: it stays open, and the size stays.
assert {r.source_id for r in store.roles(limit=50)} == open_before
assert settled["closed"] == 0
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
# ------------------------------------------------------- storage and the API
def test_terms_are_stored_normalised_and_kept_by_a_save_without_them(store: Store) -> None:
agent = store.save_agent("Wolf", fetch_terms=[" engineer ", "Engineer", "", "head of"])
assert agent.fetch_terms == ["engineer", "head of"]
kept = store.save_agent("Wolf", agent_id=agent.id, intent="Data leadership.")
assert kept.fetch_terms == ["engineer", "head of"]
cleared = store.save_agent("Wolf", agent_id=agent.id, fetch_terms=[])
assert cleared.fetch_terms == []
assert store.save_agent("New").fetch_terms == []
def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
path = tmp_path / "old.db"
Store(path).save_agent("Wolf")
with sqlite3.connect(path) as conn:
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
conn.execute("PRAGMA user_version = 22")
migrated = Store(path)
assert migrated.migrations_applied == [23, 24, 25, 26, 27]
assert migrated.agents()[0].fetch_terms == []
@pytest.fixture
def app(tmp_path: Path) -> FastAPI:
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
@pytest.fixture
def client(app: FastAPI) -> Iterator[TestClient]:
with TestClient(app) as c:
yield c
def test_the_agent_editor_saves_and_lists_the_terms(client: TestClient) -> None:
res = client.post("/api/agents", json={"name": "Wolf",
"fetch_terms": ["engineer", " data ", "DATA"]})
assert res.status_code == 200 and res.json()["fetch_terms"] == ["engineer", "data"]
agent_id = res.json()["id"]
[listed] = client.get("/api/agents").json()
assert listed["fetch_terms"] == ["engineer", "data"]
# The card's switch posts the row back as it came: the terms stay.
res = client.post("/api/agents", json=listed | {"enabled": False})
assert res.json()["fetch_terms"] == ["engineer", "data"]
# A body without the key keeps them; an empty list clears them.
res = client.post("/api/agents", json={"id": agent_id, "name": "Wolf"})
assert res.json()["fetch_terms"] == ["engineer", "data"]
res = client.post("/api/agents", json={"id": agent_id, "name": "Wolf", "fetch_terms": []})
assert res.json()["fetch_terms"] == []
def test_the_companies_list_says_the_terms(client: TestClient, app: FastAPI) -> None:
store: Store = app.state.store
_wolf(store, ["engineer", "data", "head of"])
_board(store, "smartrecruiters:BoschGroup", "Bosch")
row = next(c for c in client.get("/api/companies").json() if c["key"] == "bosch")
assert row["reach"] == \
"all places, too many countries to ask · follows agents' terms: data, engineer, head of"