Send fetch terms to every big board whose host takes one query, and on a series overflow let a narrow read keep its places and drop the terms

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-04 14:50:33 +02:00
1 parent 060ccf66e6
commit 5fe5a31747
6 files changed
+159 -62

No files matched your search

+97 -11
View File
@@ -1,5 +1,5 @@
"""Fetch terms (see "Fetch terms" in `docs/decisions.md`): an agent's words that
narrow what a big board read with the wide reach sends. The terms of a pull,
narrow what a big board sends, on the narrow reach and on the wide one. The terms of a pull,
where they apply, how the runner makes series of them, the request each host
sends (the parameters `docs/filter-survey.md` verified), the guards on a
term read, and the agent editor's API. No network."""
@@ -9,6 +9,7 @@ from __future__ import annotations
import json
import sqlite3
import sys
import types
from collections.abc import Iterator
from pathlib import Path
from typing import Any, ClassVar
@@ -30,7 +31,7 @@ from app.models import Agent, Gate # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import ( # noqa: E402
Ask, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken,
Ask, Filters, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken,
smartrecruiters, successfactors, workday,
)
@@ -116,14 +117,39 @@ def test_a_tagged_board_carries_the_terms_and_the_wide_mark(store: Store) -> Non
assert {"DE", "DK", "FR"} <= set(found["countries"])
def test_the_narrow_reach_takes_no_terms(store: Store) -> None:
def test_the_narrow_reach_takes_the_terms_too(store: Store) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
found = plan.for_subscription(sub)
assert found is not None and "wide" not in found
ask = _runner.ask_for(smartrecruiters, sub.params, found)
assert ask == Ask(countries=("DK", "SE"), terms=("data", "engineer"))
assert reach.line(plan, sub) == ("follows agents: DK, SE · remote not asked on "
"SmartRecruiters · follows agents' terms: data, engineer")
def test_a_company_set_to_all_roles_takes_no_terms_on_the_narrow_reach(store: Store) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
store.set_all_roles("bosch", True)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub))
assert ask == Ask(countries=("DK", "SE"))
assert reach.line(plan, sub) == "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"
assert reach.line(plan, sub) == \
"follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"
def test_an_agent_that_takes_any_place_still_carries_the_terms(store: Store) -> None:
store.save_agent("Anywhere", fetch_terms=["engineer"])
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
assert _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)) == \
Ask(terms=("engineer",))
assert reach.line(plan, sub) == "an agent takes any place · follows agents' terms: engineer"
@pytest.mark.parametrize("size, params", [
@@ -210,10 +236,44 @@ def test_a_one_host_drops_europe_and_keeps_the_terms() -> None:
assert ask == Ask(terms=("engineer", "data"))
def test_terms_need_the_wide_mark_or_a_dropped_reach() -> None:
assert _runner.ask_for(phenom, {}, {"countries": ["DK", "SE"], "terms": TERMS}).terms == ()
assert _runner.ask_for(phenom, {}, {"terms": TERMS}).terms == ()
assert _runner.ask_for(phenom, {}, {"terms": TERMS, "wide": True}).terms == ("engineer", "data")
def test_terms_go_with_a_narrow_or_a_wide_read() -> None:
for given in ({"countries": ["DK", "SE"]}, {}, {"wide": True}):
ask = _runner.ask_for(phenom, {}, {"terms": TERMS} | given)
assert ask.terms == ("engineer", "data")
@pytest.fixture
def one_host() -> types.ModuleType:
"""A board host that filters by country one a request, by remote, and by
query: no shipped reader declares all three."""
reads = smartrecruiters.READS.model_copy(update={
"filters": Filters(country="one", remote="one", query="one")})
module = types.ModuleType("one_host")
module.READS = reads # type: ignore[attr-defined]
return module
def test_over_the_cap_a_narrow_read_keeps_the_places_and_drops_the_terms(
one_host: types.ModuleType) -> None:
three = ["engineer", "data", "product"]
# Three terms times (DK, SE, and the remote series) is nine series: over eight.
narrow = {"countries": ["DK", "SE"], "remote": True, "terms": three}
ask = _runner.ask_for(one_host, {}, narrow)
assert ask.countries == ("DK", "SE") and ask.remote is True and ask.terms == ()
assert not _runner.narrowed(one_host, ask)
# The same reach read wide keeps the terms and drops the places.
wide = _runner.ask_for(one_host, {}, narrow | {"wide": True})
assert wide == Ask(terms=tuple(three))
assert _runner.narrowed(one_host, wide)
# Two terms are six series: both stay.
ask = _runner.ask_for(one_host, {}, narrow | {"terms": three[:2]})
assert ask.countries == ("DK", "SE") and ask.remote is True and ask.terms == ("engineer", "data")
def test_a_narrow_read_over_the_cap_by_terms_alone_drops_them() -> None:
nine = [f"word{n}" for n in range(9)]
ask = _runner.ask_for(phenom, {}, {"countries": ["DK"], "terms": nine})
assert ask == Ask(countries=("DK",))
def test_hosts_without_a_query_and_the_search_providers_take_no_terms() -> None:
@@ -387,10 +447,10 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def run() -> dict[str, Any]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type]
batches = pull._boards(FakeConfig(), store)
results = [pull.run_batch(FakeConfig(), store, **b)
for b in batches]
return pull.close_boards(FakeConfig(), store, # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store,
Tally(results=tuple(results)))
# No terms: Europe is too many countries for SmartRecruiters, so whole.
@@ -415,6 +475,32 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
def test_a_narrow_term_read_asks_a_term_in_each_country_and_closes_nothing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:bfl", "Black Forest Labs", tagged=False)
asked: list[str] = []
def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
asked.append(url)
q = _query(url).get("q", [""])[0]
ids = {"engineer": ["1"], "data": ["2"]}[q]
return json.dumps({"totalFound": len(ids),
"content": [_posting(i) for i in ids]}).encode()
monkeypatch.setattr(ats, "_get", get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type]
for b in batches]
settled = pull.close_boards(FakeConfig(), store, # type: ignore[arg-type]
Tally(results=tuple(results)))
assert sorted((_query(u)["q"][0], _query(u)["country"][0]) for u in asked) == [
("data", "dk"), ("data", "se"), ("engineer", "dk"), ("engineer", "se")]
assert store.state["ats:narrowed:smartrecruiters:bfl"] == "1"
assert settled["closed"] == 0
# ------------------------------------------------------- storage and the API
def test_terms_are_stored_normalised_and_kept_by_a_save_without_them(store: Store) -> None: