Send fetch terms to every big board whose host takes one query, and on a series overflow let a narrow read keep its places and drop the terms

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-04 14:50:33 +02:00
1 parent 060ccf66e6
commit 5fe5a31747
6 files changed
+159 -62

No files matched your search

+97 -11
View File
@@ -1,5 +1,5 @@
"""Fetch terms (see "Fetch terms" in `docs/decisions.md`): an agent's words that
narrow what a big board read with the wide reach sends. The terms of a pull,
narrow what a big board sends, on the narrow reach and on the wide one. The terms of a pull,
where they apply, how the runner makes series of them, the request each host
sends (the parameters `docs/filter-survey.md` verified), the guards on a
term read, and the agent editor's API. No network."""
@@ -9,6 +9,7 @@ from __future__ import annotations
import json
import sqlite3
import sys
import types
from collections.abc import Iterator
from pathlib import Path
from typing import Any, ClassVar
@@ -30,7 +31,7 @@ from app.models import Agent, Gate # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import ( # noqa: E402
Ask, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken,
Ask, Filters, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken,
smartrecruiters, successfactors, workday,
)
@@ -116,14 +117,39 @@ def test_a_tagged_board_carries_the_terms_and_the_wide_mark(store: Store) -> Non
assert {"DE", "DK", "FR"} <= set(found["countries"])
def test_the_narrow_reach_takes_no_terms(store: Store) -> None:
def test_the_narrow_reach_takes_the_terms_too(store: Store) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
found = plan.for_subscription(sub)
assert found is not None and "wide" not in found
ask = _runner.ask_for(smartrecruiters, sub.params, found)
assert ask == Ask(countries=("DK", "SE"), terms=("data", "engineer"))
assert reach.line(plan, sub) == ("follows agents: DK, SE · remote not asked on "
"SmartRecruiters · follows agents' terms: data, engineer")
def test_a_company_set_to_all_roles_takes_no_terms_on_the_narrow_reach(store: Store) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
store.set_all_roles("bosch", True)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub))
assert ask == Ask(countries=("DK", "SE"))
assert reach.line(plan, sub) == "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"
assert reach.line(plan, sub) == \
"follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"
def test_an_agent_that_takes_any_place_still_carries_the_terms(store: Store) -> None:
store.save_agent("Anywhere", fetch_terms=["engineer"])
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
assert _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)) == \
Ask(terms=("engineer",))
assert reach.line(plan, sub) == "an agent takes any place · follows agents' terms: engineer"
@pytest.mark.parametrize("size, params", [
@@ -210,10 +236,44 @@ def test_a_one_host_drops_europe_and_keeps_the_terms() -> None:
assert ask == Ask(terms=("engineer", "data"))
def test_terms_need_the_wide_mark_or_a_dropped_reach() -> None:
assert _runner.ask_for(phenom, {}, {"countries": ["DK", "SE"], "terms": TERMS}).terms == ()
assert _runner.ask_for(phenom, {}, {"terms": TERMS}).terms == ()
assert _runner.ask_for(phenom, {}, {"terms": TERMS, "wide": True}).terms == ("engineer", "data")
def test_terms_go_with_a_narrow_or_a_wide_read() -> None:
for given in ({"countries": ["DK", "SE"]}, {}, {"wide": True}):
ask = _runner.ask_for(phenom, {}, {"terms": TERMS} | given)
assert ask.terms == ("engineer", "data")
@pytest.fixture
def one_host() -> types.ModuleType:
"""A board host that filters by country one a request, by remote, and by
query: no shipped reader declares all three."""
reads = smartrecruiters.READS.model_copy(update={
"filters": Filters(country="one", remote="one", query="one")})
module = types.ModuleType("one_host")
module.READS = reads # type: ignore[attr-defined]
return module
def test_over_the_cap_a_narrow_read_keeps_the_places_and_drops_the_terms(
one_host: types.ModuleType) -> None:
three = ["engineer", "data", "product"]
# Three terms times (DK, SE, and the remote series) is nine series: over eight.
narrow = {"countries": ["DK", "SE"], "remote": True, "terms": three}
ask = _runner.ask_for(one_host, {}, narrow)
assert ask.countries == ("DK", "SE") and ask.remote is True and ask.terms == ()
assert not _runner.narrowed(one_host, ask)
# The same reach read wide keeps the terms and drops the places.
wide = _runner.ask_for(one_host, {}, narrow | {"wide": True})
assert wide == Ask(terms=tuple(three))
assert _runner.narrowed(one_host, wide)
# Two terms are six series: both stay.
ask = _runner.ask_for(one_host, {}, narrow | {"terms": three[:2]})
assert ask.countries == ("DK", "SE") and ask.remote is True and ask.terms == ("engineer", "data")
def test_a_narrow_read_over_the_cap_by_terms_alone_drops_them() -> None:
nine = [f"word{n}" for n in range(9)]
ask = _runner.ask_for(phenom, {}, {"countries": ["DK"], "terms": nine})
assert ask == Ask(countries=("DK",))
def test_hosts_without_a_query_and_the_search_providers_take_no_terms() -> None:
@@ -387,10 +447,10 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def run() -> dict[str, Any]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type]
batches = pull._boards(FakeConfig(), store)
results = [pull.run_batch(FakeConfig(), store, **b)
for b in batches]
return pull.close_boards(FakeConfig(), store, # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store,
Tally(results=tuple(results)))
# No terms: Europe is too many countries for SmartRecruiters, so whole.
@@ -415,6 +475,32 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
def test_a_narrow_term_read_asks_a_term_in_each_country_and_closes_nothing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:bfl", "Black Forest Labs", tagged=False)
asked: list[str] = []
def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
asked.append(url)
q = _query(url).get("q", [""])[0]
ids = {"engineer": ["1"], "data": ["2"]}[q]
return json.dumps({"totalFound": len(ids),
"content": [_posting(i) for i in ids]}).encode()
monkeypatch.setattr(ats, "_get", get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type]
for b in batches]
settled = pull.close_boards(FakeConfig(), store, # type: ignore[arg-type]
Tally(results=tuple(results)))
assert sorted((_query(u)["q"][0], _query(u)["country"][0]) for u in asked) == [
("data", "dk"), ("data", "se"), ("engineer", "dk"), ("engineer", "se")]
assert store.state["ats:narrowed:smartrecruiters:bfl"] == "1"
assert settled["closed"] == 0
# ------------------------------------------------------- storage and the API
def test_terms_are_stored_normalised_and_kept_by_a_save_without_them(store: Store) -> None:
+9 -7
View File
@@ -1,6 +1,6 @@
"""A company's two fetch overrides (see "The agents set the reach of a pull"
and "Fetch terms" in `docs/decisions.md`): All countries reads its big boards
with the agents' wide reach, All roles with no role-type narrowing. The four
with the agents' wide reach, All roles with no fetch terms. The four
combinations, the reach line, migration 24 and a merge. No network."""
from __future__ import annotations
@@ -51,7 +51,7 @@ def _read(store: Store, target: str) -> tuple[Ask, str]:
@pytest.mark.parametrize("countries, roles, line", [
(False, False, "follows agents: DK, SE · remote not asked on Phenom · all roles"),
(False, False, "follows agents: DK, SE · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
(False, True, "follows agents: DK, SE · remote not asked on Phenom · all roles"),
(True, False, "all countries (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
(True, True, "all countries (Europe) · remote not asked on Phenom · all roles"),
@@ -67,15 +67,17 @@ def test_the_four_combinations(store: Store, countries: bool, roles: bool, line:
assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30
else:
assert ask.countries == ("DK", "SE")
# Terms go only with the wide reach, and never to a company set to All roles.
assert ask.terms == (("data", "engineer", "product") if countries and not roles else ())
# Terms go to every big board, and never to a company set to All roles.
assert ask.terms == (() if roles else ("data", "engineer", "product"))
def test_all_roles_alone_changes_nothing_today(store: Store) -> None:
def test_all_roles_alone_drops_the_terms_and_keeps_the_places(store: Store) -> None:
_board(store, PHILIPS, "Philips")
before = _read(store, PHILIPS)
before, _ = _read(store, PHILIPS)
assert before.terms == ("data", "engineer", "product")
store.set_all_roles("philips", True)
assert _read(store, PHILIPS) == before
after, _ = _read(store, PHILIPS)
assert after == before.model_copy(update={"terms": ()})
@pytest.mark.parametrize("target, size, line", [