diff --git a/app/jobs/limits.py b/app/jobs/limits.py
index 3b9e362..09fa01c 100644
--- a/app/jobs/limits.py
+++ b/app/jobs/limits.py
@@ -37,7 +37,7 @@ WORDS: dict[str, str] = {
"nothing asked": "the host knows none of the places asked, so the read asked nothing and closed nothing",
"size 0": "the board was sized at no roles",
"over the ceiling": "the board is at or over what the host answers at all, so the read is incomplete",
- "text per role": "the list has no text; enrich reads each role's page, ranked roles first",
+ "text per role": "the list has no text; fetch text reads each role's page, for the roles in the cut",
"no text": "the host gives no text",
"excerpt only": "the host gives a teaser and nothing more",
"no pay": "the host states no pay",
diff --git a/app/jobs/reach.py b/app/jobs/reach.py
index cbd2f64..5ae83ac 100644
--- a/app/jobs/reach.py
+++ b/app/jobs/reach.py
@@ -35,21 +35,22 @@ A reach of more countries than a `one`-filter host should be asked one by
one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`,
which knows the reader (`MAX_SERIES`), so such a board is read whole.
-**Fetch terms** (`Agent.fetch_terms`) narrow a wide read by words, so a big
+**Fetch terms** (`Agent.fetch_terms`) narrow a read by words, so a big
board sends the roles that match one of them rather than every role:
- The terms of a pull are the union over the enabled agents. An enabled
agent with none wants every role, so then there are none.
-- They go only with the wide reach: a company set to All countries (`wide`
- in the reach), and a board whose reach `ask_for` drops as too many
- countries for a `one` host. The narrow reach (DK, SE) already leaves at
- most about 150 roles on any big board, and small, unknown and overridden
+- They go to every company board of `SMALL` roles or more whose reach reason
+ is `agents` or `countries`, on the narrow reach (DK, SE) as on the wide
+ one, if its reader declares `query="one"`. Small, unknown and overridden
boards get no reach at all, so none of those take terms.
- A company set to All roles (`companies.all_roles`, global) takes none on
- any of its boards, and no later role filter (category, level) either. It
- changes nothing on the narrow reach today, which takes no terms.
+ any of its boards: no fetch terms and no titles to leave out (the scoring
+ side is in `app/jobs/score.py`).
- Only a reader that declares `query="one"` takes them, one series a term;
- `ask_for` has the rule for terms times the reach's series.
+ `ask_for` has the rule for terms times the reach's series. Over the cap, a
+ wide read keeps the terms and drops the places, a narrow read keeps the
+ places and drops the terms.
- A read with terms is never the whole board: the pull closes nothing
against it (`app/jobs/pull.py`).
- The search providers take none. Every saved search has a query of its
@@ -254,7 +255,7 @@ def _continent(codes: Iterable[str], gaz: places.Gazetteer) -> str:
class Plan:
"""The reach of one pull: `base` for every board, `wide` for the boards
of the companies in `all_countries`, each board's size by target, the
- fetch `terms` of the wide reads, which skip the companies in
+ fetch `terms` of the big boards, which skip the companies in
`all_roles`, `wide_names`, what the agents call the wide reach, and
`facets`, the host ids the runner keeps for a board on a host that
filters by them, for the Companies line."""
@@ -294,10 +295,10 @@ class Plan:
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
"""The `reach` a board's batch hands `ask_for`; None for none. With
- fetch terms, and unless its company is set to All roles, a board on
- the wide reach carries them and `wide`, and a board on the agents'
- reach carries them for `ask_for` to use if it drops the countries as
- too many (see the module docstring). A board with no size carries
+ fetch terms, and unless its company is set to All roles, a big board
+ carries them, and `wide` when its company is set to All countries, for
+ `ask_for` to decide which side gives way over `MAX_SERIES` (see the
+ module docstring). A board with no size carries
`unsized` and `small`, and the reach of a big board unless its params
set their own, for its batch to size it first."""
found, why = self.why(sub)
@@ -437,8 +438,8 @@ def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]]
def _roles(ask: readers.Ask) -> str:
- """The role-type half of the line: the agents' terms the board is
- asked for, or all roles."""
+ """The role half of the line: the agents' terms the board is asked for,
+ or all roles."""
if not ask.terms:
return "all roles"
terms = ", ".join(ask.terms) if len(ask.terms) <= 5 else f"{len(ask.terms)} terms"
@@ -448,18 +449,21 @@ def _roles(ask: readers.Ask) -> str:
def line(found: Plan, sub: Subscription) -> str:
"""What a pull asks `sub`'s board for, once its reader has narrowed the
reach, in plain words, countries first and role types second:
- "follows agents: DK, SE, remote · all roles", "all countries (Europe) ·
- follows agents' terms: engineer, product, data", "all countries (Europe)
- · all roles", "set here: DE · all roles", "all places, too many countries
- to ask · follows agents' terms: engineer", or, for a board read whole,
- why: "under 300 roles: read whole", "Greenhouse cannot filter: read
- whole". A board with no size yet says it is sized on the next pull. On
- a host that filters by its own ids, the line names what the site does
- not know once the runner has asked it (`_site`): "follows agents: DK ·
- SE not on this site · no remote option · all roles", or "site lists
- none of DK, SE: nothing asked, not closing", or, on a host with no remote
- filter while the agents want remote, "follows agents: DK, SE · remote not
- asked on SmartRecruiters · all roles"."""
+ "follows agents: DK, SE, remote · follows agents' terms: engineer,
+ product", "follows agents: DK, SE, remote · all roles" (a company set to
+ All roles, or a reader without a query, or terms over `MAX_SERIES`),
+ "all countries (Europe) · follows agents' terms: engineer, product,
+ data", "all countries (Europe) · all roles", "set here: DE · all roles",
+ "all places, too many countries to ask · follows agents' terms:
+ engineer", or, for a board read whole, why: "under 300 roles: read
+ whole", "Greenhouse cannot filter: read whole". A board with no size yet
+ says it is sized on the next pull. On a host that filters by its own ids,
+ the line names what the site does not know once the runner has asked it
+ (`_site`): "follows agents: DK · SE not on this site · no remote option ·
+ all roles", or "site lists none of DK, SE: nothing asked, not closing",
+ or, on a host with no remote filter while the agents want remote,
+ "follows agents: DK, SE · remote not asked on SmartRecruiters · all
+ roles"."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None:
return ""
@@ -487,7 +491,8 @@ def line(found: Plan, sub: Subscription) -> str:
return f"follows agents: {_places(ask)}{unasked} · {roles}"
if ask.terms:
if given.everywhere:
- return f"all countries (everywhere) · {roles}"
+ what = "all countries (everywhere)" if why == "countries" else _WHOLE[why]
+ return f"{what} · {roles}"
return f"all places, too many countries to ask · {roles}"
if why in ("agents", "countries") and given.everywhere:
return f"{_WHOLE[why]}: read whole"
diff --git a/app/templates/settings/sources.html b/app/templates/settings/sources.html
index cd5f6af..95b594c 100644
--- a/app/templates/settings/sources.html
+++ b/app/templates/settings/sources.html
@@ -149,7 +149,7 @@
-
+
diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py
index edd6915..201c573 100644
--- a/plugins/readers/_runner.py
+++ b/plugins/readers/_runner.py
@@ -391,16 +391,18 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
A remote the reader cannot filter on is dropped here; `remote_wanted`
still says it was wanted.
- Fetch terms (`reach["terms"]`) go only to a board reader that declares
- `query="one"`, and only on a wide read: `reach["wide"]` (a company set
- to All countries), or a reach this rule just dropped as too many
- countries. A company set to All roles gets no terms in its reach (see
- `app/jobs/reach.py`), and a board whose params name a query of their
- own takes none. Terms times the reach's series (`_cost`) may cost
- at most `MAX_SERIES`: over it, the place part of the reach goes first,
- whole, since remote without its countries would read only the remote
- roles; over it with terms alone, the terms go too and the board is read
- unfiltered."""
+ Fetch terms (`reach["terms"]`) go to a board reader that declares
+ `query="one"`, on the wide and the narrow read alike. A company set to All
+ roles, and a board that is small, unknown or overridden, gets no terms in
+ its reach (see `app/jobs/reach.py`), and a board whose params name a query
+ of their own takes none. Terms times the reach's series (`_cost`) may cost
+ at most `MAX_SERIES`. Over it, a wide read (`reach["wide"]`, a company set
+ to All countries, or a reach this rule just dropped as too many
+ countries) drops the place part, whole, since remote without its
+ countries would read only the remote roles, and keeps the terms. A narrow
+ read keeps the places, which the agent's fetch gate needs, and drops the
+ terms, so it is not `narrowed`. Terms alone over it are dropped on either
+ and the board is read unfiltered."""
reads: Reads = module.READS
params = dict(subscription_params or {})
given = reach or {}
@@ -408,12 +410,14 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
declared_now = reads.filters.declared()
wanted = {name: value for name, value in wanted.items() if name in declared_now}
terms = _terms(given.get("terms"))
- if (not wide or reads.scope != "board" or reads.filters.query != "one"
- or "query" in wanted):
+ if reads.scope != "board" or reads.filters.query != "one" or "query" in wanted:
terms = ()
if terms and len(terms) * _cost(reads, wanted) > MAX_SERIES:
- wanted = {name: value for name, value in wanted.items()
- if name not in ("country", "place", "remote")}
+ if wide:
+ wanted = {name: value for name, value in wanted.items()
+ if name not in ("country", "place", "remote")}
+ else:
+ terms = ()
if len(terms) > MAX_SERIES:
terms = ()
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()}
diff --git a/tests/test_fetch_terms.py b/tests/test_fetch_terms.py
index ddfea15..8781e38 100644
--- a/tests/test_fetch_terms.py
+++ b/tests/test_fetch_terms.py
@@ -1,5 +1,5 @@
"""Fetch terms (see "Fetch terms" in `docs/decisions.md`): an agent's words that
-narrow what a big board read with the wide reach sends. The terms of a pull,
+narrow what a big board sends, on the narrow reach and on the wide one. The terms of a pull,
where they apply, how the runner makes series of them, the request each host
sends (the parameters `docs/filter-survey.md` verified), the guards on a
term read, and the agent editor's API. No network."""
@@ -9,6 +9,7 @@ from __future__ import annotations
import json
import sqlite3
import sys
+import types
from collections.abc import Iterator
from pathlib import Path
from typing import Any, ClassVar
@@ -30,7 +31,7 @@ from app.models import Agent, Gate # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import ( # noqa: E402
- Ask, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken,
+ Ask, Filters, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken,
smartrecruiters, successfactors, workday,
)
@@ -116,14 +117,39 @@ def test_a_tagged_board_carries_the_terms_and_the_wide_mark(store: Store) -> Non
assert {"DE", "DK", "FR"} <= set(found["countries"])
-def test_the_narrow_reach_takes_no_terms(store: Store) -> None:
+def test_the_narrow_reach_takes_the_terms_too(store: Store) -> None:
_wolf(store, TERMS)
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
plan = reach.plan(store)
sub = _sub(store, "smartrecruiters:BoschGroup")
+ found = plan.for_subscription(sub)
+ assert found is not None and "wide" not in found
+ ask = _runner.ask_for(smartrecruiters, sub.params, found)
+ assert ask == Ask(countries=("DK", "SE"), terms=("data", "engineer"))
+ assert reach.line(plan, sub) == ("follows agents: DK, SE · remote not asked on "
+ "SmartRecruiters · follows agents' terms: data, engineer")
+
+
+def test_a_company_set_to_all_roles_takes_no_terms_on_the_narrow_reach(store: Store) -> None:
+ _wolf(store, TERMS)
+ _board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
+ store.set_all_roles("bosch", True)
+ plan = reach.plan(store)
+ sub = _sub(store, "smartrecruiters:BoschGroup")
ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub))
assert ask == Ask(countries=("DK", "SE"))
- assert reach.line(plan, sub) == "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"
+ assert reach.line(plan, sub) == \
+ "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"
+
+
+def test_an_agent_that_takes_any_place_still_carries_the_terms(store: Store) -> None:
+ store.save_agent("Anywhere", fetch_terms=["engineer"])
+ _board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
+ plan = reach.plan(store)
+ sub = _sub(store, "smartrecruiters:BoschGroup")
+ assert _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)) == \
+ Ask(terms=("engineer",))
+ assert reach.line(plan, sub) == "an agent takes any place · follows agents' terms: engineer"
@pytest.mark.parametrize("size, params", [
@@ -210,10 +236,44 @@ def test_a_one_host_drops_europe_and_keeps_the_terms() -> None:
assert ask == Ask(terms=("engineer", "data"))
-def test_terms_need_the_wide_mark_or_a_dropped_reach() -> None:
- assert _runner.ask_for(phenom, {}, {"countries": ["DK", "SE"], "terms": TERMS}).terms == ()
- assert _runner.ask_for(phenom, {}, {"terms": TERMS}).terms == ()
- assert _runner.ask_for(phenom, {}, {"terms": TERMS, "wide": True}).terms == ("engineer", "data")
+def test_terms_go_with_a_narrow_or_a_wide_read() -> None:
+ for given in ({"countries": ["DK", "SE"]}, {}, {"wide": True}):
+ ask = _runner.ask_for(phenom, {}, {"terms": TERMS} | given)
+ assert ask.terms == ("engineer", "data")
+
+
+@pytest.fixture
+def one_host() -> types.ModuleType:
+ """A board host that filters by country one a request, by remote, and by
+ query: no shipped reader declares all three."""
+ reads = smartrecruiters.READS.model_copy(update={
+ "filters": Filters(country="one", remote="one", query="one")})
+ module = types.ModuleType("one_host")
+ module.READS = reads # type: ignore[attr-defined]
+ return module
+
+
+def test_over_the_cap_a_narrow_read_keeps_the_places_and_drops_the_terms(
+ one_host: types.ModuleType) -> None:
+ three = ["engineer", "data", "product"]
+ # Three terms times (DK, SE, and the remote series) is nine series: over eight.
+ narrow = {"countries": ["DK", "SE"], "remote": True, "terms": three}
+ ask = _runner.ask_for(one_host, {}, narrow)
+ assert ask.countries == ("DK", "SE") and ask.remote is True and ask.terms == ()
+ assert not _runner.narrowed(one_host, ask)
+ # The same reach read wide keeps the terms and drops the places.
+ wide = _runner.ask_for(one_host, {}, narrow | {"wide": True})
+ assert wide == Ask(terms=tuple(three))
+ assert _runner.narrowed(one_host, wide)
+ # Two terms are six series: both stay.
+ ask = _runner.ask_for(one_host, {}, narrow | {"terms": three[:2]})
+ assert ask.countries == ("DK", "SE") and ask.remote is True and ask.terms == ("engineer", "data")
+
+
+def test_a_narrow_read_over_the_cap_by_terms_alone_drops_them() -> None:
+ nine = [f"word{n}" for n in range(9)]
+ ask = _runner.ask_for(phenom, {}, {"countries": ["DK"], "terms": nine})
+ assert ask == Ask(countries=("DK",))
def test_hosts_without_a_query_and_the_search_providers_take_no_terms() -> None:
@@ -387,10 +447,10 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def run() -> dict[str, Any]:
- batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
- results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type]
+ batches = pull._boards(FakeConfig(), store)
+ results = [pull.run_batch(FakeConfig(), store, **b)
for b in batches]
- return pull.close_boards(FakeConfig(), store, # type: ignore[arg-type]
+ return pull.close_boards(FakeConfig(), store,
Tally(results=tuple(results)))
# No terms: Europe is too many countries for SmartRecruiters, so whole.
@@ -415,6 +475,32 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
+def test_a_narrow_term_read_asks_a_term_in_each_country_and_closes_nothing(
+ store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
+ _wolf(store, TERMS)
+ _board(store, "smartrecruiters:bfl", "Black Forest Labs", tagged=False)
+ asked: list[str] = []
+
+ def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
+ asked.append(url)
+ q = _query(url).get("q", [""])[0]
+ ids = {"engineer": ["1"], "data": ["2"]}[q]
+ return json.dumps({"totalFound": len(ids),
+ "content": [_posting(i) for i in ids]}).encode()
+
+ monkeypatch.setattr(ats, "_get", get)
+ monkeypatch.setattr(ats, "PAGE_DELAY", 0)
+ batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
+ results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type]
+ for b in batches]
+ settled = pull.close_boards(FakeConfig(), store, # type: ignore[arg-type]
+ Tally(results=tuple(results)))
+ assert sorted((_query(u)["q"][0], _query(u)["country"][0]) for u in asked) == [
+ ("data", "dk"), ("data", "se"), ("engineer", "dk"), ("engineer", "se")]
+ assert store.state["ats:narrowed:smartrecruiters:bfl"] == "1"
+ assert settled["closed"] == 0
+
+
# ------------------------------------------------------- storage and the API
def test_terms_are_stored_normalised_and_kept_by_a_save_without_them(store: Store) -> None:
diff --git a/tests/test_overrides.py b/tests/test_overrides.py
index 4a4bc68..b4ebd62 100644
--- a/tests/test_overrides.py
+++ b/tests/test_overrides.py
@@ -1,6 +1,6 @@
"""A company's two fetch overrides (see "The agents set the reach of a pull"
and "Fetch terms" in `docs/decisions.md`): All countries reads its big boards
-with the agents' wide reach, All roles with no role-type narrowing. The four
+with the agents' wide reach, All roles with no fetch terms. The four
combinations, the reach line, migration 24 and a merge. No network."""
from __future__ import annotations
@@ -51,7 +51,7 @@ def _read(store: Store, target: str) -> tuple[Ask, str]:
@pytest.mark.parametrize("countries, roles, line", [
- (False, False, "follows agents: DK, SE · remote not asked on Phenom · all roles"),
+ (False, False, "follows agents: DK, SE · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
(False, True, "follows agents: DK, SE · remote not asked on Phenom · all roles"),
(True, False, "all countries (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
(True, True, "all countries (Europe) · remote not asked on Phenom · all roles"),
@@ -67,15 +67,17 @@ def test_the_four_combinations(store: Store, countries: bool, roles: bool, line:
assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30
else:
assert ask.countries == ("DK", "SE")
- # Terms go only with the wide reach, and never to a company set to All roles.
- assert ask.terms == (("data", "engineer", "product") if countries and not roles else ())
+ # Terms go to every big board, and never to a company set to All roles.
+ assert ask.terms == (() if roles else ("data", "engineer", "product"))
-def test_all_roles_alone_changes_nothing_today(store: Store) -> None:
+def test_all_roles_alone_drops_the_terms_and_keeps_the_places(store: Store) -> None:
_board(store, PHILIPS, "Philips")
- before = _read(store, PHILIPS)
+ before, _ = _read(store, PHILIPS)
+ assert before.terms == ("data", "engineer", "product")
store.set_all_roles("philips", True)
- assert _read(store, PHILIPS) == before
+ after, _ = _read(store, PHILIPS)
+ assert after == before.model_copy(update={"terms": ()})
@pytest.mark.parametrize("target, size, line", [