diff --git a/app/jobs/limits.py b/app/jobs/limits.py index 3b9e362..09fa01c 100644 --- a/app/jobs/limits.py +++ b/app/jobs/limits.py @@ -37,7 +37,7 @@ WORDS: dict[str, str] = { "nothing asked": "the host knows none of the places asked, so the read asked nothing and closed nothing", "size 0": "the board was sized at no roles", "over the ceiling": "the board is at or over what the host answers at all, so the read is incomplete", - "text per role": "the list has no text; enrich reads each role's page, ranked roles first", + "text per role": "the list has no text; fetch text reads each role's page, for the roles in the cut", "no text": "the host gives no text", "excerpt only": "the host gives a teaser and nothing more", "no pay": "the host states no pay", diff --git a/app/jobs/reach.py b/app/jobs/reach.py index cbd2f64..5ae83ac 100644 --- a/app/jobs/reach.py +++ b/app/jobs/reach.py @@ -35,21 +35,22 @@ A reach of more countries than a `one`-filter host should be asked one by one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`, which knows the reader (`MAX_SERIES`), so such a board is read whole. -**Fetch terms** (`Agent.fetch_terms`) narrow a wide read by words, so a big +**Fetch terms** (`Agent.fetch_terms`) narrow a read by words, so a big board sends the roles that match one of them rather than every role: - The terms of a pull are the union over the enabled agents. An enabled agent with none wants every role, so then there are none. -- They go only with the wide reach: a company set to All countries (`wide` - in the reach), and a board whose reach `ask_for` drops as too many - countries for a `one` host. The narrow reach (DK, SE) already leaves at - most about 150 roles on any big board, and small, unknown and overridden +- They go to every company board of `SMALL` roles or more whose reach reason + is `agents` or `countries`, on the narrow reach (DK, SE) as on the wide + one, if its reader declares `query="one"`. Small, unknown and overridden boards get no reach at all, so none of those take terms. - A company set to All roles (`companies.all_roles`, global) takes none on - any of its boards, and no later role filter (category, level) either. It - changes nothing on the narrow reach today, which takes no terms. + any of its boards: no fetch terms and no titles to leave out (the scoring + side is in `app/jobs/score.py`). - Only a reader that declares `query="one"` takes them, one series a term; - `ask_for` has the rule for terms times the reach's series. + `ask_for` has the rule for terms times the reach's series. Over the cap, a + wide read keeps the terms and drops the places, a narrow read keeps the + places and drops the terms. - A read with terms is never the whole board: the pull closes nothing against it (`app/jobs/pull.py`). - The search providers take none. Every saved search has a query of its @@ -254,7 +255,7 @@ def _continent(codes: Iterable[str], gaz: places.Gazetteer) -> str: class Plan: """The reach of one pull: `base` for every board, `wide` for the boards of the companies in `all_countries`, each board's size by target, the - fetch `terms` of the wide reads, which skip the companies in + fetch `terms` of the big boards, which skip the companies in `all_roles`, `wide_names`, what the agents call the wide reach, and `facets`, the host ids the runner keeps for a board on a host that filters by them, for the Companies line.""" @@ -294,10 +295,10 @@ class Plan: def for_subscription(self, sub: Subscription) -> dict[str, Any] | None: """The `reach` a board's batch hands `ask_for`; None for none. With - fetch terms, and unless its company is set to All roles, a board on - the wide reach carries them and `wide`, and a board on the agents' - reach carries them for `ask_for` to use if it drops the countries as - too many (see the module docstring). A board with no size carries + fetch terms, and unless its company is set to All roles, a big board + carries them, and `wide` when its company is set to All countries, for + `ask_for` to decide which side gives way over `MAX_SERIES` (see the + module docstring). A board with no size carries `unsized` and `small`, and the reach of a big board unless its params set their own, for its batch to size it first.""" found, why = self.why(sub) @@ -437,8 +438,8 @@ def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] def _roles(ask: readers.Ask) -> str: - """The role-type half of the line: the agents' terms the board is - asked for, or all roles.""" + """The role half of the line: the agents' terms the board is asked for, + or all roles.""" if not ask.terms: return "all roles" terms = ", ".join(ask.terms) if len(ask.terms) <= 5 else f"{len(ask.terms)} terms" @@ -448,18 +449,21 @@ def _roles(ask: readers.Ask) -> str: def line(found: Plan, sub: Subscription) -> str: """What a pull asks `sub`'s board for, once its reader has narrowed the reach, in plain words, countries first and role types second: - "follows agents: DK, SE, remote · all roles", "all countries (Europe) · - follows agents' terms: engineer, product, data", "all countries (Europe) - · all roles", "set here: DE · all roles", "all places, too many countries - to ask · follows agents' terms: engineer", or, for a board read whole, - why: "under 300 roles: read whole", "Greenhouse cannot filter: read - whole". A board with no size yet says it is sized on the next pull. On - a host that filters by its own ids, the line names what the site does - not know once the runner has asked it (`_site`): "follows agents: DK · - SE not on this site · no remote option · all roles", or "site lists - none of DK, SE: nothing asked, not closing", or, on a host with no remote - filter while the agents want remote, "follows agents: DK, SE · remote not - asked on SmartRecruiters · all roles".""" + "follows agents: DK, SE, remote · follows agents' terms: engineer, + product", "follows agents: DK, SE, remote · all roles" (a company set to + All roles, or a reader without a query, or terms over `MAX_SERIES`), + "all countries (Europe) · follows agents' terms: engineer, product, + data", "all countries (Europe) · all roles", "set here: DE · all roles", + "all places, too many countries to ask · follows agents' terms: + engineer", or, for a board read whole, why: "under 300 roles: read + whole", "Greenhouse cannot filter: read whole". A board with no size yet + says it is sized on the next pull. On a host that filters by its own ids, + the line names what the site does not know once the runner has asked it + (`_site`): "follows agents: DK · SE not on this site · no remote option · + all roles", or "site lists none of DK, SE: nothing asked, not closing", + or, on a host with no remote filter while the agents want remote, + "follows agents: DK, SE · remote not asked on SmartRecruiters · all + roles".""" module = readers.HOSTS.get(sub.target.partition(":")[0]) if module is None: return "" @@ -487,7 +491,8 @@ def line(found: Plan, sub: Subscription) -> str: return f"follows agents: {_places(ask)}{unasked} · {roles}" if ask.terms: if given.everywhere: - return f"all countries (everywhere) · {roles}" + what = "all countries (everywhere)" if why == "countries" else _WHOLE[why] + return f"{what} · {roles}" return f"all places, too many countries to ask · {roles}" if why in ("agents", "countries") and given.everywhere: return f"{_WHOLE[why]}: read whole" diff --git a/app/templates/settings/sources.html b/app/templates/settings/sources.html index cd5f6af..95b594c 100644 --- a/app/templates/settings/sources.html +++ b/app/templates/settings/sources.html @@ -149,7 +149,7 @@ - + diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index edd6915..201c573 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -391,16 +391,18 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = A remote the reader cannot filter on is dropped here; `remote_wanted` still says it was wanted. - Fetch terms (`reach["terms"]`) go only to a board reader that declares - `query="one"`, and only on a wide read: `reach["wide"]` (a company set - to All countries), or a reach this rule just dropped as too many - countries. A company set to All roles gets no terms in its reach (see - `app/jobs/reach.py`), and a board whose params name a query of their - own takes none. Terms times the reach's series (`_cost`) may cost - at most `MAX_SERIES`: over it, the place part of the reach goes first, - whole, since remote without its countries would read only the remote - roles; over it with terms alone, the terms go too and the board is read - unfiltered.""" + Fetch terms (`reach["terms"]`) go to a board reader that declares + `query="one"`, on the wide and the narrow read alike. A company set to All + roles, and a board that is small, unknown or overridden, gets no terms in + its reach (see `app/jobs/reach.py`), and a board whose params name a query + of their own takes none. Terms times the reach's series (`_cost`) may cost + at most `MAX_SERIES`. Over it, a wide read (`reach["wide"]`, a company set + to All countries, or a reach this rule just dropped as too many + countries) drops the place part, whole, since remote without its + countries would read only the remote roles, and keeps the terms. A narrow + read keeps the places, which the agent's fetch gate needs, and drops the + terms, so it is not `narrowed`. Terms alone over it are dropped on either + and the board is read unfiltered.""" reads: Reads = module.READS params = dict(subscription_params or {}) given = reach or {} @@ -408,12 +410,14 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = declared_now = reads.filters.declared() wanted = {name: value for name, value in wanted.items() if name in declared_now} terms = _terms(given.get("terms")) - if (not wide or reads.scope != "board" or reads.filters.query != "one" - or "query" in wanted): + if reads.scope != "board" or reads.filters.query != "one" or "query" in wanted: terms = () if terms and len(terms) * _cost(reads, wanted) > MAX_SERIES: - wanted = {name: value for name, value in wanted.items() - if name not in ("country", "place", "remote")} + if wide: + wanted = {name: value for name, value in wanted.items() + if name not in ("country", "place", "remote")} + else: + terms = () if len(terms) > MAX_SERIES: terms = () fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()} diff --git a/tests/test_fetch_terms.py b/tests/test_fetch_terms.py index ddfea15..8781e38 100644 --- a/tests/test_fetch_terms.py +++ b/tests/test_fetch_terms.py @@ -1,5 +1,5 @@ """Fetch terms (see "Fetch terms" in `docs/decisions.md`): an agent's words that -narrow what a big board read with the wide reach sends. The terms of a pull, +narrow what a big board sends, on the narrow reach and on the wide one. The terms of a pull, where they apply, how the runner makes series of them, the request each host sends (the parameters `docs/filter-survey.md` verified), the guards on a term read, and the agent editor's API. No network.""" @@ -9,6 +9,7 @@ from __future__ import annotations import json import sqlite3 import sys +import types from collections.abc import Iterator from pathlib import Path from typing import Any, ClassVar @@ -30,7 +31,7 @@ from app.models import Agent, Gate # noqa: E402 from app.storage import Store # noqa: E402 from plugins import ats # noqa: E402 from plugins.readers import ( # noqa: E402 - Ask, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken, + Ask, Filters, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken, smartrecruiters, successfactors, workday, ) @@ -116,14 +117,39 @@ def test_a_tagged_board_carries_the_terms_and_the_wide_mark(store: Store) -> Non assert {"DE", "DK", "FR"} <= set(found["countries"]) -def test_the_narrow_reach_takes_no_terms(store: Store) -> None: +def test_the_narrow_reach_takes_the_terms_too(store: Store) -> None: _wolf(store, TERMS) _board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False) plan = reach.plan(store) sub = _sub(store, "smartrecruiters:BoschGroup") + found = plan.for_subscription(sub) + assert found is not None and "wide" not in found + ask = _runner.ask_for(smartrecruiters, sub.params, found) + assert ask == Ask(countries=("DK", "SE"), terms=("data", "engineer")) + assert reach.line(plan, sub) == ("follows agents: DK, SE · remote not asked on " + "SmartRecruiters · follows agents' terms: data, engineer") + + +def test_a_company_set_to_all_roles_takes_no_terms_on_the_narrow_reach(store: Store) -> None: + _wolf(store, TERMS) + _board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False) + store.set_all_roles("bosch", True) + plan = reach.plan(store) + sub = _sub(store, "smartrecruiters:BoschGroup") ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)) assert ask == Ask(countries=("DK", "SE")) - assert reach.line(plan, sub) == "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles" + assert reach.line(plan, sub) == \ + "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles" + + +def test_an_agent_that_takes_any_place_still_carries_the_terms(store: Store) -> None: + store.save_agent("Anywhere", fetch_terms=["engineer"]) + _board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False) + plan = reach.plan(store) + sub = _sub(store, "smartrecruiters:BoschGroup") + assert _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)) == \ + Ask(terms=("engineer",)) + assert reach.line(plan, sub) == "an agent takes any place · follows agents' terms: engineer" @pytest.mark.parametrize("size, params", [ @@ -210,10 +236,44 @@ def test_a_one_host_drops_europe_and_keeps_the_terms() -> None: assert ask == Ask(terms=("engineer", "data")) -def test_terms_need_the_wide_mark_or_a_dropped_reach() -> None: - assert _runner.ask_for(phenom, {}, {"countries": ["DK", "SE"], "terms": TERMS}).terms == () - assert _runner.ask_for(phenom, {}, {"terms": TERMS}).terms == () - assert _runner.ask_for(phenom, {}, {"terms": TERMS, "wide": True}).terms == ("engineer", "data") +def test_terms_go_with_a_narrow_or_a_wide_read() -> None: + for given in ({"countries": ["DK", "SE"]}, {}, {"wide": True}): + ask = _runner.ask_for(phenom, {}, {"terms": TERMS} | given) + assert ask.terms == ("engineer", "data") + + +@pytest.fixture +def one_host() -> types.ModuleType: + """A board host that filters by country one a request, by remote, and by + query: no shipped reader declares all three.""" + reads = smartrecruiters.READS.model_copy(update={ + "filters": Filters(country="one", remote="one", query="one")}) + module = types.ModuleType("one_host") + module.READS = reads # type: ignore[attr-defined] + return module + + +def test_over_the_cap_a_narrow_read_keeps_the_places_and_drops_the_terms( + one_host: types.ModuleType) -> None: + three = ["engineer", "data", "product"] + # Three terms times (DK, SE, and the remote series) is nine series: over eight. + narrow = {"countries": ["DK", "SE"], "remote": True, "terms": three} + ask = _runner.ask_for(one_host, {}, narrow) + assert ask.countries == ("DK", "SE") and ask.remote is True and ask.terms == () + assert not _runner.narrowed(one_host, ask) + # The same reach read wide keeps the terms and drops the places. + wide = _runner.ask_for(one_host, {}, narrow | {"wide": True}) + assert wide == Ask(terms=tuple(three)) + assert _runner.narrowed(one_host, wide) + # Two terms are six series: both stay. + ask = _runner.ask_for(one_host, {}, narrow | {"terms": three[:2]}) + assert ask.countries == ("DK", "SE") and ask.remote is True and ask.terms == ("engineer", "data") + + +def test_a_narrow_read_over_the_cap_by_terms_alone_drops_them() -> None: + nine = [f"word{n}" for n in range(9)] + ask = _runner.ask_for(phenom, {}, {"countries": ["DK"], "terms": nine}) + assert ask == Ask(countries=("DK",)) def test_hosts_without_a_query_and_the_search_providers_take_no_terms() -> None: @@ -387,10 +447,10 @@ def test_a_term_read_writes_no_size_and_closes_nothing( monkeypatch.setattr(ats, "PAGE_DELAY", 0) def run() -> dict[str, Any]: - batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] - results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type] + batches = pull._boards(FakeConfig(), store) + results = [pull.run_batch(FakeConfig(), store, **b) for b in batches] - return pull.close_boards(FakeConfig(), store, # type: ignore[arg-type] + return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # No terms: Europe is too many countries for SmartRecruiters, so whole. @@ -415,6 +475,32 @@ def test_a_term_read_writes_no_size_and_closes_nothing( assert store.state["ats:size:smartrecruiters:bfl"] == "1000" +def test_a_narrow_term_read_asks_a_term_in_each_country_and_closes_nothing( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _wolf(store, TERMS) + _board(store, "smartrecruiters:bfl", "Black Forest Labs", tagged=False) + asked: list[str] = [] + + def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes: + asked.append(url) + q = _query(url).get("q", [""])[0] + ids = {"engineer": ["1"], "data": ["2"]}[q] + return json.dumps({"totalFound": len(ids), + "content": [_posting(i) for i in ids]}).encode() + + monkeypatch.setattr(ats, "_get", get) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] + results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type] + for b in batches] + settled = pull.close_boards(FakeConfig(), store, # type: ignore[arg-type] + Tally(results=tuple(results))) + assert sorted((_query(u)["q"][0], _query(u)["country"][0]) for u in asked) == [ + ("data", "dk"), ("data", "se"), ("engineer", "dk"), ("engineer", "se")] + assert store.state["ats:narrowed:smartrecruiters:bfl"] == "1" + assert settled["closed"] == 0 + + # ------------------------------------------------------- storage and the API def test_terms_are_stored_normalised_and_kept_by_a_save_without_them(store: Store) -> None: diff --git a/tests/test_overrides.py b/tests/test_overrides.py index 4a4bc68..b4ebd62 100644 --- a/tests/test_overrides.py +++ b/tests/test_overrides.py @@ -1,6 +1,6 @@ """A company's two fetch overrides (see "The agents set the reach of a pull" and "Fetch terms" in `docs/decisions.md`): All countries reads its big boards -with the agents' wide reach, All roles with no role-type narrowing. The four +with the agents' wide reach, All roles with no fetch terms. The four combinations, the reach line, migration 24 and a merge. No network.""" from __future__ import annotations @@ -51,7 +51,7 @@ def _read(store: Store, target: str) -> tuple[Ask, str]: @pytest.mark.parametrize("countries, roles, line", [ - (False, False, "follows agents: DK, SE · remote not asked on Phenom · all roles"), + (False, False, "follows agents: DK, SE · remote not asked on Phenom · follows agents' terms: data, engineer, product"), (False, True, "follows agents: DK, SE · remote not asked on Phenom · all roles"), (True, False, "all countries (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"), (True, True, "all countries (Europe) · remote not asked on Phenom · all roles"), @@ -67,15 +67,17 @@ def test_the_four_combinations(store: Store, countries: bool, roles: bool, line: assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30 else: assert ask.countries == ("DK", "SE") - # Terms go only with the wide reach, and never to a company set to All roles. - assert ask.terms == (("data", "engineer", "product") if countries and not roles else ()) + # Terms go to every big board, and never to a company set to All roles. + assert ask.terms == (() if roles else ("data", "engineer", "product")) -def test_all_roles_alone_changes_nothing_today(store: Store) -> None: +def test_all_roles_alone_drops_the_terms_and_keeps_the_places(store: Store) -> None: _board(store, PHILIPS, "Philips") - before = _read(store, PHILIPS) + before, _ = _read(store, PHILIPS) + assert before.terms == ("data", "engineer", "product") store.set_all_roles("philips", True) - assert _read(store, PHILIPS) == before + after, _ = _read(store, PHILIPS) + assert after == before.model_copy(update={"terms": ()}) @pytest.mark.parametrize("target, size, line", [