From a63eed2e56ee9c7b4fd2191093f458d94f2b1822 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 16:11:27 +0200 Subject: [PATCH] Say remote not asked on the Companies line, show a failing search on its row, forget a removed search's record, and mark a read that asked nothing as not closing Co-Authored-By: Claude Fable 5.1 --- app/jobs/reach.py | 28 +++++++++++++++++++--------- app/jobs/sources.py | 5 ++++- app/static/sources.js | 9 +++++++++ app/web/api.py | 21 +++++++++++++++------ plugins/readers/_shared.py | 5 ++--- tests/test_board_size.py | 10 +++++----- tests/test_fetch_terms.py | 4 ++-- tests/test_overrides.py | 8 ++++---- tests/test_reach.py | 2 +- tests/test_web_reach.py | 10 +++++----- 10 files changed, 66 insertions(+), 36 deletions(-) diff --git a/app/jobs/reach.py b/app/jobs/reach.py index c79c8fb..edfd1bf 100644 --- a/app/jobs/reach.py +++ b/app/jobs/reach.py @@ -328,7 +328,7 @@ def site_facets(store: Store, subs: Iterable[Subscription]) -> dict[str, dict[st out: dict[str, dict[str, dict[str, str]]] = {} for sub in subs: module = readers.HOSTS.get(sub.target.partition(":")[0]) - if module is None or not _runner.declared(module) \ + if module is None \ or "ids" not in (module.READS.filters.country, module.READS.filters.remote): continue try: @@ -399,8 +399,9 @@ def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] before any are kept), as `_runner.series` drops them: what is asked, then what the site does not know ("DK · SE not on this site · no remote option"), after `lead` and before `roles`; or, with nothing left to - ask, "site lists none of DK, SE: nothing asked". "" when the site knows - every value, or `lead` is "" and something is still asked.""" + ask, "site lists none of DK, SE: nothing asked, not closing", since the + runner reads such a board as incomplete. "" when the site knows every + value, or `lead` is "" and something is still asked.""" if facets is None: return "" reads = module.READS.filters @@ -418,9 +419,9 @@ def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] remote = bool(ask.remote) and not no_remote if not (known or ask.places or remote): if not missing: - return "site has no remote option: nothing asked" + return "site has no remote option: nothing asked, not closing" named = _codes(missing) + (["remote"] if no_remote else []) - return f"site lists none of {', '.join(named)}: nothing asked" + return f"site lists none of {', '.join(named)}: nothing asked, not closing" if not lead: return "" words = [", ".join(_codes(known) + list(ask.places) + (["remote"] if remote else []))] @@ -452,7 +453,9 @@ def line(found: Plan, sub: Subscription) -> str: a host that filters by its own ids, the line names what the site does not know once the runner has asked it (`_site`): "follows agents: DK · SE not on this site · no remote option · all roles", or "site lists - none of DK, SE: nothing asked".""" + none of DK, SE: nothing asked, not closing", or, on a host with no remote + filter while the agents want remote, "follows agents: DK, SE · remote not + asked on SmartRecruiters · all roles".""" module = readers.HOSTS.get(sub.target.partition(":")[0]) if module is None: return "" @@ -465,12 +468,19 @@ def line(found: Plan, sub: Subscription) -> str: lead = {"agents": "follows agents", "override": "set here"}.get(why, "") if site := _site(module, ask, found.facets.get(sub.target), lead, roles): return site + # The reach wants remote roles and the host cannot be asked for them: + # the read leaves them out, and the settle keeps the ones it has + # (`app.jobs.pull`), so the line says so rather than "DK, SE". + unasked = (f" · remote not asked on {module.LABEL}" + if module.READS.filters.remote == "none" + and _runner.remote_wanted(module, sub.params, found.for_subscription(sub)) + else "") if why == "countries": named = found.wide_names or f"{len(found.wide.countries)} countries" - return f"all countries ({named}) · {roles}" + return f"all countries ({named}){unasked} · {roles}" if why == "override": - return f"set here: {_places(ask)} · {roles}" - return f"follows agents: {_places(ask)} · {roles}" + return f"set here: {_places(ask)}{unasked} · {roles}" + return f"follows agents: {_places(ask)}{unasked} · {roles}" if ask.terms: if given.everywhere: return f"all countries (everywhere) · {roles}" diff --git a/app/jobs/sources.py b/app/jobs/sources.py index add9a2f..ef495b7 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -20,7 +20,7 @@ from urllib.parse import urlsplit from app.config import Config from app.models import HAND_REASONS, Company, Subscription, company_key from app.storage import Store -from app.jobs.pull import FAILING, search_target +from app.jobs.pull import FAILING, SEARCH_FAILING, search_target from plugins import readers if TYPE_CHECKING: @@ -436,6 +436,9 @@ def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]: if sub is None or sub.source != "search": raise KeyError(sub_id) store.unsubscribe(sub_id) + # A record of the search failing would otherwise outlive it, as a + # board's would (`_forget_failing`). + store.state.pop(SEARCH_FAILING + sub.target, None) closed = store.close_subscription(sub_id) if closed: store.rebuild_dupes() diff --git a/app/static/sources.js b/app/static/sources.js index 39feee6..b71756a 100644 --- a/app/static/sources.js +++ b/app/static/sources.js @@ -122,6 +122,15 @@ const parts = [q, `${source.open_roles} open`]; if (source.countries.length) parts.push(countries(source)); sub.append(parts.join(" · "), " "); + if (source.failing_since) { + // The search has failed on every pull since then; after 21 days its + // roles close, as a board's do (app/jobs/pull.py). + const failing = document.createElement("span"); + failing.className = "failing"; + failing.textContent = `not answering since ${source.failing_since.slice(0, 10)}` + + (source.failing_reason ? `: ${source.failing_reason}` : ""); + sub.append("· ", failing, " "); + } const edit = document.createElement("button"); edit.type = "button"; edit.className = "linky"; diff --git a/app/web/api.py b/app/web/api.py index e06a7ba..1df1e3a 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -811,7 +811,9 @@ def mark_careers_opened(key: str, store: StoreDep) -> Company: class SourceOut(BaseModel): """A board or a saved search, with what it brings in: its open roles, and - the countries they are in, most first.""" + the countries they are in, most first. `failing_since` and + `failing_reason` say since when it has failed on every fetch and why it + last did (`jobs.pull.failing_boards`), both empty while it answers.""" id: int source: str @@ -822,6 +824,8 @@ class SourceOut(BaseModel): on: bool open_roles: int countries: list[tuple[str, int]] + failing_since: str = "" + failing_reason: str = "" class SourcesOut(BaseModel): @@ -831,19 +835,23 @@ class SourcesOut(BaseModel): adzuna_key: bool -def _source_out(sub: Subscription, countries: dict[str, int]) -> SourceOut: +def _source_out(sub: Subscription, countries: dict[str, int], + failing: dict[str, tuple[str, str]]) -> SourceOut: + since, reason = failing.get(sub.target, ("", "")) return SourceOut( id=sub.id, source=sub.source, target=sub.target, label=sub.label, provider=sub.board, params=sub.params, on=bool(sub.subscribed and sub.fetch), open_roles=sum(countries.values()), - countries=sorted(countries.items(), key=lambda kv: -kv[1])) + countries=sorted(countries.items(), key=lambda kv: -kv[1]), + failing_since=since, failing_reason=reason) def _one_source(store: Store, sub_id: int) -> SourceOut: sub = store.subscription(sub_id) if sub is None: raise HTTPException(404, f"no such board or search: {sub_id}") - return _source_out(sub, store.open_countries_by_subscription().get(sub_id, {})) + return _source_out(sub, store.open_countries_by_subscription().get(sub_id, {}), + jobs.pull.failing_boards(store)) @router.get("/sources") @@ -851,9 +859,10 @@ def list_sources(store: StoreDep, config: ConfigDep) -> SourcesOut: """Every board not detached, and every search not removed. A board switched off stays listed, so it can be switched back on.""" counts = store.open_countries_by_subscription() - searches = [_source_out(s, counts.get(s.id, {})) + failing = jobs.pull.failing_boards(store) + searches = [_source_out(s, counts.get(s.id, {}), failing) for s in store.subscriptions(source="search") if s.show] - boards = [_source_out(s, counts.get(s.id, {})) + boards = [_source_out(s, counts.get(s.id, {}), failing) for s in store.subscriptions(source="ats") if s.show] return SourcesOut(searches=searches, boards=boards, adzuna_key=jobs.sources.has_key(config, "adzuna")) diff --git a/plugins/readers/_shared.py b/plugins/readers/_shared.py index d928d89..2eb16ee 100644 --- a/plugins/readers/_shared.py +++ b/plugins/readers/_shared.py @@ -18,9 +18,8 @@ import httpx from ._contract import Place -#: What an undeclared module's parser turns a feed into; a module that -#: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`. -#: One dict per row, with the keys +#: What a reader's parser turns a feed into, as `dict[str, Any]`; the keys a +#: row carries are `_contract.Row`'s. One dict per row, with the keys #: `source_id`, `title`, `location`, `url`, `posted`, `department`, #: `description`, `remote` and `salary_text`, and, where a host states pay, #: `salary_min`, `salary_max` and `currency`. diff --git a/tests/test_board_size.py b/tests/test_board_size.py index a522f53..57e8090 100644 --- a/tests/test_board_size.py +++ b/tests/test_board_size.py @@ -307,7 +307,7 @@ WORKDAY = "workday:acme.wd3/Jobs" "follows agents: DK, remote · SE not on this site · all roles"), ({"country": {"DE": "c=de"}, "remote": {"remote": "r=1"}}, "follows agents: remote · DK, SE not on this site · all roles"), - ({"country": {"DE": "c=de"}}, "site lists none of DK, SE, remote: nothing asked"), + ({"country": {"DE": "c=de"}}, "site lists none of DK, SE, remote: nothing asked, not closing"), ], ids=["none kept", "all known", "one country, no remote", "one country and remote", "remote only", "none"]) def test_the_line_names_what_a_site_does_not_know( @@ -322,7 +322,7 @@ def test_the_line_names_what_a_site_does_not_know( assert reach.line(plan, sub) == line # What the line says is left to ask is what the runner asks. ask = _runner.ask_for(workday, sub.params, plan.for_subscription(sub)) - assert (_runner.series(workday, ask, facets) == []) == line.endswith("nothing asked") + assert (_runner.series(workday, ask, facets) == []) == line.endswith("nothing asked, not closing") def test_the_line_says_nothing_is_asked_in_the_plans_words(store: Store) -> None: @@ -331,7 +331,7 @@ def test_the_line_says_nothing_is_asked_in_the_plans_words(store: Store) -> None store.state[f"ats:facets:{WORKDAY}"] = json.dumps( {"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DE": "c=de"}}}) assert reach.line(reach.plan(store), _sub(store, WORKDAY)) == \ - "site lists none of DK, SE: nothing asked" + "site lists none of DK, SE: nothing asked, not closing" def test_a_boards_own_reach_is_named_the_same_and_remote_alone_is_read_whole( @@ -378,7 +378,7 @@ def test_recheck_sizes_the_board_again_and_queues_a_pull( res = client.post(f"/api/sources/{sub_id}/size") assert res.status_code == 200, res.text assert res.json() == {"size": 4830, "size_checked": TODAY, - "reach": "follows agents: DK, SE · all roles"} + "reach": "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"} assert app.state.queue.jobs.last("pull", states=("queued", "running", "done")) is not None [row] = client.get("/api/companies").json() assert (row["size"], row["size_checked"]) == (4830, TODAY) @@ -408,7 +408,7 @@ def test_the_cli_sizes_one_board_or_every_board_without_a_size( monkeypatch.setattr(ats, "_get", SmartRecruiters(4830)) assert _cli(tmp_path, "size", "bosch") == 0 assert json.loads(capsys.readouterr().out) == { - "board": BOSCH, "size": 4830, "reach": "follows agents: DK, SE · all roles"} + "board": BOSCH, "size": 4830, "reach": "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles"} def some(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes: if slug == "gone": diff --git a/tests/test_fetch_terms.py b/tests/test_fetch_terms.py index 8b4836e..fd08e84 100644 --- a/tests/test_fetch_terms.py +++ b/tests/test_fetch_terms.py @@ -123,7 +123,7 @@ def test_the_narrow_reach_takes_no_terms(store: Store) -> None: sub = _sub(store, "smartrecruiters:BoschGroup") ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)) assert ask == Ask(countries=("DK", "SE")) - assert reach.line(plan, sub) == "follows agents: DK, SE · all roles" + assert reach.line(plan, sub) == "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles" @pytest.mark.parametrize("size, params", [ @@ -240,7 +240,7 @@ def test_the_reach_line_counts_the_terms(store: Store) -> None: plan = reach.plan(store) philips = _sub(store, "phenom:www.careers.philips.com/global/en") assert reach.line(plan, philips) == \ - "all countries (Europe) · follows agents' terms: data, engineer, head of" + "all countries (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, head of" assert reach.line(plan, _sub(store, "smartrecruiters:BoschGroup")) == \ "all places, too many countries to ask · follows agents' terms: data, engineer, head of" diff --git a/tests/test_overrides.py b/tests/test_overrides.py index e2b75e9..487b175 100644 --- a/tests/test_overrides.py +++ b/tests/test_overrides.py @@ -51,10 +51,10 @@ def _read(store: Store, target: str) -> tuple[Ask, str]: @pytest.mark.parametrize("countries, roles, line", [ - (False, False, "follows agents: DK, SE · all roles"), - (False, True, "follows agents: DK, SE · all roles"), - (True, False, "all countries (Europe) · follows agents' terms: data, engineer, product"), - (True, True, "all countries (Europe) · all roles"), + (False, False, "follows agents: DK, SE · remote not asked on Phenom · all roles"), + (False, True, "follows agents: DK, SE · remote not asked on Phenom · all roles"), + (True, False, "all countries (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"), + (True, True, "all countries (Europe) · remote not asked on Phenom · all roles"), ], ids=["neither", "all roles alone", "all countries alone", "both"]) def test_the_four_combinations(store: Store, countries: bool, roles: bool, line: str) -> None: _board(store, PHILIPS, "Philips") diff --git a/tests/test_reach.py b/tests/test_reach.py index 3d29918..1ce3063 100644 --- a/tests/test_reach.py +++ b/tests/test_reach.py @@ -146,7 +146,7 @@ def test_a_tag_asks_phenom_for_europe_in_one_request(store: Store) -> None: store.set_all_countries("mistral-ai", True) [sub] = store.subscriptions(source="ats") found = reach.plan(store) - assert reach.line(found, sub) == "all countries (Europe) · all roles" + assert reach.line(found, sub) == "all countries (Europe) · remote not asked on Phenom · all roles" def test_an_agent_without_a_required_place_makes_the_tag_everywhere(store: Store) -> None: diff --git a/tests/test_web_reach.py b/tests/test_web_reach.py index 2f53b55..a9fbab7 100644 --- a/tests/test_web_reach.py +++ b/tests/test_web_reach.py @@ -53,7 +53,7 @@ def _row(client: TestClient) -> dict[str, object]: def test_a_company_row_says_what_its_board_is_asked_for(client: TestClient, bosch: int) -> None: row = _row(client) # SmartRecruiters cannot ask for remote, so only the country is asked. - assert (row["reach"], row["reach_params"]) == ("follows agents: DK · all roles", {}) + assert (row["reach"], row["reach_params"]) == ("follows agents: DK · remote not asked on SmartRecruiters · all roles", {}) def test_a_boards_own_reach_is_kept_in_its_params_and_wins( @@ -61,14 +61,14 @@ def test_a_boards_own_reach_is_kept_in_its_params_and_wins( res = client.post(f"/api/sources/{bosch}/reach", json={"mode": "own", "countries": ["de", "se"], "remote": True}) assert res.status_code == 200, res.text - assert res.json()["reach"] == "set here: DE, SE · all roles" + assert res.json()["reach"] == "set here: DE, SE · remote not asked on SmartRecruiters · all roles" sub = store.subscription(bosch) assert sub is not None assert sub.params == {"note": "kept", "countries": ["DE", "SE"], "remote": True} assert _row(client)["reach_params"] == {"countries": ["DE", "SE"], "remote": True} res = client.post(f"/api/sources/{bosch}/reach", json={"mode": "agents"}) - assert res.json()["reach"] == "follows agents: DK · all roles" + assert res.json()["reach"] == "follows agents: DK · remote not asked on SmartRecruiters · all roles" assert store.subscription(bosch).params == {"note": "kept"} # type: ignore[union-attr] @@ -123,7 +123,7 @@ def test_all_roles_is_its_own_switch_and_on_queues_a_pull( row = _row(client) # The narrow reach takes no terms, so on its own it changes nothing today. assert (row["all_countries"], row["all_roles"], row["reach"]) == ( - False, True, "follows agents: DK · all roles") + False, True, "follows agents: DK · remote not asked on SmartRecruiters · all roles") @pytest.mark.parametrize("path, setter", [("all-countries", "set_all_countries"), @@ -136,7 +136,7 @@ def test_turning_an_override_off_queues_nothing( assert res.status_code == 200 assert (res.json()["all_countries"], res.json()["all_roles"]) == (0, 0) assert app.state.queue.jobs.last("pull", states=_ANY_STATE) is None - assert _row(client)["reach"] == "follows agents: DK · all roles" + assert _row(client)["reach"] == "follows agents: DK · remote not asked on SmartRecruiters · all roles" assert client.post(f"/api/companies/nobody/{path}", json={"on": True}).status_code == 404