Merge two strikes for boards swept across pulls (wp-b-strikes)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
19401cfc7a
5 files changed
+373
-11
No files matched your search
+52
-1
@@ -24,6 +24,13 @@ filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
|||||||
under a tenth of what the same read listed last time is recorded by the
|
under a tenth of what the same read listed last time is recorded by the
|
||||||
runner as a failure, not as a list to close against.
|
runner as a failure, not as a list to close against.
|
||||||
|
|
||||||
|
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
||||||
|
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
||||||
|
row missed it (`_close_swept`): offset paging drifts when roles close between
|
||||||
|
pulls, so one sweep can miss a role that is still open. The ids a finished
|
||||||
|
sweep missed are kept in `ats:missing:<target>`, and a complete read that was
|
||||||
|
not swept clears them.
|
||||||
|
|
||||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||||
the roles matching a word, which says nothing of the roles it did not list,
|
the roles matching a word, which says nothing of the roles it did not list,
|
||||||
so it closes nothing; only an unfiltered or a place-filtered complete read
|
so it closes nothing; only an unfiltered or a place-filtered complete read
|
||||||
@@ -282,7 +289,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
|
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
|
||||||
continue
|
continue
|
||||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||||
if store.state.get(f"{source}:filtered:{sub.target}", ""):
|
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
|
||||||
|
if store.state.get(f"{source}:swept:{sub.target}", ""):
|
||||||
|
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
|
||||||
|
continue
|
||||||
|
store.state.pop(f"{source}:missing:{sub.target}", None)
|
||||||
|
if is_filtered:
|
||||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
||||||
else:
|
else:
|
||||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||||
@@ -313,6 +325,45 @@ def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription
|
|||||||
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
||||||
|
|
||||||
|
|
||||||
|
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||||
|
filtered: bool) -> int:
|
||||||
|
"""The close of a board whose complete read finished a sweep across
|
||||||
|
pulls: its open roles missing from `seen_ids` that also are in
|
||||||
|
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
||||||
|
The ones missing now are kept there for the next sweep. A role one sweep
|
||||||
|
missed may only have shifted back past the cursor when roles closed
|
||||||
|
between pulls, so one sweep is not evidence that it is gone. A filtered
|
||||||
|
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
||||||
|
stays open, and stays in `missing`."""
|
||||||
|
key = f"{source}:missing:{sub.target}"
|
||||||
|
try:
|
||||||
|
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
earlier = set()
|
||||||
|
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||||
|
AND uid NOT IN (SELECT role_uid FROM applications
|
||||||
|
WHERE role_uid IS NOT NULL)""" if filtered else ""
|
||||||
|
with store.transaction() as conn:
|
||||||
|
missed = [row[0] for row in conn.execute(
|
||||||
|
"""SELECT source_id FROM roles
|
||||||
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
|
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
||||||
|
(source, sub.id, json.dumps(list(seen_ids))))]
|
||||||
|
closing = [row[0] for row in conn.execute(
|
||||||
|
f"""SELECT source_id FROM roles
|
||||||
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
|
AND source_id IN (SELECT value FROM json_each(?))
|
||||||
|
AND source_id IN (SELECT value FROM json_each(?)){acted_on}""",
|
||||||
|
(source, sub.id, json.dumps(sorted(earlier)), json.dumps(missed)))]
|
||||||
|
count = conn.execute(
|
||||||
|
"""UPDATE roles SET closed_at = ?
|
||||||
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
|
AND source_id IN (SELECT value FROM json_each(?))""",
|
||||||
|
(store.now(), source, sub.id, json.dumps(closing))).rowcount
|
||||||
|
store.state[key] = json.dumps(sorted(set(missed) - set(closing)))
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||||
failed: when the failures began, and what the latest one said."""
|
failed: when the failures began, and what the latest one said."""
|
||||||
|
|||||||
@@ -23,6 +23,11 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
|||||||
count `{"asks", "count"}`, the roles the last whole read listed, for
|
count `{"asks", "count"}`, the roles the last whole read listed, for
|
||||||
the 10% guard
|
the 10% guard
|
||||||
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
||||||
|
swept "1" when the last stamped listing finished a sweep carried over
|
||||||
|
from an earlier pull (`Listing.swept`), else "". Offset paging
|
||||||
|
drifts when roles close between pulls, so `app.jobs.pull` closes
|
||||||
|
a role of such a board only when two finished sweeps in a row
|
||||||
|
missed it (it keeps the ids the last one missed in `missing`)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -126,7 +131,10 @@ class Listing:
|
|||||||
state to keep for the next pull, None when no sweep is under way.
|
state to keep for the next pull, None when no sweep is under way.
|
||||||
`filtered` says the read asked the host for the roles in some places,
|
`filtered` says the read asked the host for the roles in some places,
|
||||||
`narrowed` for the roles matching words (`narrowed()`): the ids of
|
`narrowed` for the roles matching words (`narrowed()`): the ids of
|
||||||
neither are the whole board."""
|
neither are the whole board. `swept` says a complete listing finished a
|
||||||
|
sweep carried over from an earlier pull: `ids` were read across pulls,
|
||||||
|
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||||
|
roles back past the cursor, so `ids` can miss a role that is still open."""
|
||||||
|
|
||||||
rows: list[Row]
|
rows: list[Row]
|
||||||
complete: bool
|
complete: bool
|
||||||
@@ -135,6 +143,7 @@ class Listing:
|
|||||||
sweep: dict[str, Any] | None = None
|
sweep: dict[str, Any] | None = None
|
||||||
asks: str = ""
|
asks: str = ""
|
||||||
narrowed: bool = False
|
narrowed: bool = False
|
||||||
|
swept: bool = False
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- modules
|
# --------------------------------------------------------------- modules
|
||||||
@@ -521,8 +530,9 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
||||||
the `sweep` to keep, and the next pull, handed it back, continues from
|
the `sweep` to keep, and the next pull, handed it back, continues from
|
||||||
its series and cursor; the sweep is complete when it reaches the end,
|
its series and cursor; the sweep is complete when it reaches the end,
|
||||||
and `ids` then hold every id it listed. A read at the reader's `ceiling`
|
and `ids` then hold every id it listed, and the listing is `swept`. A
|
||||||
is not swept: the host answers nothing past it."""
|
read at the reader's `ceiling` is not swept: the host answers nothing
|
||||||
|
past it."""
|
||||||
limits = limits or Limits()
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
asks = series(module, ask, facets)
|
asks = series(module, ask, facets)
|
||||||
@@ -539,7 +549,8 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
never = reads is not None and reads.complete == "never"
|
never = reads is not None and reads.complete == "never"
|
||||||
ceiling = reads.ceiling if reads is not None else None
|
ceiling = reads.ceiling if reads is not None else None
|
||||||
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
||||||
if sweep and sweep.get("asks") == key:
|
resumed = bool(sweep and sweep.get("asks") == key)
|
||||||
|
if sweep and resumed:
|
||||||
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
|
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
|
||||||
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
|
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
|
||||||
whole = bool(sweep.get("whole", True))
|
whole = bool(sweep.get("whole", True))
|
||||||
@@ -591,7 +602,8 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
|
|
||||||
listed = list(dict.fromkeys(earlier + list(rows)))
|
listed = list(dict.fromkeys(earlier + list(rows)))
|
||||||
if not stop:
|
if not stop:
|
||||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words)
|
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
|
||||||
|
resumed and whole)
|
||||||
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
||||||
module.NAME, slug, pages, stop)
|
module.NAME, slug, pages, stop)
|
||||||
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
||||||
@@ -674,10 +686,12 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
|||||||
"""Record a read whose rows are stored: `ok` (the run's marker),
|
"""Record a read whose rows are stored: `ok` (the run's marker),
|
||||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||||
against), the sweep to continue or none, and the count for the 10%
|
against), whether it finished a sweep from earlier pulls (`swept`), the
|
||||||
guard once a read is whole."""
|
sweep to continue or none, and the count for the 10% guard once a read
|
||||||
|
is whole."""
|
||||||
state[f"{prefix}:ok:{target}"] = run
|
state[f"{prefix}:ok:{target}"] = run
|
||||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||||
|
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||||
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
||||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||||
|
|||||||
+78
-3
@@ -206,14 +206,17 @@ def _row(job_id: str) -> dict[str, object]:
|
|||||||
"description": "", "remote": None, "salary_text": ""}
|
"description": "", "remote": None, "salary_text": ""}
|
||||||
|
|
||||||
|
|
||||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False) -> None:
|
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False,
|
||||||
|
swept: bool = False) -> None:
|
||||||
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
|
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
|
||||||
with `filtered`, a read the board's filter narrowed."""
|
with `filtered`, a read the board's filter narrowed, with `swept` one that
|
||||||
|
finished a sweep across pulls."""
|
||||||
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||||
if isinstance(answer, Exception):
|
if isinstance(answer, Exception):
|
||||||
raise answer
|
raise answer
|
||||||
rows, complete = answer # type: ignore[misc]
|
rows, complete = answer # type: ignore[misc]
|
||||||
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered)
|
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered,
|
||||||
|
swept=swept)
|
||||||
|
|
||||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||||
|
|
||||||
@@ -302,6 +305,78 @@ def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on(
|
|||||||
assert _open(acme) == {"workable:acme:1"}
|
assert _open(acme) == {"workable:acme:1"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_swept_read_closes_only_what_two_sweeps_in_a_row_missed(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(i) for i in "123"], True))
|
||||||
|
_pull(acme)
|
||||||
|
|
||||||
|
_answer(monkeypatch, ([_row("1")], True), swept=True)
|
||||||
|
result = _pull(acme)
|
||||||
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
||||||
|
assert result["closed"] == 0
|
||||||
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == [
|
||||||
|
"workable:acme:2", "workable:acme:3"]
|
||||||
|
|
||||||
|
# 3 came back, 2 did not: 2 has two strikes, 3 starts over.
|
||||||
|
_answer(monkeypatch, ([_row("1"), _row("3")], True), swept=True)
|
||||||
|
result = _pull(acme)
|
||||||
|
assert _open(acme) == {"workable:acme:1", "workable:acme:3"}
|
||||||
|
assert result["closed"] == 1
|
||||||
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_incomplete_read_between_two_sweeps_keeps_the_strike(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
||||||
|
_pull(acme)
|
||||||
|
_answer(monkeypatch, ([_row("1")], True), swept=True)
|
||||||
|
_pull(acme)
|
||||||
|
_answer(monkeypatch, ([_row("1")], False))
|
||||||
|
_pull(acme)
|
||||||
|
_answer(monkeypatch, ValueError("down"))
|
||||||
|
_pull(acme)
|
||||||
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == ["workable:acme:2"]
|
||||||
|
|
||||||
|
_answer(monkeypatch, ([_row("1")], True), swept=True)
|
||||||
|
_pull(acme)
|
||||||
|
assert _open(acme) == {"workable:acme:1"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filtered_swept_read_keeps_the_roles_acted_on(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
|
||||||
|
_pull(acme)
|
||||||
|
uid = {r.source_id: r.uid for r in acme.roles()}
|
||||||
|
agent = acme.save_agent("Hunt")
|
||||||
|
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
|
||||||
|
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
|
||||||
|
|
||||||
|
_answer(monkeypatch, ([_row("1")], True), filtered=True, swept=True)
|
||||||
|
assert _pull(acme)["closed"] == 0
|
||||||
|
assert _open(acme) == {f"workable:acme:{i}" for i in "1234"}
|
||||||
|
|
||||||
|
result = _pull(acme)
|
||||||
|
assert result["closed"] == 1
|
||||||
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
||||||
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == [
|
||||||
|
"workable:acme:2", "workable:acme:3"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_narrowed_swept_read_closes_nothing_and_keeps_the_record(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
||||||
|
_pull(acme)
|
||||||
|
_answer(monkeypatch, ([_row("1")], True), swept=True)
|
||||||
|
_pull(acme)
|
||||||
|
|
||||||
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||||
|
return _runner.Listing([_row("1")], True, ["1"], swept=True, narrowed=True)
|
||||||
|
|
||||||
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||||
|
assert _pull(acme)["closed"] == 0
|
||||||
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == ["workable:acme:2"]
|
||||||
|
|
||||||
|
|
||||||
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
|
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
|
||||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
||||||
|
|||||||
@@ -269,6 +269,28 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
|
|||||||
assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4]
|
assert [index for index, _ in module.asked] == [0, 1, 2, 3, 4]
|
||||||
assert "ats:sweep:fake:acme" not in state
|
assert "ats:sweep:fake:acme" not in state
|
||||||
assert json.loads(state["ats:count:fake:acme"])["count"] == 5
|
assert json.loads(state["ats:count:fake:acme"])["count"] == 5
|
||||||
|
assert state["ats:swept:fake:acme"] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_read_in_one_pull_is_not_swept() -> None:
|
||||||
|
pages = [["1"], ["2"]]
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
listing = _runner.read(state, "ats", "fake:acme", _paged(pages, total=2), "acme",
|
||||||
|
_pages_web(pages), limits=NO_DELAY)
|
||||||
|
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||||
|
assert (listing.complete, listing.swept, state["ats:swept:fake:acme"]) == (True, False, "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_sweep_restarted_by_a_changed_ask_is_not_swept() -> None:
|
||||||
|
pages = [["1"], ["2"], ["3"]]
|
||||||
|
module = _paged(pages, filters=Filters(country="many"))
|
||||||
|
web, state = _pages_web(pages), {}
|
||||||
|
limits = _runner.Limits(max_pages=2, page_delay=0)
|
||||||
|
first = _runner.read(state, "ats", "t", module, "acme", web, Ask(), limits=limits)
|
||||||
|
_runner.stamp(state, "ats", "t", "r", first, first.ids)
|
||||||
|
again = _runner.read(state, "ats", "t", module, "acme", web, Ask(countries=("DK",)),
|
||||||
|
limits=NO_DELAY)
|
||||||
|
assert (again.complete, again.swept) == (True, False)
|
||||||
|
|
||||||
|
|
||||||
def test_a_sweep_starts_again_when_the_ask_changed() -> None:
|
def test_a_sweep_starts_again_when_the_ask_changed() -> None:
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
"""Two strikes for swept boards, end to end: a fake offset-paged host read
|
||||||
|
across pulls by the real `ATSSource`, run by the real `pull` batch and
|
||||||
|
settle. Roles closing on the host between pulls shift later roles back past
|
||||||
|
the saved cursor, so a finished sweep misses a role that is still open: it
|
||||||
|
closes only when two finished sweeps in a row missed it. No network."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import types
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, ClassVar
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from pipeline import Tally # noqa: E402
|
||||||
|
from app.jobs import pull # noqa: E402
|
||||||
|
from app.storage import Store # noqa: E402
|
||||||
|
from plugins import ats, readers # noqa: E402
|
||||||
|
from plugins.readers import Reads, _runner # noqa: E402
|
||||||
|
|
||||||
|
PAGE = 2
|
||||||
|
BOARD = "fake:acme"
|
||||||
|
|
||||||
|
|
||||||
|
class FakeConfig:
|
||||||
|
sources: ClassVar[dict[str, object]] = {}
|
||||||
|
searches: ClassVar[list[object]] = []
|
||||||
|
|
||||||
|
|
||||||
|
class Host:
|
||||||
|
"""An offset-paged board over `live`, two roles a page, stating its
|
||||||
|
current total."""
|
||||||
|
|
||||||
|
def __init__(self, live: list[str]) -> None:
|
||||||
|
self.live = live
|
||||||
|
|
||||||
|
def module(self) -> Any:
|
||||||
|
module: Any = types.ModuleType("fake")
|
||||||
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||||
|
module.slug_ok = lambda slug: slug.isalnum()
|
||||||
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||||
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||||
|
module.READS = Reads(scope="board", listing="paged", text="listing", complete="stated",
|
||||||
|
places="many", robots="test", page_size=PAGE)
|
||||||
|
|
||||||
|
def page(get: Any, slug: str, cursor: Any, ask: Any) -> Any:
|
||||||
|
offset = int(cursor or 0)
|
||||||
|
ids = self.live[offset:offset + PAGE]
|
||||||
|
more = offset + PAGE < len(self.live)
|
||||||
|
return ids, [{"id": i} for i in ids], len(self.live), offset + PAGE if more else None
|
||||||
|
|
||||||
|
module.page = page
|
||||||
|
module.rows = lambda ids, slug: [
|
||||||
|
{"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
|
||||||
|
"places": [{"text": "Berlin", "remote": None}]} for i in ids]
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def host(monkeypatch: pytest.MonkeyPatch) -> Host:
|
||||||
|
fake = Host([str(i) for i in range(1, 7)])
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "fake", fake.module())
|
||||||
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||||
|
monkeypatch.setattr(ats, "MAX_PAGES", 1)
|
||||||
|
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b"")
|
||||||
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def store(tmp_path: Path, host: Host) -> Store:
|
||||||
|
found = Store(tmp_path / "roledesk.db")
|
||||||
|
found.add_subscription("ats", BOARD, label="Acme", company="Acme", board="fake")
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def _pull(store: Store) -> dict[str, Any]:
|
||||||
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||||
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||||
|
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def _open(store: Store) -> list[str]:
|
||||||
|
return sorted(r.source_id.rpartition(":")[2] for r in store.roles())
|
||||||
|
|
||||||
|
|
||||||
|
def _missing(store: Store) -> list[str] | None:
|
||||||
|
kept = store.state.get(f"ats:missing:{BOARD}")
|
||||||
|
return None if kept is None else sorted(i.rpartition(":")[2] for i in json.loads(kept))
|
||||||
|
|
||||||
|
|
||||||
|
def _sweep(store: Store) -> dict[str, Any]:
|
||||||
|
"""Pull until a sweep finishes, three pulls of one page each."""
|
||||||
|
result: dict[str, Any] = {}
|
||||||
|
for _ in range(3):
|
||||||
|
result = _pull(store)
|
||||||
|
if store.state.get(f"ats:complete:{BOARD}"):
|
||||||
|
break
|
||||||
|
assert store.state.get(f"ats:complete:{BOARD}") == "1"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _whole_read(store: Store, monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||||
|
monkeypatch.setattr(ats, "MAX_PAGES", 10)
|
||||||
|
result = _pull(store)
|
||||||
|
monkeypatch.setattr(ats, "MAX_PAGES", 1)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_one_sweep_missed_stays_open_and_two_close_it(
|
||||||
|
store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_whole_read(store, monkeypatch)
|
||||||
|
assert _open(store) == list("123456")
|
||||||
|
assert store.state[f"ats:swept:{BOARD}"] == ""
|
||||||
|
|
||||||
|
# Sweep one: role 1 closes on the host after page one, role 3 shifts back
|
||||||
|
# past the cursor and is not listed, though it is open.
|
||||||
|
_pull(store)
|
||||||
|
assert store.state[f"ats:complete:{BOARD}"] == ""
|
||||||
|
host.live.remove("1")
|
||||||
|
_sweep(store)
|
||||||
|
assert store.state[f"ats:swept:{BOARD}"] == "1"
|
||||||
|
assert json.loads(store.state[f"ats:ids:{BOARD}"]) == [
|
||||||
|
f"{BOARD}:{i}" for i in "12456"]
|
||||||
|
assert _open(store) == list("123456")
|
||||||
|
assert _missing(store) == ["3"]
|
||||||
|
|
||||||
|
# Sweep two lists 3 again and misses 1, which the host closed: one strike.
|
||||||
|
_sweep(store)
|
||||||
|
assert _open(store) == list("123456")
|
||||||
|
assert _missing(store) == ["1"]
|
||||||
|
|
||||||
|
# Sweep three misses 1 again: the second strike closes it, only it.
|
||||||
|
result = _sweep(store)
|
||||||
|
assert _open(store) == list("23456")
|
||||||
|
assert result["closed"] == 1
|
||||||
|
assert _missing(store) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_that_comes_back_between_sweeps_loses_its_strike(
|
||||||
|
store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_whole_read(store, monkeypatch)
|
||||||
|
host.live.remove("6")
|
||||||
|
_sweep(store)
|
||||||
|
assert _open(store) == list("123456") and _missing(store) == ["6"]
|
||||||
|
|
||||||
|
host.live.append("6")
|
||||||
|
_sweep(store)
|
||||||
|
assert _open(store) == list("123456") and _missing(store) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_complete_read_that_was_not_swept_clears_the_record(
|
||||||
|
store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_whole_read(store, monkeypatch)
|
||||||
|
host.live.remove("6")
|
||||||
|
_sweep(store)
|
||||||
|
assert _missing(store) == ["6"]
|
||||||
|
|
||||||
|
host.live.remove("5")
|
||||||
|
result = _whole_read(store, monkeypatch)
|
||||||
|
assert store.state[f"ats:swept:{BOARD}"] == ""
|
||||||
|
assert _missing(store) is None
|
||||||
|
# Not swept, so as before: one complete read closes what it did not list.
|
||||||
|
assert _open(store) == list("1234") and result["closed"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_sweep_that_is_not_finished_leaves_the_record_alone(
|
||||||
|
store: Store, host: Host, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_whole_read(store, monkeypatch)
|
||||||
|
host.live.remove("6")
|
||||||
|
_sweep(store)
|
||||||
|
assert _missing(store) == ["6"]
|
||||||
|
_pull(store)
|
||||||
|
assert store.state[f"ats:complete:{BOARD}"] == ""
|
||||||
|
assert _missing(store) == ["6"]
|
||||||
|
assert _open(store) == list("123456")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_listing_says_swept_only_for_a_sweep_carried_over(host: Host) -> None:
|
||||||
|
module = host.module()
|
||||||
|
limits = _runner.Limits(max_pages=1, page_delay=0)
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
first = _runner.read(state, "ats", BOARD, module, "acme", lambda url, **kw: b"",
|
||||||
|
limits=limits)
|
||||||
|
assert (first.complete, first.swept) == (False, False)
|
||||||
|
_runner.stamp(state, "ats", BOARD, "r", first, first.ids)
|
||||||
|
assert state[f"ats:swept:{BOARD}"] == ""
|
||||||
|
while not first.complete:
|
||||||
|
first = _runner.read(state, "ats", BOARD, module, "acme", lambda url, **kw: b"",
|
||||||
|
limits=limits)
|
||||||
|
_runner.stamp(state, "ats", BOARD, "r", first, first.ids)
|
||||||
|
assert first.swept and state[f"ats:swept:{BOARD}"] == "1"
|
||||||
|
whole = _runner.read({}, "ats", BOARD, module, "acme", lambda url, **kw: b"",
|
||||||
|
limits=_runner.Limits(max_pages=10, page_delay=0))
|
||||||
|
assert (whole.complete, whole.swept) == (True, False)
|
||||||
Reference in new issue
Block a user