Keep the sweep of a reader whose cursor is a tuple, so Workday and SuccessFactors boards over 100 pages finish across pulls, and end a read incomplete on a page with nothing new while a cursor follows, whatever the total, which SuccessFactors now hands back on a repeated page
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
85aed0807b
commit
7f37270eb3
6 files changed
+219
-18
No files matched your search
@@ -1,15 +1,17 @@
|
||||
"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved
|
||||
career-site pages. No network. The fixtures are trimmed pages of Vestas and
|
||||
career-site pages, and against written ones through a sweep and a pull with
|
||||
its settle. No network. The fixtures are trimmed pages of Vestas and
|
||||
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
|
||||
with a path prefix), and the job pages of Vestas (`addressCountry`),
|
||||
Systematic (`streetAddress`) and Novo Nordisk (no microdata)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -18,13 +20,28 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from pipeline import Tally # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import successfactors # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
|
||||
class FakeConfig:
|
||||
sources: ClassVar[dict[str, object]] = {}
|
||||
searches: ClassVar[list[object]] = []
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, Any]:
|
||||
"""One pull of every board, batches and settle, as `test_sweep_strikes`."""
|
||||
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _read(name: str) -> str:
|
||||
return (FIXTURES / name).read_text(encoding="utf-8")
|
||||
|
||||
@@ -264,6 +281,70 @@ def test_a_site_that_ignores_startrow_does_not_loop(monkeypatch: pytest.MonkeyPa
|
||||
assert len(asked) == 2
|
||||
|
||||
|
||||
def _untotalled(ids: list[int]) -> str:
|
||||
"""A table page that states no total."""
|
||||
return _table_page(ids, 0).split("</span>", 1)[1]
|
||||
|
||||
|
||||
def test_the_repeated_page_hands_back_a_cursor_with_no_results() -> None:
|
||||
text = _untotalled([1_000_000_001, 1_000_000_002])
|
||||
_decoded, results, total, cursor = successfactors.page(
|
||||
lambda url, **kw: text.encode(), "careers.acme.com", [2, "1000000001"])
|
||||
assert (results, total, cursor) == ([], None, (2, "1000000001"))
|
||||
|
||||
|
||||
def test_a_site_that_ignores_startrow_and_states_no_total_closes_nothing(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""The repeated first page used to end the read with no cursor, which a
|
||||
site that states no total made complete, and the settle closed every
|
||||
role past the first page."""
|
||||
store = Store(tmp_path / "roledesk.db")
|
||||
store.add_subscription("ats", "successfactors:careers.acme.com", label="Acme",
|
||||
company="Acme", board="successfactors")
|
||||
ids = [1_000_000_000 + n for n in range(4)]
|
||||
site = {"ignores_startrow": False}
|
||||
|
||||
def fake_get(client, url, host, slug, **kw):
|
||||
start = 0 if site["ignores_startrow"] else int(dict(httpx.URL(url).params)["startrow"])
|
||||
return _untotalled(ids[start:start + 2]).encode()
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
_pull(store)
|
||||
assert store.state["ats:complete:successfactors:careers.acme.com"] == "1"
|
||||
assert len(store.roles()) == 4
|
||||
|
||||
site["ignores_startrow"] = True
|
||||
result = _pull(store)
|
||||
assert store.state["ats:complete:successfactors:careers.acme.com"] == ""
|
||||
assert (result["incomplete"], result["closed"], len(store.roles())) == (1, 0, 4)
|
||||
|
||||
|
||||
def test_a_site_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The cursor `(start row, first id)` is a tuple, kept in state as a
|
||||
list. The sweep is kept, and the next pull reads on from it."""
|
||||
asked = _site(monkeypatch, 5, 2)
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
||||
target = "successfactors:careers.acme.com"
|
||||
state: dict[str, str] = {}
|
||||
|
||||
def pull_once() -> list[Any]:
|
||||
source = ats.ATSSource({"boards": [{"url": target, "label": "Acme", "params": {}}],
|
||||
"run": "r"})
|
||||
source.state = state
|
||||
return list(source.fetch())
|
||||
|
||||
assert len(pull_once()) == 4
|
||||
assert state[f"ats:complete:{target}"] == ""
|
||||
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [4, "1000000002"]
|
||||
assert len(pull_once()) == 1
|
||||
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
|
||||
assert json.loads(state[f"ats:ids:{target}"]) == [
|
||||
f"{target}:{1_000_000_000 + n}" for n in range(5)]
|
||||
assert [int(u.rsplit("=", 1)[1]) for u in asked] == [0, 2, 4]
|
||||
|
||||
|
||||
def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_site(monkeypatch, 0, 10)
|
||||
assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type]
|
||||
|
||||
Reference in new issue
Block a user