plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
178 lines
7.8 KiB
Python
178 lines
7.8 KiB
Python
"""The HR Manager host module against saved feeds (STS Group's, trimmed to
|
|
five roles, and NNIT's, which had none), 2026-09-29. No network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.jobs.discover import boards_in # noqa: E402
|
|
from plugins import ats, readers # noqa: E402
|
|
from plugins.readers import hrmanager # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
|
EMPTY = (FIXTURES / "hrmanager-empty.xml").read_bytes()
|
|
FEED = "https://api.hr-manager.net/JobPortal.svc/stsgroup_tr/PositionList/rss/?incads=true"
|
|
|
|
|
|
def _feed(skip: int, slug: str = "stsgroup_tr") -> str:
|
|
return (f"https://api.hr-manager.net/JobPortal.svc/{slug}/PositionList/rss/"
|
|
f"?incads=true&skip={skip}&take=100")
|
|
|
|
|
|
def test_rows_read_the_advertisement_and_the_publication_date() -> None:
|
|
rows = hrmanager.parse(BOARD, "stsgroup_tr")
|
|
assert [r["source_id"] for r in rows] == ["143857", "143844", "143854", "143849", "143848"]
|
|
first = rows[0]
|
|
assert first["title"] == "Commissioning Engineer- T&D" # `&` in the XML
|
|
assert first["url"] == ("https://candidate.hr-manager.net/ApplicationInit.aspx"
|
|
"?cid=2523&ProjectId=143857&DepartmentId=18958&MediaId=5")
|
|
assert first["description"].startswith("About the Position We have an exciting opportunity")
|
|
assert "<" not in first["description"] and "&" not in first["description"]
|
|
assert "T&D Division" in first["description"]
|
|
assert first["posted"] is not None and first["posted"].isoformat() == \
|
|
"2025-03-18T17:31:36+01:00"
|
|
# The feed names no place, department, pay or way of working.
|
|
assert (first["location"], first["department"], first["salary_text"], first["remote"]) == \
|
|
("", "", "", None)
|
|
|
|
|
|
def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_feed() -> None:
|
|
assert hrmanager.parse(EMPTY, "nnit") == []
|
|
with pytest.raises(ValueError, match="not an HR Manager feed"):
|
|
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
|
|
|
|
|
|
def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None:
|
|
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
|
|
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
|
|
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
|
|
b"</channel></rss>")
|
|
assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"]
|
|
|
|
|
|
# ------------------------------------------------------------- paging
|
|
|
|
def _paged(answers: dict[str, bytes], asked: list[str]) -> Any:
|
|
def get(url: str, *, method: str = "GET", payload: Any = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
asked.append(url)
|
|
return answers[url]
|
|
return get
|
|
|
|
|
|
def test_the_page_moves_on_by_what_it_returned_and_ends_on_an_empty_one() -> None:
|
|
asked: list[str] = []
|
|
get = _paged({_feed(0): BOARD, _feed(5): EMPTY}, asked)
|
|
data, results, total, cursor = hrmanager.page(get, "stsgroup_tr", None)
|
|
assert len(results) == 5 and total is None and cursor == 5
|
|
assert len(hrmanager.rows(data, "stsgroup_tr")) == 5
|
|
_, results, total, cursor = hrmanager.page(get, "stsgroup_tr", cursor)
|
|
assert results == [] and total is None and cursor is None
|
|
assert asked == [_feed(0), _feed(5)]
|
|
|
|
|
|
@pytest.fixture
|
|
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
|
answers: dict[str, Any] = {}
|
|
asked: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
|
www_twin=False):
|
|
asked.append(url)
|
|
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
|
raise ats._Refused(url)
|
|
answer = answers[url]
|
|
if isinstance(answer, BaseException):
|
|
raise answer
|
|
return answer
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
return {"answers": answers, "asked": asked}
|
|
|
|
|
|
def test_a_board_is_complete_when_the_read_ends_on_an_empty_page(web: dict[str, Any]) -> None:
|
|
web["answers"] |= {_feed(0): BOARD, _feed(5): EMPTY}
|
|
with httpx.Client() as client:
|
|
rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr")
|
|
assert len(rows) == 5 and complete
|
|
assert web["asked"] == [_feed(0), _feed(5)]
|
|
|
|
|
|
def test_a_board_with_no_roles_is_complete_and_empty(web: dict[str, Any]) -> None:
|
|
web["answers"][_feed(0, "seaborg_tr")] = EMPTY
|
|
with httpx.Client() as client:
|
|
assert ats._fetch_rows(client, "hrmanager:seaborg_tr") == ([], True)
|
|
assert web["asked"] == [_feed(0, "seaborg_tr")]
|
|
|
|
|
|
def test_a_page_the_host_repeats_leaves_the_board_incomplete(web: dict[str, Any]) -> None:
|
|
"""A host that ignored `skip` would answer the same roles again."""
|
|
web["answers"] |= {_feed(0): BOARD, _feed(5): BOARD}
|
|
with httpx.Client() as client:
|
|
rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr")
|
|
assert len(rows) == 5 and not complete
|
|
|
|
|
|
def test_a_429_after_the_first_page_ends_the_read_incomplete(web: dict[str, Any]) -> None:
|
|
request = httpx.Request("GET", _feed(5))
|
|
web["answers"] |= {_feed(0): BOARD, _feed(5): httpx.HTTPStatusError(
|
|
"429", request=request, response=httpx.Response(429, request=request))}
|
|
with httpx.Client() as client:
|
|
rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr")
|
|
assert len(rows) == 5 and not complete
|
|
|
|
|
|
# ------------------------------------------------------- slugs and hosts
|
|
|
|
def test_the_module_is_registered_as_a_paged_host() -> None:
|
|
assert readers.HOSTS["hrmanager"] is hrmanager and ats.HOSTS["hrmanager"] == "HR Manager"
|
|
assert hrmanager.NAME == "hrmanager" and not getattr(hrmanager, "KEEP_CASE", False)
|
|
assert hasattr(hrmanager, "page") and not hasattr(hrmanager, "detail")
|
|
|
|
|
|
def test_the_urls_and_the_allowed_host() -> None:
|
|
assert hrmanager.careers_url("seaborg_tr") == \
|
|
"https://candidate.hr-manager.net/vacancies/list.aspx?customer=seaborg_tr"
|
|
assert hrmanager.robots_page("stsgroup_tr") == _feed(0)
|
|
allow = hrmanager.allowed("stsgroup_tr")
|
|
assert allow("api.hr-manager.net") and allow("API.HR-MANAGER.NET")
|
|
for host in ("candidate.hr-manager.net", "hr-manager.net", "api.hr-manager.net.evil.example",
|
|
"localhost"):
|
|
assert not allow(host)
|
|
|
|
|
|
@pytest.mark.parametrize("slug", ["seaborg_tr", "kpmg", "FlyingTiger", "a-b", "9x"])
|
|
def test_a_customer_is_one_path_segment(slug: str) -> None:
|
|
assert ats.is_known_target(f"hrmanager:{slug}")
|
|
|
|
|
|
@pytest.mark.parametrize("slug", [
|
|
"", "a@b", "a#b", "a?b", "a&skip=1", "..", ".", "a%2fb", "%2f", "a/b", "a:443", "a b",
|
|
"acme\n", " acme", "acme ", "127.0.0.1", "a.b", "_x", "-x", "a\\b", "évil",
|
|
])
|
|
def test_a_hostile_slug_is_refused(slug: str) -> None:
|
|
assert not hrmanager.slug_ok(slug)
|
|
assert not ats.is_known_target(f"hrmanager:{slug}")
|
|
|
|
|
|
def test_discovery_finds_the_customer() -> None:
|
|
page = ('<a href="https://candidate.hr-manager.net/vacancies/list.aspx?customer=seaborg_tr">Jobs</a>'
|
|
'<a href="https://candidate.hr-manager.net/Vacancies/List.aspx?customer=KPMG">KPMG</a>'
|
|
'<a href="https://candidate.hr-manager.net/vacancies/list.aspx?_sp=3a6f.1535&customer=nnit">N</a>'
|
|
'<a href="https://api.hr-manager.net/JobPortal.svc/flyingtiger/PositionList/rss/">R</a>'
|
|
'<a href="https://candidate.hr-manager.net/ApplicationInit.aspx?cid=1917">A</a>')
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [
|
|
("hrmanager", "seaborg_tr"), ("hrmanager", "kpmg"), ("hrmanager", "nnit"),
|
|
("hrmanager", "flyingtiger")]
|