Files
jobboard/tests/test_readers_jobbank.py
T

258 lines
12 KiB
Python

"""The Akademikernes Jobbank reader: the RSS feed of a search, 100 items a page,
and the JSON-LD of an ad's own page for its text. No network: `get` is a stub."""
from __future__ import annotations
import sys
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urlsplit
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, _runner, jobbank # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
REMOTE = (FIXTURES / "jobbank.xml").read_bytes()
EMPTY = (FIXTURES / "jobbank-empty.xml").read_bytes()
AD = (FIXTURES / "jobbank-detail.html").read_bytes()
def _rows(body: bytes = PLACES, ask: Ask | None = None) -> list[dict[str, Any]]:
decoded, _results, _total, _cursor = jobbank.page(
lambda url, **kw: body, "", None, ask or Ask(params={"q": "x"}))
return list(jobbank.rows(decoded, ""))
def _feed(items: int) -> bytes:
body = "".join(f"<item><title>Role {n}</title><description>Fuldtidsjob hos Acme A/S, "
f"Fyn (Ansøgningsfrist: løbende)</description>"
f"<link>https://jobbank.dk/job/{n + 1}/acme/role</link></item>"
for n in range(items))
return f"<rss><channel><title>Jobbank</title>{body}</channel></rss>".encode()
def _item(description: str) -> bytes:
return (f"<rss><channel><item><title>Role</title><description><![CDATA[{description}]]>"
"</description><link>https://jobbank.dk/job/7/acme/role</link></item>"
"</channel></rss>").encode()
def test_it_is_a_search_reader_with_the_robots_decision() -> None:
assert readers.PROVIDERS["jobbank"] is jobbank
reads = jobbank.READS
assert (reads.scope, reads.page_size, reads.ceiling, reads.text, reads.complete) == (
"search", 100, 2000, "detail", "never")
assert "/job/" in reads.robots and "/fwd/webjob/" in reads.robots
assert jobbank.allowed("")("jobbank.dk") and not jobbank.allowed("")("www.jobbank.dk")
def test_the_regions_are_the_forms_and_the_remote_words_are_its_two() -> None:
assert jobbank.REGIONS["2"] == "Storkøbenhavn" and len(jobbank.REGIONS) == 19
assert set(jobbank.REMOTE) == {"helt", "delvist"}
assert jobbank.PARAMS == {"q": "query", "region": "place", "remote": "remote"}
def test_it_reads_title_company_regions_date_and_deadline() -> None:
first, leo, phd, orsted, *_ = _rows()
assert first["source_id"] == "3111914"
assert first["url"] == ("https://jobbank.dk/job/3111914/rigsarkivet-staten/"
"data-scientister-og-historiske")
assert first["title"].startswith("Data scientister og historiske")
assert first["company"] == "Rigsarkivet - Statens Arkiver"
assert [p["text"] for p in first["places"]] == ["Storkøbenhavn", "Fyn"]
assert first["location"] == "Storkøbenhavn" and first["country"] == "DK"
assert first["posted"].date().isoformat() == "2026-09-04"
assert (first["jobType"], first["applicationDeadline"]) == ("Fuldtidsjob", "2026-10-07")
assert "applicationDeadline" not in leo and leo["company"] == "LEO PHARMA A/S"
assert phd["jobType"] == "Ph.d. & Postdoc"
assert [p["text"] for p in orsted["places"]] == [
"Storkøbenhavn", "Sydvestjylland (Esbjerg)", "Østjylland (Aarhus)"]
assert all("description" not in row and "teaser" not in row for row in _rows())
def test_a_comma_in_the_employer_stays_with_the_employer() -> None:
row, = jobbank.rows(jobbank.page(
lambda url, **kw: _item("Fuldtidsjob hos AgainBio (Luabio Aps, Secondcircle Aps), "
"Storkøbenhavn (Ansøgningsfrist: løbende)"),
"", None, Ask())[0], "")
assert row["company"] == "AgainBio (Luabio Aps, Secondcircle Aps)"
assert [p["text"] for p in row["places"]] == ["Storkøbenhavn"]
def test_an_employer_with_no_region_after_it_keeps_its_whole_name() -> None:
row, = _rows(_item("Fuldtidsjob hos Acme, Inc. (Ansøgningsfrist: løbende)"))
assert row["company"] == "Acme, Inc." and row["places"] == []
def test_the_country_is_stated_only_when_the_regions_name_one() -> None:
by_id = {row["source_id"]: row for row in _rows()}
assert by_id["3111914"]["country"] == "DK"
assert "country" not in by_id["3119220"] # Norway and "øvrige"
assert "country" not in by_id["3112442"] # Denmark, Sweden and Norway
assert _rows(_item("Fuldtidsjob hos Elopak, Udlandet (Norge) "
"(Ansøgningsfrist: løbende)"))[0]["country"] == "NO"
def test_regions_the_feed_cut_off_name_no_place_and_no_country() -> None:
cowi = next(row for row in _rows() if row["company"] == "COWI A/S")
assert [p["text"] for p in cowi["places"]] == ["Fyn", "Sydøstjylland", "Østjylland (Aarhus)"]
assert "country" not in cowi
row, = _rows(_item("Fuldtidsjob hos DANFOSS A/S, Storkøbenhavn +1 andre "
"(Ansøgningsfrist: løbende)"))
assert [p["text"] for p in row["places"]] == ["Storkøbenhavn"] and "country" not in row
def test_a_search_for_fully_remote_roles_marks_its_places_remote() -> None:
asked = Ask(params={"q": "x", "remote": "helt"})
assert all(p["remote"] is True for row in _rows(REMOTE, asked) for p in row["places"])
assert all(p["remote"] is None for row in _rows(REMOTE) for p in row["places"])
partly = Ask(params={"q": "x", "remote": "delvist"})
assert all(p["remote"] is None for row in _rows(REMOTE, partly) for p in row["places"])
def test_it_asks_for_the_query_the_region_the_remote_word_and_the_page() -> None:
asked: list[str] = []
def get(url: str, **kwargs: Any) -> bytes:
asked.append(url)
return _feed(3)
jobbank.page(get, "", None, Ask(params={"q": "data director", "region": "2",
"remote": "helt"}))
parts = urlsplit(asked[0])
assert (parts.scheme, parts.netloc, parts.path) == ("https", "jobbank.dk", "/job/rss/")
assert parse_qs(parts.query) == {"key": ["data director"], "amt": ["2"],
"fjernarbejde": ["helt"], "page": ["1"]}
jobbank.page(get, "", 4, Ask(params={"q": "x"}))
assert "amt" not in asked[1] and "fjernarbejde" not in asked[1] and "page=4" in asked[1]
jobbank.page(get, "", None, Ask(query="y", places=("13",), params={"q": "x"}))
assert parse_qs(urlsplit(asked[2]).query)["key"] == ["y"]
assert parse_qs(urlsplit(asked[2]).query)["amt"] == ["13"]
@pytest.mark.parametrize("params", [{"q": "x", "region": "99"}, {"q": "x", "remote": "ja"}])
def test_a_region_or_remote_word_it_does_not_know_is_not_requested(params: dict[str, str]) -> None:
def get(url: str, **kwargs: Any) -> bytes:
raise AssertionError("no request")
with pytest.raises(ValueError, match="jobbank"):
jobbank.page(get, "", None, Ask(params=params))
@pytest.mark.parametrize(("items", "cursor", "after"), [
(100, None, 2), # a full page has a next
(99, None, None), # a short page is the end
(0, None, None),
(100, 20, None), # the twentieth page reaches the 2,000 ceiling
(100, 19, 20),
])
def test_it_pages_until_a_short_page_or_the_ceiling(
items: int, cursor: int | None, after: int | None) -> None:
_decoded, results, total, nxt = jobbank.page(
lambda url, **kw: _feed(items), "", cursor, Ask(params={"q": "x"}))
assert (len(results), total, nxt) == (items, None, after)
def test_an_empty_page_ends_the_list() -> None:
decoded, results, total, nxt = jobbank.page(
lambda url, **kw: EMPTY, "", 17, Ask(params={"q": "x"}))
assert (results, total, nxt, jobbank.rows(decoded, "")) == ([], None, None, [])
def test_a_dropped_item_does_not_pass_for_the_end() -> None:
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
assert len(results) == 100 and nxt == 2
rows = jobbank.rows(decoded, "")
assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
b"<!DOCTYPE html><title>Just a moment...</title><p>"])
def test_an_answer_that_is_not_a_feed_raises(body: bytes) -> None:
with pytest.raises(ValueError, match="jobbank"):
jobbank.page(lambda url, **kw: body, "", None, Ask())
def test_saved_params_keep_the_query_a_real_region_and_a_remote_word() -> None:
assert jobbank.clean_params({"q": "x"}) == {"q": "x"}
assert jobbank.clean_params({"q": "x", "region": 2, "remote": "delvist"}) == {
"q": "x", "region": "2", "remote": "delvist"}
assert jobbank.clean_params({"q": "x", "region": "", "remote": ""}) == {"q": "x"}
with pytest.raises(ValueError, match="region"):
jobbank.clean_params({"q": "x", "region": "99"})
with pytest.raises(ValueError, match="remote"):
jobbank.clean_params({"q": "x", "remote": "true"})
# ------------------------------------------------------------------ detail
def _detail(row: dict[str, Any] | None = None, body: bytes = AD) -> dict[str, Any]:
row = row or _rows()[0]
return jobbank.detail(lambda url, **kw: body, "", row) # type: ignore[arg-type]
def test_detail_takes_the_description_and_adds_the_town_to_the_regions() -> None:
found = _detail()
assert found["description"].startswith("Vil du være med til at opbygge Historisk Personregister")
assert "## Om Rigsarkivet" in found["description"] and "&aelig;" not in found["description"]
assert [p["text"] for p in found["places"]] == ["Storkøbenhavn", "Fyn", "Odense"]
assert "posted" not in found # the feed has the date
def test_detail_dates_a_role_the_feed_did_not() -> None:
row = _rows()[0] | {"posted": None}
assert _detail(row)["posted"].date().isoformat() == "2026-09-04"
def test_detail_asks_for_the_ads_own_page_only() -> None:
asked: list[str] = []
def get(url: str, **kwargs: Any) -> bytes:
asked.append(url)
return AD
row = _rows()[0]
jobbank.detail(get, "", row) # type: ignore[arg-type]
assert asked == [row["url"]]
@pytest.mark.parametrize("url", ["https://jobbank.dk/fwd/webjob/3111914/",
"https://jobbank.dk/link/3111914",
"https://example.com/job/3111914/x"])
def test_detail_never_follows_anything_but_a_job_page(url: str) -> None:
def get(url: str, **kwargs: Any) -> bytes:
raise AssertionError("no request")
with pytest.raises(ValueError, match="/job/"):
jobbank.detail(get, "", _rows()[0] | {"url": url}) # type: ignore[arg-type]
@pytest.mark.parametrize("body", [
b"<html><head><title>Just a moment...</title></head><body>Checking your browser</body>",
b'<script type="application/ld+json">{"@type": "Organization"}</script>',
b'<script type="application/ld+json">not json</script>',
b""])
def test_a_page_without_a_job_posting_is_an_error_not_an_empty_text(body: bytes) -> None:
with pytest.raises(ValueError, match="JobPosting"):
_detail(body=body)
def test_detail_finds_the_posting_among_other_blocks_and_in_a_list() -> None:
posting = '{"@type": "JobPosting", "description": "<p>Hej</p>"}'
body = ('<script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
f'<script type="application/ld+json">[{posting}]</script>').encode()
assert _detail(body=body)["description"] == "Hej"
def test_detail_without_a_town_leaves_the_places_as_they_are() -> None:
body = b'<script type="application/ld+json">{"@type": "JobPosting", "description": "x"}</script>'
assert "places" not in _detail(body=body)