Add the Jobnet search reader, which reads jobnet.dk's backend with the x-csrf header and 100 ads a page
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
018a1ad895
commit
0ed0360b09
5 files changed
+545
No files matched your search
@@ -0,0 +1,169 @@
|
||||
"""The Jobnet reader (`/bff/FindJob/Search`): 100 ads a page, page numbers
|
||||
from 1, the header `x-csrf: 1`. No network: `get` is a stub."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, jobnet # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
|
||||
OTHER = (FIXTURES / "jobnet.json").read_bytes()
|
||||
END = (FIXTURES / "jobnet-end.json").read_bytes()
|
||||
|
||||
|
||||
def _stock(total: int | None, stock: int, asked: list[tuple[str, dict[str, str]]]) -> Any:
|
||||
def get(url: str, **kwargs: Any) -> bytes:
|
||||
asked.append((url, kwargs.get("headers") or {}))
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
start = (int(query["pageNumber"][0]) - 1) * int(query["resultsPerPage"][0])
|
||||
size = int(query["resultsPerPage"][0])
|
||||
body: dict[str, Any] = {"jobAds": [
|
||||
{"jobAdId": str(n), "title": f"Role {n}", "jobAdUrl": ""}
|
||||
for n in range(start, min(start + size, stock))]}
|
||||
if total is not None:
|
||||
body["totalJobAdCount"] = total
|
||||
return json.dumps(body).encode()
|
||||
return get
|
||||
|
||||
|
||||
def _rows(body: bytes) -> list[dict[str, Any]]:
|
||||
decoded, *_ = jobnet.page(lambda url, **kw: body, "", None, Ask())
|
||||
return jobnet.rows(decoded, "")
|
||||
|
||||
|
||||
def test_it_is_a_search_reader_naming_the_redirect_and_the_header() -> None:
|
||||
assert readers.PROVIDERS["jobnet"] is jobnet
|
||||
reads = jobnet.READS
|
||||
assert (reads.scope, reads.page_size, reads.ceiling, reads.places, reads.complete) == (
|
||||
"search", 100, 10000, "one", "stated")
|
||||
assert "redirects" in reads.robots and "x-csrf" in reads.robots and "2026-10-03" in reads.robots
|
||||
assert jobnet.PARAMS == {"q": "query", "region": "place"}
|
||||
assert jobnet.allowed("")("jobnet.dk") and not jobnet.allowed("")("example.com")
|
||||
|
||||
|
||||
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
||||
first, taxi, _metal, roskilde, *_ = _rows(PLACES)
|
||||
assert first["source_id"].startswith("5e6a1854-")
|
||||
assert first["title"] == "kok til asiatisk takeaway"
|
||||
assert first["company"] == "Mywok ApS"
|
||||
assert first["country"] == "DK"
|
||||
assert first["posted"].date().isoformat() == "2026-10-03"
|
||||
assert first["description"] and "<" not in first["description"]
|
||||
assert taxi["description"].startswith("## ") # a heading stays one
|
||||
assert roskilde["description"].count("• ") == 9 # a list keeps its bullets
|
||||
assert "remote" not in first and "teaser" not in first
|
||||
|
||||
|
||||
def test_an_ad_without_its_own_link_opens_jobnets_page_for_it() -> None:
|
||||
first, _taxi, _metal, roskilde, *_ = _rows(PLACES)
|
||||
assert first["url"] == f"https://jobnet.dk/find-job/{first['source_id']}"
|
||||
assert roskilde["url"].startswith("https://candidate.hr-manager.net/")
|
||||
|
||||
|
||||
def test_the_place_is_the_municipality_else_the_postal_district_else_none() -> None:
|
||||
first, taxi, metal, *_ = _rows(PLACES)
|
||||
assert first["places"] == [{"text": "Frederiksberg", "remote": None}]
|
||||
assert taxi["places"] == [{"text": "Gladsaxe", "remote": None}] # not Søborg
|
||||
assert (metal["places"], metal["location"]) == (
|
||||
[{"text": "Rødovre", "remote": None}], "Rødovre")
|
||||
german, aau = _rows(OTHER)
|
||||
assert (german["places"], aau["places"], aau["location"]) == ([], [], "")
|
||||
|
||||
|
||||
def test_the_country_is_danish_unless_the_ad_names_another() -> None:
|
||||
german, aau = _rows(OTHER)
|
||||
assert (german["country"], aau["country"]) == ("DE", "DK")
|
||||
assert german["url"].startswith("https://germantechjobs.de/")
|
||||
assert jobnet._country("") == "DK"
|
||||
assert jobnet._country("Norge") == "NO" and jobnet._country("Sweden") == "SE"
|
||||
assert jobnet._country("Atlantis") is None
|
||||
|
||||
|
||||
def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None:
|
||||
german, _aau = _rows(OTHER)
|
||||
assert german["applicationDeadline"] == "2027-01-02T00:00:00+01:00"
|
||||
ad = json.loads(OTHER)["jobAds"][0] | {"applicationDeadline": "1900-01-01T00:00:00+01:00"}
|
||||
[row] = jobnet.rows({"jobAds": [ad]}, "")
|
||||
assert "applicationDeadline" not in row
|
||||
|
||||
|
||||
def test_an_ad_without_an_id_or_a_title_is_skipped() -> None:
|
||||
ads = json.loads(OTHER)["jobAds"]
|
||||
assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == []
|
||||
|
||||
|
||||
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
|
||||
asked: list[tuple[str, dict[str, str]]] = []
|
||||
get = _stock(250, 250, asked)
|
||||
jobnet.page(get, "", None, Ask(params={"q": "data science", "region": "Midtjylland"}))
|
||||
jobnet.page(get, "", 2, Ask(params={"q": "data", "region": ""}))
|
||||
jobnet.page(get, "", None, Ask(query="head of AI", places=("Nordjylland",)))
|
||||
assert all(headers == {"x-csrf": "1"} for _url, headers in asked)
|
||||
assert parse_qs(urlsplit(asked[0][0]).query) == {
|
||||
"searchString": ["data science"], "orderType": ["PublicationDate"],
|
||||
"resultsPerPage": ["100"], "pageNumber": ["1"], "regions": ["Midtjylland"]}
|
||||
assert "regions" not in urlsplit(asked[1][0]).query and "pageNumber=2" in asked[1][0]
|
||||
assert "searchString=head+of+AI" in asked[2][0] and "regions=Nordjylland" in asked[2][0]
|
||||
assert urlsplit(asked[0][0]).netloc == "jobnet.dk"
|
||||
assert urlsplit(asked[0][0]).path == "/bff/FindJob/Search"
|
||||
|
||||
|
||||
def test_it_states_the_total_and_counts_the_ads_it_listed() -> None:
|
||||
_decoded, found, total, cursor = jobnet.page(
|
||||
lambda url, **kw: PLACES, "", None, Ask())
|
||||
assert total == 57 and len(found) == 6 and cursor is None # a short page ends it
|
||||
assert found[0]["id"] == _rows(PLACES)[0]["source_id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("cursor", "stock", "after"), [
|
||||
(None, 250, 2), (2, 250, 3), (3, 250, None), # a short page is the end
|
||||
(99, 20000, 100), (100, 20000, None), # the ceiling: page 100 is the last
|
||||
])
|
||||
def test_it_pages_by_number_up_to_the_ceiling(
|
||||
cursor: int | None, stock: int, after: int | None) -> None:
|
||||
_decoded, _found, _total, nxt = jobnet.page(
|
||||
_stock(None, stock, []), "", cursor, Ask(params={"q": "x"}))
|
||||
assert nxt == after
|
||||
|
||||
|
||||
def test_the_page_past_the_last_is_empty_and_ends_the_read() -> None:
|
||||
decoded, found, total, nxt = jobnet.page(lambda url, **kw: END, "", 101, Ask())
|
||||
assert (found, total, nxt, jobnet.rows(decoded, "")) == ([], 20684, None, [])
|
||||
|
||||
|
||||
def test_an_answer_that_is_no_search_result_raises() -> None:
|
||||
with pytest.raises(ValueError, match="jobAds"):
|
||||
jobnet.page(lambda url, **kw: b'{"message": "denied"}', "", None, Ask())
|
||||
|
||||
|
||||
def test_it_refuses_an_unknown_region() -> None:
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
jobnet.page(lambda url, **kw: b"{}", "", None,
|
||||
Ask(params={"q": "x", "region": "Skane"}))
|
||||
|
||||
|
||||
def test_the_regions_are_the_five_with_their_danish_labels() -> None:
|
||||
assert list(jobnet.REGIONS) == [
|
||||
"HovedstadenOgBornholm", "Midtjylland", "Syddanmark", "OevrigeSjaelland", "Nordjylland"]
|
||||
assert jobnet.REGIONS["OevrigeSjaelland"] == "Øvrige Sjælland"
|
||||
|
||||
|
||||
def test_saved_params_keep_a_region_or_none() -> None:
|
||||
assert jobnet.clean_params({"q": "x", "region": " Midtjylland "}) == {
|
||||
"q": "x", "region": "Midtjylland"}
|
||||
assert jobnet.clean_params({"q": "x"}) == {"q": "x", "region": ""}
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
jobnet.clean_params({"q": "x", "region": "Skane"})
|
||||
Reference in new issue
Block a user