From b78c9dfc67a33f74e9d4dce342ab06b10f40bffa Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 09:08:25 +0200 Subject: [PATCH] Search providers as reader modules: jobindex, adzuna, platsbanken Each declares READS with scope=search, keeps every place and hands the excerpt over as teaser. search_params and PROVIDERS follow the declarations. Co-Authored-By: Claude Opus 5.5 --- app/jobs/sources.py | 30 ++-- plugins/readers/_words.py | 19 +++ plugins/readers/adzuna.py | 135 ++++++++++++++++ plugins/readers/jobindex.py | 150 ++++++++++++++++++ plugins/readers/platsbanken.py | 130 +++++++++++++++ tests/fixtures/search/adzuna-places.json | 42 +++++ tests/fixtures/search/adzuna-teaser.json | 42 +++++ tests/fixtures/search/jobindex-places.xml | 125 +++++++++++++++ tests/fixtures/search/jobindex-teaser.xml | 125 +++++++++++++++ tests/fixtures/search/platsbanken-places.json | 97 +++++++++++ tests/test_readers_adzuna.py | 91 +++++++++++ tests/test_readers_jobindex.py | 111 +++++++++++++ tests/test_readers_platsbanken.py | 111 +++++++++++++ tests/test_sources.py | 11 ++ 14 files changed, 1202 insertions(+), 17 deletions(-) create mode 100644 plugins/readers/_words.py create mode 100644 plugins/readers/adzuna.py create mode 100644 plugins/readers/jobindex.py create mode 100644 plugins/readers/platsbanken.py create mode 100644 tests/fixtures/search/adzuna-places.json create mode 100644 tests/fixtures/search/adzuna-teaser.json create mode 100644 tests/fixtures/search/jobindex-places.xml create mode 100644 tests/fixtures/search/jobindex-teaser.xml create mode 100644 tests/fixtures/search/platsbanken-places.json create mode 100644 tests/test_readers_adzuna.py create mode 100644 tests/test_readers_jobindex.py create mode 100644 tests/test_readers_platsbanken.py diff --git a/app/jobs/sources.py b/app/jobs/sources.py index eb5d7da..7a5a132 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -21,12 +21,13 @@ from app.config import Config from app.models import HAND_REASONS, Company, Subscription, company_key from app.storage import Store from app.jobs.pull import FAILING, search_target +from plugins import readers if TYPE_CHECKING: from app.jobs.discover import Discovery -#: The providers `plugins.search` runs, and which of them needs a key. -PROVIDERS = ("jobindex", "adzuna", "platsbanken") +#: The providers, from the reader modules that declare `scope="search"`. +PROVIDERS = tuple(readers.PROVIDERS) def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]: @@ -339,25 +340,20 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: def search_params(provider: str, q: str, *, area: int = 0, country: str = "", where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]: """The `params` a search subscription stores, with only the keys its - provider reads.""" + provider reads: those of its module's `PARAMS` whose filter dimension + `READS.filters` declares, cleaned by the module's `clean_params`, which + refuses a value the provider would not accept.""" if provider not in PROVIDERS: raise ValueError(f"provider is one of {', '.join(PROVIDERS)}") if not q.strip(): raise ValueError("a search needs a query") - params: dict[str, Any] = {"q": q.strip()} - if provider == "jobindex" and area: - params["area"] = int(area) - if provider == "adzuna": - params["country"] = country.strip().lower() - if where.strip(): - params["where"] = where.strip() - if provider == "platsbanken": - from plugins.search import PLATSBANKEN_REGIONS - - region = region.strip() - if region and region not in PLATSBANKEN_REGIONS: - raise ValueError("region is empty or the id of a Swedish län") - params["region"] = region + module = readers.PROVIDERS[provider] + declared = module.READS.filters.declared() + given: dict[str, Any] = {"q": q.strip(), "area": area, "country": country, + "where": where, "region": region} + wanted = {name: given[name] for name, dimension in module.PARAMS.items() + if dimension in declared} + params = module.clean_params(wanted) if pages: params["pages"] = int(pages) return params diff --git a/plugins/readers/_words.py b/plugins/readers/_words.py new file mode 100644 index 0000000..e8d4cba --- /dev/null +++ b/plugins/readers/_words.py @@ -0,0 +1,19 @@ +"""What the search providers' headlines and town lines say about where the +work is done, in Danish, Swedish and English. Shared by the search readers; +a board states its own remote flag and needs none of this.""" + +from __future__ import annotations + +import re + +#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". +_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) +_REMOTE = re.compile( + r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE) + + +def remote_word(text: str) -> str | None: + """`remote`, `hybrid` or None: the word the text says.""" + if _REMOTE.search(text): + return "remote" + return "hybrid" if _HYBRID.search(text) else None diff --git a/plugins/readers/adzuna.py b/plugins/readers/adzuna.py new file mode 100644 index 0000000..e5957c0 --- /dev/null +++ b/plugins/readers/adzuna.py @@ -0,0 +1,135 @@ +"""Adzuna: an aggregator with a documented API for most of Europe, the UK and +the US, but not Denmark (adzuna.dk does not exist). Needs a free key from +developer.adzuna.com, as `sources.search.adzuna: {app_id, app_key}` or +`ADZUNA_APP_ID` and `ADZUNA_APP_KEY`; the runner resolves it and passes it as +`ask.params["key"]`. + +Built against Adzuna's documented response and not yet run live. One request +is `/v1/api/jobs//search/` with `what`, `where`, `sort_by=date` +and 50 results a page; the answer is `{"count", "results": [...]}` and the +search is complete when as many distinct results were read as `count` states. +Titles and snippets carry `` highlights. The description is a snippet +of the ad, so it is the `teaser`. A salary Adzuna estimated +(`salary_is_predicted`) is not the posting's, so it is dropped. +""" + +from __future__ import annotations + +import html +import json +from collections.abc import Callable +from typing import Any +from urllib.parse import urlencode + +from desk.sources import strip_html + +from ._contract import Ask, Fields, Filters, Key, Place, Reads +from ._words import remote_word +from ._shared import Get, Page, Row, is_web_url, only, parse_dt + +NAME = "adzuna" +LABEL = "Adzuna" +RANK = 20 + +READS = Reads( + scope="search", listing="paged", page_size=50, + text="excerpt", complete="stated", + filters=Filters(query="one", country="one", place="one"), + places="one", + fields=Fields(pay="figures", posted="trusted", country="stated"), + key=Key(name="adzuna", fields=("app_id", "app_key"), + env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")), + robots="robots.txt disallows / for every agent (checked 2026-09-30); the documented " + "API is read with the account's key, decision of the owner pending") + +#: Saved param -> the filter dimension it fills. +PARAMS = {"q": "query", "country": "country", "where": "place"} + +SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}" + +#: Adzuna's countries in Europe, and the currency its salaries are in. +COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR", + "be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"} + + +def allowed(slug: str) -> Callable[[str], bool]: + return only("api.adzuna.com") + + +def clean_params(params: dict[str, Any]) -> dict[str, Any]: + """The saved params of a search: `q`, `country` (one of `COUNTRIES`) and + `where` when given.""" + country = str(params.get("country") or "").strip().lower() + if country not in COUNTRIES: + raise ValueError(f"country is one of {', '.join(sorted(COUNTRIES))}") + out: dict[str, Any] = {"q": params["q"], "country": country} + if str(params.get("where") or "").strip(): + out["where"] = str(params["where"]).strip() + return out + + +def _text(fragment: str) -> str: + return html.unescape(strip_html(fragment)).strip() + + +def _country(ask: Ask) -> str: + country = (ask.countries[0] if ask.countries else str(ask.params.get("country") or "")) + country = country.lower() + if country and country not in COUNTRIES: + raise ValueError(f"adzuna country {country!r} is not one of " + f"{', '.join(sorted(COUNTRIES))}") + return country + + +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: + """One page of a search, newest first; the cursor is the page number. + The next page follows a full page. A missing key or country is not + refused here: the runner resolves the key (`READS.key`) and refuses a + search without one, `clean_params` refuses a search without a country, + and Adzuna answers 401 or 404 to a request that lacks either.""" + key = ask.params.get("key") or {} + country = _country(ask) + number = int(cursor or 1) + query: dict[str, Any] = {"app_id": key.get("app_id", ""), "app_key": key.get("app_key", ""), + "results_per_page": READS.page_size, "sort_by": "date", + "what": ask.query or str(ask.params.get("q") or "")} + where = ask.places[0] if ask.places else ask.params.get("where") + if where: + query["where"] = where + url = SEARCH.format(country=country, page=number) + "?" + urlencode(query) + data = json.loads(get(url)) + results = [job for job in data.get("results") or [] if isinstance(job, dict)] + count = data.get("count") + decoded = {"country": country, "results": results} + more = len(results) >= (READS.page_size or 50) + return (decoded, [{"id": str(job.get("id") or "")} for job in results], + count if isinstance(count, int) else None, number + 1 if more else None) + + +def rows(decoded: Any, slug: str) -> list[Row]: + country = decoded["country"] + out: list[Row] = [] + for job in decoded["results"]: + title = _text(job.get("title") or "") + url = (job.get("redirect_url") or "").strip() + if not title or not is_web_url(url): + continue + predicted = str(job.get("salary_is_predicted") or "0") == "1" + town = ((job.get("location") or {}).get("display_name") or "").strip() + places: list[Place] = [{"text": town, "remote": None}] if town else [] + out.append({ + "source_id": str(job.get("id") or ""), + "title": title, + "company": ((job.get("company") or {}).get("display_name") or "").strip(), + "url": url, + "places": places, + "location": town, + "country": country.upper(), + "remote": remote_word(title), + "posted": parse_dt(job.get("created")), + "teaser": _text(job.get("description") or ""), + "salary_min": None if predicted else job.get("salary_min"), + "salary_max": None if predicted else job.get("salary_max"), + "currency": None if predicted else COUNTRIES.get(country), + }) + return out diff --git a/plugins/readers/jobindex.py b/plugins/readers/jobindex.py new file mode 100644 index 0000000..012acaa --- /dev/null +++ b/plugins/readers/jobindex.py @@ -0,0 +1,150 @@ +"""Jobindex: Denmark's largest job board, searched through the RSS twin of +its search page, `/jobsoegning.rss`; no key. + +Checked live on 2026-09-23 and moved here from `plugins/search.py`. `q` takes +Jobindex's own syntax: words are OR, `"a b"` a phrase, `+word` required. +`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region Sjælland, 2 +Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. 20 results a page, and no +page past the tenth answers anything, so a search reads 200 results at most. +The feed states no total, so a search is never complete. + +An item is `` "Role, Company", a `<link>` to Jobindex's own page for it +and a `<description>` of HTML: the `<h4>` is the title alone, a paid ad's logo +has the company as its `alt`, the `jix_robotjob--area` span is the town and +the paragraphs are an excerpt of the ad, handed over as `teaser`. A crawled ad +("robot job") has no logo, so its company is what follows the title's last +comma, or '' when there is none. An answer with no `<channel>` is not a search +result and raises, since an empty list would read as a search that ran out. +""" + +from __future__ import annotations + +import html +import re +import xml.etree.ElementTree as ET +from collections.abc import Callable +from typing import Any +from urllib.parse import urlencode + +from desk.sources import strip_html + +from ._words import remote_word +from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._contract import Ask, Fields, Filters, Place, Reads + +NAME = "jobindex" +LABEL = "Jobindex" +RANK = 10 + +READS = Reads( + scope="search", listing="paged", page_size=20, ceiling=200, + text="excerpt", complete="never", + filters=Filters(query="one", place="one"), + places="one", + fields=Fields(posted="trusted", country="stated"), + robots="read though robots.txt disallows sort=, page= and geoareaid=: owner, 2026-09-30") + +#: Saved param -> the filter dimension it fills. +PARAMS = {"q": "query", "area": "place"} + +RSS = "https://www.jobindex.dk/jobsoegning.rss" + +#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked +#: against the titles its feeds answer with. 0 is all of Denmark. +AREAS: dict[int, str] = { + 0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland", + 2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland", + 8: "Aarhus", 7: "Aalborg", +} + +_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE) +_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL) +_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL) +_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL) +#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden". +_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende") + + +def allowed(slug: str) -> Callable[[str], bool]: + return only("www.jobindex.dk") + + +def clean_params(params: dict[str, Any]) -> dict[str, Any]: + """The saved params of a search: `q`, and `area` when it is a number + other than 0.""" + out: dict[str, Any] = {"q": params["q"]} + if params.get("area"): + out["area"] = int(params["area"]) + return out + + +def _text(fragment: str) -> str: + """HTML to plain text. Jobindex breaks titles after a slash with a + zero-width space, which would otherwise end up in `title_key`.""" + return html.unescape(strip_html(fragment)).replace("​", "").strip() + + +def _query(ask: Ask, number: int) -> str: + query: dict[str, Any] = {"q": ask.query or str(ask.params.get("q") or ""), + "sort": "date"} + area = ask.places[0] if ask.places else ask.params.get("area") + if area: + query["geoareaid"] = area + return RSS + "?" + urlencode(query | {"page": number}) + + +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: + """One page of a search, newest first; the cursor is the page number. + The next page follows a full page (counted before any item is dropped, so + a dropped item does not pass for the end) while the 200-result ceiling is + not reached.""" + number = int(cursor or 1) + root = ET.fromstring(get(_query(ask, number))) + channel = root.find("channel") + if channel is None: + raise ValueError("jobindex answered without an RSS channel") + match = _SEARCH_AREA.search(channel.findtext("title") or "") + items = [{ + "id": (item.findtext("link") or "").strip().rstrip("/").rsplit("/", 1)[-1], + "link": (item.findtext("link") or "").strip(), + "title": (item.findtext("title") or "").strip(), + "description": item.findtext("description") or "", + "published": item.findtext("pubDate"), + } for item in channel.findall("item")] + decoded = {"area": match.group(1).strip() if match else "", "items": items} + size, ceiling = READS.page_size or 20, READS.ceiling or 200 + more = len(items) >= size and number * size < ceiling + return decoded, [{"id": item["id"]} for item in items], None, number + 1 if more else None + + +def rows(decoded: Any, slug: str) -> list[Row]: + out: list[Row] = [] + for item in decoded["items"]: + link, full_title = item["link"], item["title"] + if not full_title or not is_web_url(link): + continue + description = item["description"] + h4 = _H4.search(description) + title = _text(h4.group(1)) if h4 else full_title + alt = _ALT.search(description) + company = html.unescape(alt.group(1)).strip() if alt else "" + if not company and full_title.startswith(title + ", "): + company = full_title[len(title) + 2:].strip() + area = _AREA.search(description) + town = _text(area.group(1)) if area else decoded["area"] + remote = remote_word(f"{town} {title}") + places: list[Place] = [{"text": town, "remote": True if remote == "remote" else None}] \ + if town else [] + out.append({ + "source_id": item["id"], + "title": title or full_title, + "company": company, + "url": link, + "places": places, + "location": town, + "country": "DK", + "remote": remote, + "posted": parse_dt(item["published"]), + "teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)), + }) + return out diff --git a/plugins/readers/platsbanken.py b/plugins/readers/platsbanken.py new file mode 100644 index 0000000..6f2e246 --- /dev/null +++ b/plugins/readers/platsbanken.py @@ -0,0 +1,130 @@ +"""Platsbanken: Sweden's public employment service, through the JobTech search +API at jobsearch.api.jobtechdev.se; no key. + +Checked live on 2026-09-29 and 2026-09-30, moved here from `plugins/search.py`. +`region` is the id of a län in the JobTech taxonomy (Skåne län is +`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page, and an `offset` +above 2000 is refused, so a search reads 2,100 results at most. It states +`total`. A hit has the whole ad's text, so the list has the full text. An ad +may name several workplaces: `workplace_addresses` holds one entry each, and +every one is a place of the role. `workplace_model` is mostly empty, so +`remote` comes from the headline and the model together. +""" + +from __future__ import annotations + +import html +import json +from collections.abc import Callable +from typing import Any +from urllib.parse import urlencode + +from desk.sources import strip_html + +from ._contract import Ask, Fields, Filters, Place, Reads +from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._words import remote_word + +NAME = "platsbanken" +LABEL = "Platsbanken" +RANK = 30 + +READS = Reads( + scope="search", listing="paged", page_size=100, ceiling=2000, + text="listing", complete="stated", + filters=Filters(query="one", place="one"), + places="many", + fields=Fields(posted="trusted", country="stated"), + robots="read: the host publishes no robots.txt, /robots.txt answers 404 " + "(checked 2026-09-30)") + +#: Saved param -> the filter dimension it fills. +PARAMS = {"q": "query", "region": "place"} + +SEARCH = "https://jobsearch.api.jobtechdev.se/search" + +#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region` +#: concepts (which also list regions outside Sweden). "" is all of Sweden and +#: is not a key here. +REGIONS: dict[str, str] = { + "DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län", + "K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län", + "wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län", + "MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län", + "tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län", + "CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län", + "s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län", + "EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län", + "NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län", + "zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län", + "oLT3_Q9p_3nn": "Östergötlands län", +} + + +def allowed(slug: str) -> Callable[[str], bool]: + return only("jobsearch.api.jobtechdev.se") + + +def clean_params(params: dict[str, Any]) -> dict[str, Any]: + """The saved params of a search: `q` and `region`, the id of a län or + empty for all of Sweden.""" + region = str(params.get("region") or "").strip() + if region and region not in REGIONS: + raise ValueError("region is empty or the id of a Swedish län") + return {"q": params["q"], "region": region} + + +def _text(fragment: str) -> str: + return html.unescape(strip_html(fragment)).strip() + + +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: + """One page of a search, newest first; the cursor is the offset. The next + page follows a full page while the offset is within the ceiling.""" + region = str(ask.places[0] if ask.places else ask.params.get("region") or "") + if region and region not in REGIONS: + raise ValueError(f"platsbanken region {region!r} is not one of the ids in REGIONS") + size, ceiling = READS.page_size or 100, READS.ceiling or 2000 + offset = int(cursor or 0) + query: dict[str, Any] = {"q": ask.query or str(ask.params.get("q") or ""), + "sort": "pubdate-desc", "limit": size, "offset": offset} + if region: + query["region"] = region + data = json.loads(get(SEARCH + "?" + urlencode(query))) + hits = [hit for hit in data.get("hits") or [] if isinstance(hit, dict)] + total = (data.get("total") or {}).get("value") + more = len(hits) >= size and offset + size <= ceiling + return (data, [{"id": str(hit.get("id") or "")} for hit in hits], + total if isinstance(total, int) else None, offset + size if more else None) + + +def _place(address: dict[str, Any]) -> Place | None: + text = (address.get("municipality") or address.get("region") + or address.get("country") or "").strip() + return {"text": text, "remote": None} if text else None + + +def rows(decoded: Any, slug: str) -> list[Row]: + out: list[Row] = [] + for hit in decoded.get("hits") or []: + title = _text(hit.get("headline") or "") + url = (hit.get("webpage_url") or "").strip() + if not title or not is_web_url(url): + continue + addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}] + places = [place for address in addresses if isinstance(address, dict) + if (place := _place(address))] + model = (hit.get("workplace_model") or {}).get("label") or "" + out.append({ + "source_id": str(hit.get("id") or ""), + "title": title, + "company": ((hit.get("employer") or {}).get("name") or "").strip(), + "url": url, + "places": places, + "location": places[0]["text"] if places else "", + "country": "SE", + "remote": remote_word(f"{title} {model}"), + "posted": parse_dt(hit.get("publication_date")), + "description": _text((hit.get("description") or {}).get("text") or ""), + }) + return out diff --git a/tests/fixtures/search/adzuna-places.json b/tests/fixtures/search/adzuna-places.json new file mode 100644 index 0000000..28bfa41 --- /dev/null +++ b/tests/fixtures/search/adzuna-places.json @@ -0,0 +1,42 @@ +{ + "__CLASS__": "Adzuna::API::Response::JobSearchResults", + "count": 3, + "mean": 65000, + "results": [ + { + "__CLASS__": "Adzuna::API::Response::Job", + "id": "4412345678", + "title": "Senior <strong>Backend</strong> Engineer (m/w/d) - Remote", + "description": "Build our <strong>backend</strong> services in Python & Go.", + "created": "2026-09-20T08:15:00Z", + "redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345678?se=x", + "company": {"__CLASS__": "Adzuna::API::Response::Company", "display_name": "Acme GmbH"}, + "location": {"__CLASS__": "Adzuna::API::Response::Location", + "display_name": "Berlin, Deutschland", "area": ["Deutschland", "Berlin"]}, + "category": {"__CLASS__": "Adzuna::API::Response::Category", + "label": "IT-Stellen", "tag": "it-jobs"}, + "salary_min": 70000, + "salary_max": 90000, + "salary_is_predicted": "0", + "contract_time": "full_time" + }, + { + "__CLASS__": "Adzuna::API::Response::Job", + "id": "4412345679", + "title": "Platform Engineer", + "description": "Kubernetes and Terraform.", + "created": "2026-09-19T10:00:00Z", + "redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345679", + "company": {"display_name": "Beta AG"}, + "location": {"display_name": "Hamburg"}, + "salary_min": 64000, + "salary_max": 64000, + "salary_is_predicted": "1" + }, + { + "id": "4412345680", + "title": "Refused", + "redirect_url": "javascript:alert(1)" + } + ] +} diff --git a/tests/fixtures/search/adzuna-teaser.json b/tests/fixtures/search/adzuna-teaser.json new file mode 100644 index 0000000..28bfa41 --- /dev/null +++ b/tests/fixtures/search/adzuna-teaser.json @@ -0,0 +1,42 @@ +{ + "__CLASS__": "Adzuna::API::Response::JobSearchResults", + "count": 3, + "mean": 65000, + "results": [ + { + "__CLASS__": "Adzuna::API::Response::Job", + "id": "4412345678", + "title": "Senior <strong>Backend</strong> Engineer (m/w/d) - Remote", + "description": "Build our <strong>backend</strong> services in Python & Go.", + "created": "2026-09-20T08:15:00Z", + "redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345678?se=x", + "company": {"__CLASS__": "Adzuna::API::Response::Company", "display_name": "Acme GmbH"}, + "location": {"__CLASS__": "Adzuna::API::Response::Location", + "display_name": "Berlin, Deutschland", "area": ["Deutschland", "Berlin"]}, + "category": {"__CLASS__": "Adzuna::API::Response::Category", + "label": "IT-Stellen", "tag": "it-jobs"}, + "salary_min": 70000, + "salary_max": 90000, + "salary_is_predicted": "0", + "contract_time": "full_time" + }, + { + "__CLASS__": "Adzuna::API::Response::Job", + "id": "4412345679", + "title": "Platform Engineer", + "description": "Kubernetes and Terraform.", + "created": "2026-09-19T10:00:00Z", + "redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345679", + "company": {"display_name": "Beta AG"}, + "location": {"display_name": "Hamburg"}, + "salary_min": 64000, + "salary_max": 64000, + "salary_is_predicted": "1" + }, + { + "id": "4412345680", + "title": "Refused", + "redirect_url": "javascript:alert(1)" + } + ] +} diff --git a/tests/fixtures/search/jobindex-places.xml b/tests/fixtures/search/jobindex-places.xml new file mode 100644 index 0000000..f03d6d2 --- /dev/null +++ b/tests/fixtures/search/jobindex-places.xml @@ -0,0 +1,125 @@ +<?xml version="1.0" encoding="ISO-8859-1"?> + +<rss version="2.0" + xmlns:blogChannel="http://backend.userland.com/blogChannelModule" +> + +<channel> +<title>Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer +https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date +Danmarks største jobmarked +da +http://blogs.law.harvard.edu/tech/rss +support@jobindex.dk (Jobindex) +1440 + + +Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer +https://www.jobindex.dk/img/jobindex130x30gray.gif +https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date + + +Dangerous Goods Compliance Manager – ensure global safety, DSV +https://www.jobindex.dk/vis-job/h1701165 + + +<div class="jobsearch-result"> +<div class="PaidJob"> + + + <div class="PaidJob-inner"> + + +<CENTER style="margin-bottom: 15px;"><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/img/brand/40_spo_top_dsv_1812_20250718.png" ALT="DSV" style="border: 0px; margin: 0;"></a></CENTER> +<h4><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank">Dangerous Goods Compliance Manager – ensure global safety</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">Hedehusene</span> + + </div> + +<p>Do you want to minimise workplace accidents and damage to property, while helping ensure that dangerous goods and hazardous materials are handled correctly at our nearly 3,000 locations worldwide?</p><p>“One minute, you’ll help a forwarding agent in Holland figure out what to do with a pallet. The next, you’ll help shape our strategic decisions after the merger with DB Schenker,” says Martin Guldhammer.</p> +<center><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/api/imagescaler/v1/brand/8_spo_bund_dsv_1812_20250718.png?max_width=525" alt="DSV" style="border: 0px; margin-top: 5px;"></a></center> + + </div> + +</div> +</div> + +Logistik og spedition +https://www.jobindex.dk/1701165 +Wed, 23 Sep 2026 00:00:00 +0200 + + +Senior Account Executive +https://www.jobindex.dk/vis-job/r14011701 + + +<div class=""> +<div class="jobsearch-result"> + <div class="jix_robotjob" data-jobad-tid="r14011701" data-robotjob-abtest-enabled=""> + + + <div class="jix_robotjob-inner"> + + +<h4><A HREF="https://www.jobindex.dk/vis-job/r14011701" rel="noopener" target="_blank"> + Senior Account Executive +</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">København</span> + + </div> + +<p>All Gravy is the AI-native people operating platform for hospitality and retail.</p> + + </div> + + + </div> +</div> +</div> + +Salg +https://www.jobindex.dk/14011701 +Wed, 23 Sep 2026 00:00:00 +0200 + + +Marketing Content Creator, Clerk.IO ApS +https://www.jobindex.dk/vis-job/r14011706 + + +<div class=""> +<div class="jobsearch-result"> + <div class="jix_robotjob" data-jobad-tid="r14011706" data-robotjob-abtest-enabled=""> + + + <div class="jix_robotjob-inner"> + + +<h4><A HREF="https://www.jobindex.dk/vis-job/r14011706" rel="noopener" target="_blank"> + Marketing Content Creator +</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">København</span> + + </div> + +<p>Your marketing degree gave you the foundation.</p> + + </div> + + + </div> +</div> +</div> + +Studiepraktik +Marketing +https://www.jobindex.dk/14011706 +Wed, 23 Sep 2026 00:00:00 +0200 + + + diff --git a/tests/fixtures/search/jobindex-teaser.xml b/tests/fixtures/search/jobindex-teaser.xml new file mode 100644 index 0000000..f03d6d2 --- /dev/null +++ b/tests/fixtures/search/jobindex-teaser.xml @@ -0,0 +1,125 @@ + + + + + +Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer +https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date +Danmarks største jobmarked +da +http://blogs.law.harvard.edu/tech/rss +support@jobindex.dk (Jobindex) +1440 + + +Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer +https://www.jobindex.dk/img/jobindex130x30gray.gif +https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date + + +Dangerous Goods Compliance Manager – ensure global safety, DSV +https://www.jobindex.dk/vis-job/h1701165 + + +<div class="jobsearch-result"> +<div class="PaidJob"> + + + <div class="PaidJob-inner"> + + +<CENTER style="margin-bottom: 15px;"><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/img/brand/40_spo_top_dsv_1812_20250718.png" ALT="DSV" style="border: 0px; margin: 0;"></a></CENTER> +<h4><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank">Dangerous Goods Compliance Manager – ensure global safety</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">Hedehusene</span> + + </div> + +<p>Do you want to minimise workplace accidents and damage to property, while helping ensure that dangerous goods and hazardous materials are handled correctly at our nearly 3,000 locations worldwide?</p><p>“One minute, you’ll help a forwarding agent in Holland figure out what to do with a pallet. The next, you’ll help shape our strategic decisions after the merger with DB Schenker,” says Martin Guldhammer.</p> +<center><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/api/imagescaler/v1/brand/8_spo_bund_dsv_1812_20250718.png?max_width=525" alt="DSV" style="border: 0px; margin-top: 5px;"></a></center> + + </div> + +</div> +</div> + +Logistik og spedition +https://www.jobindex.dk/1701165 +Wed, 23 Sep 2026 00:00:00 +0200 + + +Senior Account Executive +https://www.jobindex.dk/vis-job/r14011701 + + +<div class=""> +<div class="jobsearch-result"> + <div class="jix_robotjob" data-jobad-tid="r14011701" data-robotjob-abtest-enabled=""> + + + <div class="jix_robotjob-inner"> + + +<h4><A HREF="https://www.jobindex.dk/vis-job/r14011701" rel="noopener" target="_blank"> + Senior Account Executive +</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">København</span> + + </div> + +<p>All Gravy is the AI-native people operating platform for hospitality and retail.</p> + + </div> + + + </div> +</div> +</div> + +Salg +https://www.jobindex.dk/14011701 +Wed, 23 Sep 2026 00:00:00 +0200 + + +Marketing Content Creator, Clerk.IO ApS +https://www.jobindex.dk/vis-job/r14011706 + + +<div class=""> +<div class="jobsearch-result"> + <div class="jix_robotjob" data-jobad-tid="r14011706" data-robotjob-abtest-enabled=""> + + + <div class="jix_robotjob-inner"> + + +<h4><A HREF="https://www.jobindex.dk/vis-job/r14011706" rel="noopener" target="_blank"> + Marketing Content Creator +</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">København</span> + + </div> + +<p>Your marketing degree gave you the foundation.</p> + + </div> + + + </div> +</div> +</div> + +Studiepraktik +Marketing +https://www.jobindex.dk/14011706 +Wed, 23 Sep 2026 00:00:00 +0200 + + + diff --git a/tests/fixtures/search/platsbanken-places.json b/tests/fixtures/search/platsbanken-places.json new file mode 100644 index 0000000..4758ca8 --- /dev/null +++ b/tests/fixtures/search/platsbanken-places.json @@ -0,0 +1,97 @@ +{ + "total": { + "value": 32 + }, + "hits": [ + { + "id": "31457942", + "headline": "Postdoctoral Researcher", + "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942", + "publication_date": "2026-09-09T15:47:27", + "description": { + "text": "The European Spallation Source (ESS) is one of the largest." + }, + "employer": { + "name": "European Spallation Source Eric" + }, + "workplace_model": { + "concept_id": "ZxwM_gtz_n61", + "label": "Arbete på plats" + }, + "workplace_address": { + "municipality": "Lund", + "region": "Skåne län", + "region_concept_id": "CaRE_1nn_cSU", + "country": "Sverige" + }, + "workplace_addresses": [ + { + "municipality": "Lund", + "region": "Skåne län", + "region_concept_id": "CaRE_1nn_cSU", + "country": "Sverige" + }, + { + "municipality": "Stockholm", + "region": "Stockholms län", + "region_concept_id": "CifL_Rzy_Mku", + "country": "Sverige" + }, + { + "municipality": null, + "region": "Västra Götalands län", + "region_concept_id": "zdoY_6u5_Krt", + "country": "Sverige" + } + ] + }, + { + "id": "31435504", + "headline": "Postdoctoral Fellow in Robotics and Computer Vision", + "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504", + "publication_date": "2026-09-03T15:46:06", + "description": { + "text": "Robotics at ESS." + }, + "employer": { + "name": "European Spallation Source Eric" + }, + "workplace_model": { + "concept_id": null, + "label": null + }, + "workplace_address": { + "municipality": null, + "region": "Skåne län", + "region_concept_id": "CaRE_1nn_cSU", + "country": "Sverige" + }, + "workplace_addresses": [ + { + "municipality": null, + "region": "Skåne län", + "region_concept_id": "CaRE_1nn_cSU", + "country": "Sverige" + } + ] + }, + { + "id": "31400001", + "headline": "Data scientist, distansarbete möjligt", + "webpage_url": "javascript:alert(1)", + "workplace_address": {}, + "workplace_addresses": [ + {} + ] + }, + { + "id": "31400002", + "headline": "Data scientist, distansarbete möjligt", + "webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002", + "employer": { + "name": null + }, + "workplace_address": {} + } + ] +} \ No newline at end of file diff --git a/tests/test_readers_adzuna.py b/tests/test_readers_adzuna.py new file mode 100644 index 0000000..ef378c5 --- /dev/null +++ b/tests/test_readers_adzuna.py @@ -0,0 +1,91 @@ +"""The Adzuna reader, built against Adzuna's documented response (no key to +run it live). No network: `get` is a stub.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlsplit + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import readers # noqa: E402 +from plugins.readers import Ask, adzuna # noqa: E402 + +BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes() +ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin", + "key": {"app_id": "id1", "app_key": "key1"}}) + + +def _rows(ask: Ask = ASK) -> list[dict[str, Any]]: + decoded, _results, _total, _cursor = adzuna.page(lambda url, **kw: BODY, "", None, ask) + return list(adzuna.rows(decoded, "")) + + +def test_it_declares_the_key_and_states_the_robots_finding() -> None: + assert readers.PROVIDERS["adzuna"] is adzuna + reads = adzuna.READS + assert reads.key is not None and reads.key.name == "adzuna" + assert (reads.key.fields, reads.key.env) == ( + ("app_id", "app_key"), ("ADZUNA_APP_ID", "ADZUNA_APP_KEY")) + assert (reads.text, reads.complete, reads.page_size) == ("excerpt", "stated", 50) + assert "disallows /" in reads.robots and "2026-09-30" in reads.robots + + +def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None: + first, predicted = _rows() # javascript: refused + assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"] + assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote" + assert first["teaser"] == "Build our backend services in Python & Go." + assert "description" not in first + assert first["company"] == "Acme GmbH" + assert first["places"] == [{"text": "Berlin, Deutschland", "remote": None}] + assert first["country"] == "DE" and first["remote"] == "remote" + assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR") + assert (predicted["salary_min"], predicted["currency"]) == (None, None) + + +def test_it_sends_the_key_the_country_and_the_query() -> None: + asked: list[str] = [] + + def get(url: str, **kwargs: Any) -> bytes: + asked.append(url) + return BODY + + adzuna.page(get, "", None, ASK) + parts = urlsplit(asked[0]) + assert (parts.netloc, parts.path) == ("api.adzuna.com", "/v1/api/jobs/de/search/1") + assert parse_qs(parts.query) == { + "app_id": ["id1"], "app_key": ["key1"], "results_per_page": ["50"], + "sort_by": ["date"], "what": ["backend"], "where": ["Berlin"]} + + +def test_it_refuses_an_unknown_country() -> None: + with pytest.raises(ValueError, match="country"): + adzuna.page(lambda url, **kw: BODY, "", None, + Ask(params={"q": "x", "country": "dk"})) + + +@pytest.mark.parametrize(("results", "count", "after"), [(50, 120, 2), (49, 49, None)]) +def test_it_pages_on_a_full_page_and_states_the_count( + results: int, count: int, after: int | None) -> None: + body = json.dumps({"count": count, "results": [ + {"id": str(n), "title": "T", "redirect_url": f"https://x.example/{n}"} + for n in range(results)]}).encode() + _decoded, found, total, nxt = adzuna.page(lambda url, **kw: body, "", None, ASK) + assert (len(found), total, nxt) == (results, count, after) + + +def test_saved_params_need_a_known_country() -> None: + assert adzuna.clean_params({"q": "x", "country": " DE ", "where": " Berlin "}) == { + "q": "x", "country": "de", "where": "Berlin"} + assert adzuna.clean_params({"q": "x", "country": "gb", "where": ""}) == { + "q": "x", "country": "gb"} + with pytest.raises(ValueError, match="country"): + adzuna.clean_params({"q": "x", "country": "dk"}) diff --git a/tests/test_readers_jobindex.py b/tests/test_readers_jobindex.py new file mode 100644 index 0000000..d45caa1 --- /dev/null +++ b/tests/test_readers_jobindex.py @@ -0,0 +1,111 @@ +"""The Jobindex reader: the RSS twin of a search, 20 results a page, 200 at +most. No network: `get` is a stub.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlsplit + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import readers # noqa: E402 +from plugins.readers import Ask, jobindex # noqa: E402 + +BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes() + + +def _rows(body: bytes = BODY) -> list[dict[str, Any]]: + decoded, _results, _total, _cursor = jobindex.page(lambda url, **kw: body, "", None, Ask()) + return list(jobindex.rows(decoded, "")) + + +def _feed(items: int) -> bytes: + body = "".join(f"Role {n}, Acme" + f"https://www.jobindex.dk/vis-job/r{n}" + for n in range(items)) + return (f"Jobindex{body}").encode() + + +def test_it_is_a_search_reader_with_the_owners_robots_decision() -> None: + assert readers.PROVIDERS["jobindex"] is jobindex + reads = jobindex.READS + assert (reads.scope, reads.page_size, reads.ceiling, reads.text) == ( + "search", 20, 200, "excerpt") + assert "owner, 2026-09-30" in reads.robots and "geoareaid" in reads.robots + assert jobindex.allowed("")("www.jobindex.dk") and not jobindex.allowed("")("jobindex.dk") + + +def test_it_reads_title_company_town_date_and_teaser() -> None: + paid, bare, robot = _rows() + assert [r["source_id"] for r in (paid, bare, robot)] == ["h1701165", "r14011701", "r14011706"] + assert paid["company"] == "DSV" + assert paid["title"].startswith("Dangerous Goods Compliance Manager") + assert paid["places"] == [{"text": "Hedehusene", "remote": None}] + assert paid["location"] == "Hedehusene" and paid["country"] == "DK" + assert paid["posted"].date().isoformat() == "2026-09-23" + assert paid["teaser"].startswith("Do you want to minimise") + assert "description" not in paid + assert bare["company"] == "" and bare["places"][0]["text"] == "København" + assert (robot["title"], robot["company"]) == ("Marketing Content Creator", "Clerk.IO ApS") + + +def test_it_reads_hybrid_and_remote_from_the_town_line() -> None: + from plugins.readers._words import remote_word + + assert remote_word("Birkerød, Hybrid position") == "hybrid" + assert remote_word("Nordhavn og mulighed for hjemmearbejde") == "hybrid" + assert remote_word("Remote, Danmark") == "remote" + assert remote_word("Hedehusene") is None + + +def test_it_asks_for_the_query_the_area_and_the_page() -> None: + asked: list[str] = [] + + def get(url: str, **kwargs: Any) -> bytes: + asked.append(url) + return _feed(3) + + jobindex.page(get, "", None, Ask(params={"q": "backend developer", "area": 1})) + parts = urlsplit(asked[0]) + assert (parts.scheme, parts.netloc, parts.path) == ( + "https", "www.jobindex.dk", "/jobsoegning.rss") + assert parse_qs(parts.query) == {"q": ["backend developer"], "sort": ["date"], + "geoareaid": ["1"], "page": ["1"]} + jobindex.page(get, "", 4, Ask(params={"q": "x"})) + assert "geoareaid" not in asked[1] and "page=4" in asked[1] + + +@pytest.mark.parametrize(("items", "cursor", "after"), [ + (20, None, 2), # a full page has a next + (19, None, None), # a short page is the end + (20, 10, None), # the tenth page reaches the 200 ceiling + (20, 9, 10), +]) +def test_it_pages_until_a_short_page_or_the_ceiling( + items: int, cursor: int | None, after: int | None) -> None: + _decoded, results, total, nxt = jobindex.page( + lambda url, **kw: _feed(items), "", cursor, Ask(params={"q": "x"})) + assert (len(results), total, nxt) == (items, None, after) + + +def test_a_dropped_item_does_not_pass_for_the_end() -> None: + body = _feed(19).replace(b"", b"No link") + decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask()) + assert len(results) == 20 and nxt == 2 + assert len(jobindex.rows(decoded, "")) == 19 + + +def test_an_answer_without_a_channel_raises() -> None: + with pytest.raises(ValueError, match="channel"): + jobindex.page(lambda url, **kw: b"", "", None, Ask()) + + +def test_saved_params_keep_the_query_and_a_real_area() -> None: + assert jobindex.clean_params({"q": "x", "area": "3"}) == {"q": "x", "area": 3} + assert jobindex.clean_params({"q": "x", "area": 0}) == {"q": "x"} diff --git a/tests/test_readers_platsbanken.py b/tests/test_readers_platsbanken.py new file mode 100644 index 0000000..4adc47b --- /dev/null +++ b/tests/test_readers_platsbanken.py @@ -0,0 +1,111 @@ +"""The Platsbanken reader (JobTech search API): 100 hits a page, offsets to +2000. No network: `get` is a stub.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlsplit + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import readers # noqa: E402 +from plugins.readers import Ask, platsbanken # noqa: E402 + +BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes() + + +def _stock(total: int | None, stock: int, asked: list[str]) -> Any: + def get(url: str, **kwargs: Any) -> bytes: + asked.append(url) + query = parse_qs(urlsplit(url).query) + start, limit = int(query["offset"][0]), int(query["limit"][0]) + body: dict[str, Any] = {"hits": [ + {"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}"} + for n in range(start, min(start + limit, stock))]} + if total is not None: + body["total"] = {"value": total} + return json.dumps(body).encode() + return get + + +def test_it_is_a_search_reader_with_a_404_robots_txt_recorded() -> None: + assert readers.PROVIDERS["platsbanken"] is platsbanken + reads = platsbanken.READS + assert (reads.scope, reads.page_size, reads.ceiling, reads.places) == ( + "search", 100, 2000, "many") + assert "404" in reads.robots and "2026-09-30" in reads.robots + + +def test_it_keeps_every_workplace_of_an_ad() -> None: + decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask()) + rows = platsbanken.rows(decoded, "") + assert total == 32 + assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused + first, no_town, bare = rows + assert [p["text"] for p in first["places"]] == [ + "Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län + assert first["location"] == "Lund" + assert no_town["places"] == [{"text": "Skåne län", "remote": None}] + assert bare["places"] == [] + + +def test_it_reads_title_company_date_and_the_full_text() -> None: + decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask()) + first, _no_town, remote = platsbanken.rows(decoded, "") + assert first["title"] == "Postdoctoral Researcher" + assert first["company"] == "European Spallation Source Eric" + assert first["country"] == "SE" and first["remote"] is None + assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942" + assert first["posted"].date().isoformat() == "2026-09-09" + assert first["description"].startswith("The European Spallation Source") + assert "teaser" not in first + assert (remote["remote"], remote["company"], remote["posted"]) == ("remote", "", None) + + +def test_it_sends_the_region_the_offset_and_no_region_for_all_of_sweden() -> None: + asked: list[str] = [] + get = _stock(250, 250, asked) + platsbanken.page(get, "", None, Ask(params={"q": "data", "region": "CaRE_1nn_cSU"})) + platsbanken.page(get, "", 100, Ask(params={"q": "data", "region": ""})) + assert parse_qs(urlsplit(asked[0]).query) == { + "q": ["data"], "sort": ["pubdate-desc"], "limit": ["100"], "offset": ["0"], + "region": ["CaRE_1nn_cSU"]} + assert "region" not in urlsplit(asked[1]).query and "offset=100" in asked[1] + assert urlsplit(asked[0]).netloc == "jobsearch.api.jobtechdev.se" + + +@pytest.mark.parametrize(("cursor", "stock", "after"), [ + (None, 250, 100), (200, 250, None), # a short page is the end + (1900, 2500, 2000), (2000, 2500, None), # the offset ceiling +]) +def test_it_pages_by_offset_up_to_the_ceiling( + cursor: int | None, stock: int, after: int | None) -> None: + _decoded, _found, _total, nxt = platsbanken.page( + _stock(None, stock, []), "", cursor, Ask(params={"q": "x"})) + assert nxt == after + + +def test_it_refuses_an_unknown_region() -> None: + with pytest.raises(ValueError, match="region"): + platsbanken.page(lambda url, **kw: b"{}", "", None, + Ask(params={"q": "x", "region": "zYpL_aGk_QY8"})) # Midland + + +def test_the_regions_are_the_21_lan() -> None: + assert len(platsbanken.REGIONS) == 21 + assert all(name.endswith(" län") for name in platsbanken.REGIONS.values()) + + +def test_saved_params_keep_a_län_or_none() -> None: + assert platsbanken.clean_params({"q": "x", "region": " CaRE_1nn_cSU "}) == { + "q": "x", "region": "CaRE_1nn_cSU"} + assert platsbanken.clean_params({"q": "x", "region": ""}) == {"q": "x", "region": ""} + with pytest.raises(ValueError, match="region"): + platsbanken.clean_params({"q": "x", "region": "zYpL_aGk_QY8"}) diff --git a/tests/test_sources.py b/tests/test_sources.py index 754f9bc..26e01cc 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -573,6 +573,17 @@ def test_search_params_keeps_only_what_the_provider_reads() -> None: "q": "data", "region": "CaRE_1nn_cSU", "pages": 3} +def test_the_providers_and_their_params_follow_the_reader_declarations() -> None: + from plugins import readers + + assert sources.PROVIDERS == tuple(readers.PROVIDERS) == ("jobindex", "adzuna", "platsbanken") + assert sources.search_params("adzuna", " backend ", country=" DE ", where=" Berlin ", + area=1, region="x", pages=2) == { + "q": "backend", "country": "de", "where": "Berlin", "pages": 2} + with pytest.raises(ValueError, match="country"): + sources.search_params("adzuna", "backend", country="dk") + + def test_search_params_refuses_a_region_that_is_no_swedish_lan() -> None: with pytest.raises(ValueError, match="region"): sources.search_params("platsbanken", "data", region="zYpL_aGk_QY8") # Midland