Search providers as reader modules: jobindex, adzuna, platsbanken
Each declares READS with scope=search, keeps every place and hands the excerpt over as teaser. search_params and PROVIDERS follow the declarations. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
b78c9dfc67
14 files changed
+1202
-17
No files matched your search
+13
-17
@@ -21,12 +21,13 @@ from app.config import Config
|
||||
from app.models import HAND_REASONS, Company, Subscription, company_key
|
||||
from app.storage import Store
|
||||
from app.jobs.pull import FAILING, search_target
|
||||
from plugins import readers
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.jobs.discover import Discovery
|
||||
|
||||
#: The providers `plugins.search` runs, and which of them needs a key.
|
||||
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
|
||||
#: The providers, from the reader modules that declare `scope="search"`.
|
||||
PROVIDERS = tuple(readers.PROVIDERS)
|
||||
|
||||
|
||||
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
|
||||
@@ -339,25 +340,20 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
|
||||
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
||||
where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]:
|
||||
"""The `params` a search subscription stores, with only the keys its
|
||||
provider reads."""
|
||||
provider reads: those of its module's `PARAMS` whose filter dimension
|
||||
`READS.filters` declares, cleaned by the module's `clean_params`, which
|
||||
refuses a value the provider would not accept."""
|
||||
if provider not in PROVIDERS:
|
||||
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
|
||||
if not q.strip():
|
||||
raise ValueError("a search needs a query")
|
||||
params: dict[str, Any] = {"q": q.strip()}
|
||||
if provider == "jobindex" and area:
|
||||
params["area"] = int(area)
|
||||
if provider == "adzuna":
|
||||
params["country"] = country.strip().lower()
|
||||
if where.strip():
|
||||
params["where"] = where.strip()
|
||||
if provider == "platsbanken":
|
||||
from plugins.search import PLATSBANKEN_REGIONS
|
||||
|
||||
region = region.strip()
|
||||
if region and region not in PLATSBANKEN_REGIONS:
|
||||
raise ValueError("region is empty or the id of a Swedish län")
|
||||
params["region"] = region
|
||||
module = readers.PROVIDERS[provider]
|
||||
declared = module.READS.filters.declared()
|
||||
given: dict[str, Any] = {"q": q.strip(), "area": area, "country": country,
|
||||
"where": where, "region": region}
|
||||
wanted = {name: given[name] for name, dimension in module.PARAMS.items()
|
||||
if dimension in declared}
|
||||
params = module.clean_params(wanted)
|
||||
if pages:
|
||||
params["pages"] = int(pages)
|
||||
return params
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""What the search providers' headlines and town lines say about where the
|
||||
work is done, in Danish, Swedish and English. Shared by the search readers;
|
||||
a board states its own remote flag and needs none of this."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
||||
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(
|
||||
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
|
||||
|
||||
|
||||
def remote_word(text: str) -> str | None:
|
||||
"""`remote`, `hybrid` or None: the word the text says."""
|
||||
if _REMOTE.search(text):
|
||||
return "remote"
|
||||
return "hybrid" if _HYBRID.search(text) else None
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Adzuna: an aggregator with a documented API for most of Europe, the UK and
|
||||
the US, but not Denmark (adzuna.dk does not exist). Needs a free key from
|
||||
developer.adzuna.com, as `sources.search.adzuna: {app_id, app_key}` or
|
||||
`ADZUNA_APP_ID` and `ADZUNA_APP_KEY`; the runner resolves it and passes it as
|
||||
`ask.params["key"]`.
|
||||
|
||||
Built against Adzuna's documented response and not yet run live. One request
|
||||
is `/v1/api/jobs/<country>/search/<page>` with `what`, `where`, `sort_by=date`
|
||||
and 50 results a page; the answer is `{"count", "results": [...]}` and the
|
||||
search is complete when as many distinct results were read as `count` states.
|
||||
Titles and snippets carry `<strong>` highlights. The description is a snippet
|
||||
of the ad, so it is the `teaser`. A salary Adzuna estimated
|
||||
(`salary_is_predicted`) is not the posting's, so it is dropped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Key, Place, Reads
|
||||
from ._words import remote_word
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "adzuna"
|
||||
LABEL = "Adzuna"
|
||||
RANK = 20
|
||||
|
||||
READS = Reads(
|
||||
scope="search", listing="paged", page_size=50,
|
||||
text="excerpt", complete="stated",
|
||||
filters=Filters(query="one", country="one", place="one"),
|
||||
places="one",
|
||||
fields=Fields(pay="figures", posted="trusted", country="stated"),
|
||||
key=Key(name="adzuna", fields=("app_id", "app_key"),
|
||||
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")),
|
||||
robots="robots.txt disallows / for every agent (checked 2026-09-30); the documented "
|
||||
"API is read with the account's key, decision of the owner pending")
|
||||
|
||||
#: Saved param -> the filter dimension it fills.
|
||||
PARAMS = {"q": "query", "country": "country", "where": "place"}
|
||||
|
||||
SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
|
||||
|
||||
#: Adzuna's countries in Europe, and the currency its salaries are in.
|
||||
COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
|
||||
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only("api.adzuna.com")
|
||||
|
||||
|
||||
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The saved params of a search: `q`, `country` (one of `COUNTRIES`) and
|
||||
`where` when given."""
|
||||
country = str(params.get("country") or "").strip().lower()
|
||||
if country not in COUNTRIES:
|
||||
raise ValueError(f"country is one of {', '.join(sorted(COUNTRIES))}")
|
||||
out: dict[str, Any] = {"q": params["q"], "country": country}
|
||||
if str(params.get("where") or "").strip():
|
||||
out["where"] = str(params["where"]).strip()
|
||||
return out
|
||||
|
||||
|
||||
def _text(fragment: str) -> str:
|
||||
return html.unescape(strip_html(fragment)).strip()
|
||||
|
||||
|
||||
def _country(ask: Ask) -> str:
|
||||
country = (ask.countries[0] if ask.countries else str(ask.params.get("country") or ""))
|
||||
country = country.lower()
|
||||
if country and country not in COUNTRIES:
|
||||
raise ValueError(f"adzuna country {country!r} is not one of "
|
||||
f"{', '.join(sorted(COUNTRIES))}")
|
||||
return country
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of a search, newest first; the cursor is the page number.
|
||||
The next page follows a full page. A missing key or country is not
|
||||
refused here: the runner resolves the key (`READS.key`) and refuses a
|
||||
search without one, `clean_params` refuses a search without a country,
|
||||
and Adzuna answers 401 or 404 to a request that lacks either."""
|
||||
key = ask.params.get("key") or {}
|
||||
country = _country(ask)
|
||||
number = int(cursor or 1)
|
||||
query: dict[str, Any] = {"app_id": key.get("app_id", ""), "app_key": key.get("app_key", ""),
|
||||
"results_per_page": READS.page_size, "sort_by": "date",
|
||||
"what": ask.query or str(ask.params.get("q") or "")}
|
||||
where = ask.places[0] if ask.places else ask.params.get("where")
|
||||
if where:
|
||||
query["where"] = where
|
||||
url = SEARCH.format(country=country, page=number) + "?" + urlencode(query)
|
||||
data = json.loads(get(url))
|
||||
results = [job for job in data.get("results") or [] if isinstance(job, dict)]
|
||||
count = data.get("count")
|
||||
decoded = {"country": country, "results": results}
|
||||
more = len(results) >= (READS.page_size or 50)
|
||||
return (decoded, [{"id": str(job.get("id") or "")} for job in results],
|
||||
count if isinstance(count, int) else None, number + 1 if more else None)
|
||||
|
||||
|
||||
def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
country = decoded["country"]
|
||||
out: list[Row] = []
|
||||
for job in decoded["results"]:
|
||||
title = _text(job.get("title") or "")
|
||||
url = (job.get("redirect_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
||||
town = ((job.get("location") or {}).get("display_name") or "").strip()
|
||||
places: list[Place] = [{"text": town, "remote": None}] if town else []
|
||||
out.append({
|
||||
"source_id": str(job.get("id") or ""),
|
||||
"title": title,
|
||||
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
|
||||
"url": url,
|
||||
"places": places,
|
||||
"location": town,
|
||||
"country": country.upper(),
|
||||
"remote": remote_word(title),
|
||||
"posted": parse_dt(job.get("created")),
|
||||
"teaser": _text(job.get("description") or ""),
|
||||
"salary_min": None if predicted else job.get("salary_min"),
|
||||
"salary_max": None if predicted else job.get("salary_max"),
|
||||
"currency": None if predicted else COUNTRIES.get(country),
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Jobindex: Denmark's largest job board, searched through the RSS twin of
|
||||
its search page, `/jobsoegning.rss`; no key.
|
||||
|
||||
Checked live on 2026-09-23 and moved here from `plugins/search.py`. `q` takes
|
||||
Jobindex's own syntax: words are OR, `"a b"` a phrase, `+word` required.
|
||||
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region Sjælland, 2
|
||||
Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. 20 results a page, and no
|
||||
page past the tenth answers anything, so a search reads 200 results at most.
|
||||
The feed states no total, so a search is never complete.
|
||||
|
||||
An item is `<title>` "Role, Company", a `<link>` to Jobindex's own page for it
|
||||
and a `<description>` of HTML: the `<h4>` is the title alone, a paid ad's logo
|
||||
has the company as its `alt`, the `jix_robotjob--area` span is the town and
|
||||
the paragraphs are an excerpt of the ad, handed over as `teaser`. A crawled ad
|
||||
("robot job") has no logo, so its company is what follows the title's last
|
||||
comma, or '' when there is none. An answer with no `<channel>` is not a search
|
||||
result and raises, since an empty list would read as a search that ran out.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._words import remote_word
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
|
||||
NAME = "jobindex"
|
||||
LABEL = "Jobindex"
|
||||
RANK = 10
|
||||
|
||||
READS = Reads(
|
||||
scope="search", listing="paged", page_size=20, ceiling=200,
|
||||
text="excerpt", complete="never",
|
||||
filters=Filters(query="one", place="one"),
|
||||
places="one",
|
||||
fields=Fields(posted="trusted", country="stated"),
|
||||
robots="read though robots.txt disallows sort=, page= and geoareaid=: owner, 2026-09-30")
|
||||
|
||||
#: Saved param -> the filter dimension it fills.
|
||||
PARAMS = {"q": "query", "area": "place"}
|
||||
|
||||
RSS = "https://www.jobindex.dk/jobsoegning.rss"
|
||||
|
||||
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
|
||||
#: against the titles its feeds answer with. 0 is all of Denmark.
|
||||
AREAS: dict[int, str] = {
|
||||
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
|
||||
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
|
||||
8: "Aarhus", 7: "Aalborg",
|
||||
}
|
||||
|
||||
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
|
||||
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
|
||||
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
|
||||
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
|
||||
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
|
||||
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only("www.jobindex.dk")
|
||||
|
||||
|
||||
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The saved params of a search: `q`, and `area` when it is a number
|
||||
other than 0."""
|
||||
out: dict[str, Any] = {"q": params["q"]}
|
||||
if params.get("area"):
|
||||
out["area"] = int(params["area"])
|
||||
return out
|
||||
|
||||
|
||||
def _text(fragment: str) -> str:
|
||||
"""HTML to plain text. Jobindex breaks titles after a slash with a
|
||||
zero-width space, which would otherwise end up in `title_key`."""
|
||||
return html.unescape(strip_html(fragment)).replace("", "").strip()
|
||||
|
||||
|
||||
def _query(ask: Ask, number: int) -> str:
|
||||
query: dict[str, Any] = {"q": ask.query or str(ask.params.get("q") or ""),
|
||||
"sort": "date"}
|
||||
area = ask.places[0] if ask.places else ask.params.get("area")
|
||||
if area:
|
||||
query["geoareaid"] = area
|
||||
return RSS + "?" + urlencode(query | {"page": number})
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of a search, newest first; the cursor is the page number.
|
||||
The next page follows a full page (counted before any item is dropped, so
|
||||
a dropped item does not pass for the end) while the 200-result ceiling is
|
||||
not reached."""
|
||||
number = int(cursor or 1)
|
||||
root = ET.fromstring(get(_query(ask, number)))
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
raise ValueError("jobindex answered without an RSS channel")
|
||||
match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
||||
items = [{
|
||||
"id": (item.findtext("link") or "").strip().rstrip("/").rsplit("/", 1)[-1],
|
||||
"link": (item.findtext("link") or "").strip(),
|
||||
"title": (item.findtext("title") or "").strip(),
|
||||
"description": item.findtext("description") or "",
|
||||
"published": item.findtext("pubDate"),
|
||||
} for item in channel.findall("item")]
|
||||
decoded = {"area": match.group(1).strip() if match else "", "items": items}
|
||||
size, ceiling = READS.page_size or 20, READS.ceiling or 200
|
||||
more = len(items) >= size and number * size < ceiling
|
||||
return decoded, [{"id": item["id"]} for item in items], None, number + 1 if more else None
|
||||
|
||||
|
||||
def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
out: list[Row] = []
|
||||
for item in decoded["items"]:
|
||||
link, full_title = item["link"], item["title"]
|
||||
if not full_title or not is_web_url(link):
|
||||
continue
|
||||
description = item["description"]
|
||||
h4 = _H4.search(description)
|
||||
title = _text(h4.group(1)) if h4 else full_title
|
||||
alt = _ALT.search(description)
|
||||
company = html.unescape(alt.group(1)).strip() if alt else ""
|
||||
if not company and full_title.startswith(title + ", "):
|
||||
company = full_title[len(title) + 2:].strip()
|
||||
area = _AREA.search(description)
|
||||
town = _text(area.group(1)) if area else decoded["area"]
|
||||
remote = remote_word(f"{town} {title}")
|
||||
places: list[Place] = [{"text": town, "remote": True if remote == "remote" else None}] \
|
||||
if town else []
|
||||
out.append({
|
||||
"source_id": item["id"],
|
||||
"title": title or full_title,
|
||||
"company": company,
|
||||
"url": link,
|
||||
"places": places,
|
||||
"location": town,
|
||||
"country": "DK",
|
||||
"remote": remote,
|
||||
"posted": parse_dt(item["published"]),
|
||||
"teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Platsbanken: Sweden's public employment service, through the JobTech search
|
||||
API at jobsearch.api.jobtechdev.se; no key.
|
||||
|
||||
Checked live on 2026-09-29 and 2026-09-30, moved here from `plugins/search.py`.
|
||||
`region` is the id of a län in the JobTech taxonomy (Skåne län is
|
||||
`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page, and an `offset`
|
||||
above 2000 is refused, so a search reads 2,100 results at most. It states
|
||||
`total`. A hit has the whole ad's text, so the list has the full text. An ad
|
||||
may name several workplaces: `workplace_addresses` holds one entry each, and
|
||||
every one is a place of the role. `workplace_model` is mostly empty, so
|
||||
`remote` comes from the headline and the model together.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._words import remote_word
|
||||
|
||||
NAME = "platsbanken"
|
||||
LABEL = "Platsbanken"
|
||||
RANK = 30
|
||||
|
||||
READS = Reads(
|
||||
scope="search", listing="paged", page_size=100, ceiling=2000,
|
||||
text="listing", complete="stated",
|
||||
filters=Filters(query="one", place="one"),
|
||||
places="many",
|
||||
fields=Fields(posted="trusted", country="stated"),
|
||||
robots="read: the host publishes no robots.txt, /robots.txt answers 404 "
|
||||
"(checked 2026-09-30)")
|
||||
|
||||
#: Saved param -> the filter dimension it fills.
|
||||
PARAMS = {"q": "query", "region": "place"}
|
||||
|
||||
SEARCH = "https://jobsearch.api.jobtechdev.se/search"
|
||||
|
||||
#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region`
|
||||
#: concepts (which also list regions outside Sweden). "" is all of Sweden and
|
||||
#: is not a key here.
|
||||
REGIONS: dict[str, str] = {
|
||||
"DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län",
|
||||
"K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län",
|
||||
"wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län",
|
||||
"MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län",
|
||||
"tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län",
|
||||
"CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län",
|
||||
"s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län",
|
||||
"EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län",
|
||||
"NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län",
|
||||
"zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län",
|
||||
"oLT3_Q9p_3nn": "Östergötlands län",
|
||||
}
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only("jobsearch.api.jobtechdev.se")
|
||||
|
||||
|
||||
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The saved params of a search: `q` and `region`, the id of a län or
|
||||
empty for all of Sweden."""
|
||||
region = str(params.get("region") or "").strip()
|
||||
if region and region not in REGIONS:
|
||||
raise ValueError("region is empty or the id of a Swedish län")
|
||||
return {"q": params["q"], "region": region}
|
||||
|
||||
|
||||
def _text(fragment: str) -> str:
|
||||
return html.unescape(strip_html(fragment)).strip()
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of a search, newest first; the cursor is the offset. The next
|
||||
page follows a full page while the offset is within the ceiling."""
|
||||
region = str(ask.places[0] if ask.places else ask.params.get("region") or "")
|
||||
if region and region not in REGIONS:
|
||||
raise ValueError(f"platsbanken region {region!r} is not one of the ids in REGIONS")
|
||||
size, ceiling = READS.page_size or 100, READS.ceiling or 2000
|
||||
offset = int(cursor or 0)
|
||||
query: dict[str, Any] = {"q": ask.query or str(ask.params.get("q") or ""),
|
||||
"sort": "pubdate-desc", "limit": size, "offset": offset}
|
||||
if region:
|
||||
query["region"] = region
|
||||
data = json.loads(get(SEARCH + "?" + urlencode(query)))
|
||||
hits = [hit for hit in data.get("hits") or [] if isinstance(hit, dict)]
|
||||
total = (data.get("total") or {}).get("value")
|
||||
more = len(hits) >= size and offset + size <= ceiling
|
||||
return (data, [{"id": str(hit.get("id") or "")} for hit in hits],
|
||||
total if isinstance(total, int) else None, offset + size if more else None)
|
||||
|
||||
|
||||
def _place(address: dict[str, Any]) -> Place | None:
|
||||
text = (address.get("municipality") or address.get("region")
|
||||
or address.get("country") or "").strip()
|
||||
return {"text": text, "remote": None} if text else None
|
||||
|
||||
|
||||
def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
out: list[Row] = []
|
||||
for hit in decoded.get("hits") or []:
|
||||
title = _text(hit.get("headline") or "")
|
||||
url = (hit.get("webpage_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
|
||||
places = [place for address in addresses if isinstance(address, dict)
|
||||
if (place := _place(address))]
|
||||
model = (hit.get("workplace_model") or {}).get("label") or ""
|
||||
out.append({
|
||||
"source_id": str(hit.get("id") or ""),
|
||||
"title": title,
|
||||
"company": ((hit.get("employer") or {}).get("name") or "").strip(),
|
||||
"url": url,
|
||||
"places": places,
|
||||
"location": places[0]["text"] if places else "",
|
||||
"country": "SE",
|
||||
"remote": remote_word(f"{title} {model}"),
|
||||
"posted": parse_dt(hit.get("publication_date")),
|
||||
"description": _text((hit.get("description") or {}).get("text") or ""),
|
||||
})
|
||||
return out
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"__CLASS__": "Adzuna::API::Response::JobSearchResults",
|
||||
"count": 3,
|
||||
"mean": 65000,
|
||||
"results": [
|
||||
{
|
||||
"__CLASS__": "Adzuna::API::Response::Job",
|
||||
"id": "4412345678",
|
||||
"title": "Senior <strong>Backend</strong> Engineer (m/w/d) - Remote",
|
||||
"description": "Build our <strong>backend</strong> services in Python & Go.",
|
||||
"created": "2026-09-20T08:15:00Z",
|
||||
"redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345678?se=x",
|
||||
"company": {"__CLASS__": "Adzuna::API::Response::Company", "display_name": "Acme GmbH"},
|
||||
"location": {"__CLASS__": "Adzuna::API::Response::Location",
|
||||
"display_name": "Berlin, Deutschland", "area": ["Deutschland", "Berlin"]},
|
||||
"category": {"__CLASS__": "Adzuna::API::Response::Category",
|
||||
"label": "IT-Stellen", "tag": "it-jobs"},
|
||||
"salary_min": 70000,
|
||||
"salary_max": 90000,
|
||||
"salary_is_predicted": "0",
|
||||
"contract_time": "full_time"
|
||||
},
|
||||
{
|
||||
"__CLASS__": "Adzuna::API::Response::Job",
|
||||
"id": "4412345679",
|
||||
"title": "Platform Engineer",
|
||||
"description": "Kubernetes and Terraform.",
|
||||
"created": "2026-09-19T10:00:00Z",
|
||||
"redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345679",
|
||||
"company": {"display_name": "Beta AG"},
|
||||
"location": {"display_name": "Hamburg"},
|
||||
"salary_min": 64000,
|
||||
"salary_max": 64000,
|
||||
"salary_is_predicted": "1"
|
||||
},
|
||||
{
|
||||
"id": "4412345680",
|
||||
"title": "Refused",
|
||||
"redirect_url": "javascript:alert(1)"
|
||||
}
|
||||
]
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"__CLASS__": "Adzuna::API::Response::JobSearchResults",
|
||||
"count": 3,
|
||||
"mean": 65000,
|
||||
"results": [
|
||||
{
|
||||
"__CLASS__": "Adzuna::API::Response::Job",
|
||||
"id": "4412345678",
|
||||
"title": "Senior <strong>Backend</strong> Engineer (m/w/d) - Remote",
|
||||
"description": "Build our <strong>backend</strong> services in Python & Go.",
|
||||
"created": "2026-09-20T08:15:00Z",
|
||||
"redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345678?se=x",
|
||||
"company": {"__CLASS__": "Adzuna::API::Response::Company", "display_name": "Acme GmbH"},
|
||||
"location": {"__CLASS__": "Adzuna::API::Response::Location",
|
||||
"display_name": "Berlin, Deutschland", "area": ["Deutschland", "Berlin"]},
|
||||
"category": {"__CLASS__": "Adzuna::API::Response::Category",
|
||||
"label": "IT-Stellen", "tag": "it-jobs"},
|
||||
"salary_min": 70000,
|
||||
"salary_max": 90000,
|
||||
"salary_is_predicted": "0",
|
||||
"contract_time": "full_time"
|
||||
},
|
||||
{
|
||||
"__CLASS__": "Adzuna::API::Response::Job",
|
||||
"id": "4412345679",
|
||||
"title": "Platform Engineer",
|
||||
"description": "Kubernetes and Terraform.",
|
||||
"created": "2026-09-19T10:00:00Z",
|
||||
"redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345679",
|
||||
"company": {"display_name": "Beta AG"},
|
||||
"location": {"display_name": "Hamburg"},
|
||||
"salary_min": 64000,
|
||||
"salary_max": 64000,
|
||||
"salary_is_predicted": "1"
|
||||
},
|
||||
{
|
||||
"id": "4412345680",
|
||||
"title": "Refused",
|
||||
"redirect_url": "javascript:alert(1)"
|
||||
}
|
||||
]
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
<?xml version="1.0" encoding="ISO-8859-1"?>
|
||||
|
||||
<rss version="2.0"
|
||||
xmlns:blogChannel="http://backend.userland.com/blogChannelModule"
|
||||
>
|
||||
|
||||
<channel>
|
||||
<title>Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer</title>
|
||||
<link>https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date</link>
|
||||
<description>Danmarks største jobmarked</description>
|
||||
<language>da</language>
|
||||
<docs>http://blogs.law.harvard.edu/tech/rss</docs>
|
||||
<webMaster>support@jobindex.dk (Jobindex)</webMaster>
|
||||
<ttl>1440</ttl>
|
||||
|
||||
<image>
|
||||
<title>Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer</title>
|
||||
<url>https://www.jobindex.dk/img/jobindex130x30gray.gif</url>
|
||||
<link>https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date</link>
|
||||
</image>
|
||||
<item>
|
||||
<title>Dangerous Goods Compliance Manager – ensure global safety, DSV</title>
|
||||
<link>https://www.jobindex.dk/vis-job/h1701165</link>
|
||||
<description>
|
||||
|
||||
<div class="jobsearch-result">
|
||||
<div class="PaidJob">
|
||||
|
||||
|
||||
<div class="PaidJob-inner">
|
||||
|
||||
|
||||
<CENTER style="margin-bottom: 15px;"><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/img/brand/40_spo_top_dsv_1812_20250718.png" ALT="DSV" style="border: 0px; margin: 0;"></a></CENTER>
|
||||
<h4><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank">Dangerous Goods Compliance Manager – ensure global safety</a></h4>
|
||||
|
||||
<div class="jobad-element-area">
|
||||
<span class="jix_robotjob--area">Hedehusene</span>
|
||||
|
||||
</div>
|
||||
|
||||
<p>Do you want to minimise workplace accidents and damage to property, while helping ensure that dangerous goods and hazardous materials are handled correctly at our nearly 3,000 locations worldwide?</p><p>“One minute, you’ll help a forwarding agent in Holland figure out what to do with a pallet. The next, you’ll help shape our strategic decisions after the merger with DB Schenker,” says Martin Guldhammer.</p>
|
||||
<center><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/api/imagescaler/v1/brand/8_spo_bund_dsv_1812_20250718.png?max_width=525" alt="DSV" style="border: 0px; margin-top: 5px;"></a></center>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</description>
|
||||
<category>Logistik og spedition</category>
|
||||
<guid isPermaLink="false">https://www.jobindex.dk/1701165</guid>
|
||||
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>Senior Account Executive</title>
|
||||
<link>https://www.jobindex.dk/vis-job/r14011701</link>
|
||||
<description>
|
||||
|
||||
<div class="">
|
||||
<div class="jobsearch-result">
|
||||
<div class="jix_robotjob" data-jobad-tid="r14011701" data-robotjob-abtest-enabled="">
|
||||
|
||||
|
||||
<div class="jix_robotjob-inner">
|
||||
|
||||
|
||||
<h4><A HREF="https://www.jobindex.dk/vis-job/r14011701" rel="noopener" target="_blank">
|
||||
Senior Account Executive
|
||||
</a></h4>
|
||||
|
||||
<div class="jobad-element-area">
|
||||
<span class="jix_robotjob--area">København</span>
|
||||
|
||||
</div>
|
||||
|
||||
<p>All Gravy is the AI-native people operating platform for hospitality and retail.</p>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</description>
|
||||
<category>Salg</category>
|
||||
<guid isPermaLink="false">https://www.jobindex.dk/14011701</guid>
|
||||
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>Marketing Content Creator, Clerk.IO ApS</title>
|
||||
<link>https://www.jobindex.dk/vis-job/r14011706</link>
|
||||
<description>
|
||||
|
||||
<div class="">
|
||||
<div class="jobsearch-result">
|
||||
<div class="jix_robotjob" data-jobad-tid="r14011706" data-robotjob-abtest-enabled="">
|
||||
|
||||
|
||||
<div class="jix_robotjob-inner">
|
||||
|
||||
|
||||
<h4><A HREF="https://www.jobindex.dk/vis-job/r14011706" rel="noopener" target="_blank">
|
||||
Marketing Content Creator
|
||||
</a></h4>
|
||||
|
||||
<div class="jobad-element-area">
|
||||
<span class="jix_robotjob--area">København</span>
|
||||
|
||||
</div>
|
||||
|
||||
<p>Your marketing degree gave you the foundation.</p>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</description>
|
||||
<category>Studiepraktik</category>
|
||||
<category>Marketing</category>
|
||||
<guid isPermaLink="false">https://www.jobindex.dk/14011706</guid>
|
||||
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
<?xml version="1.0" encoding="ISO-8859-1"?>
|
||||
|
||||
<rss version="2.0"
|
||||
xmlns:blogChannel="http://backend.userland.com/blogChannelModule"
|
||||
>
|
||||
|
||||
<channel>
|
||||
<title>Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer</title>
|
||||
<link>https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date</link>
|
||||
<description>Danmarks største jobmarked</description>
|
||||
<language>da</language>
|
||||
<docs>http://blogs.law.harvard.edu/tech/rss</docs>
|
||||
<webMaster>support@jobindex.dk (Jobindex)</webMaster>
|
||||
<ttl>1440</ttl>
|
||||
|
||||
<image>
|
||||
<title>Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer</title>
|
||||
<url>https://www.jobindex.dk/img/jobindex130x30gray.gif</url>
|
||||
<link>https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date</link>
|
||||
</image>
|
||||
<item>
|
||||
<title>Dangerous Goods Compliance Manager – ensure global safety, DSV</title>
|
||||
<link>https://www.jobindex.dk/vis-job/h1701165</link>
|
||||
<description>
|
||||
|
||||
<div class="jobsearch-result">
|
||||
<div class="PaidJob">
|
||||
|
||||
|
||||
<div class="PaidJob-inner">
|
||||
|
||||
|
||||
<CENTER style="margin-bottom: 15px;"><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/img/brand/40_spo_top_dsv_1812_20250718.png" ALT="DSV" style="border: 0px; margin: 0;"></a></CENTER>
|
||||
<h4><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank">Dangerous Goods Compliance Manager – ensure global safety</a></h4>
|
||||
|
||||
<div class="jobad-element-area">
|
||||
<span class="jix_robotjob--area">Hedehusene</span>
|
||||
|
||||
</div>
|
||||
|
||||
<p>Do you want to minimise workplace accidents and damage to property, while helping ensure that dangerous goods and hazardous materials are handled correctly at our nearly 3,000 locations worldwide?</p><p>“One minute, you’ll help a forwarding agent in Holland figure out what to do with a pallet. The next, you’ll help shape our strategic decisions after the merger with DB Schenker,” says Martin Guldhammer.</p>
|
||||
<center><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/api/imagescaler/v1/brand/8_spo_bund_dsv_1812_20250718.png?max_width=525" alt="DSV" style="border: 0px; margin-top: 5px;"></a></center>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</description>
|
||||
<category>Logistik og spedition</category>
|
||||
<guid isPermaLink="false">https://www.jobindex.dk/1701165</guid>
|
||||
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>Senior Account Executive</title>
|
||||
<link>https://www.jobindex.dk/vis-job/r14011701</link>
|
||||
<description>
|
||||
|
||||
<div class="">
|
||||
<div class="jobsearch-result">
|
||||
<div class="jix_robotjob" data-jobad-tid="r14011701" data-robotjob-abtest-enabled="">
|
||||
|
||||
|
||||
<div class="jix_robotjob-inner">
|
||||
|
||||
|
||||
<h4><A HREF="https://www.jobindex.dk/vis-job/r14011701" rel="noopener" target="_blank">
|
||||
Senior Account Executive
|
||||
</a></h4>
|
||||
|
||||
<div class="jobad-element-area">
|
||||
<span class="jix_robotjob--area">København</span>
|
||||
|
||||
</div>
|
||||
|
||||
<p>All Gravy is the AI-native people operating platform for hospitality and retail.</p>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</description>
|
||||
<category>Salg</category>
|
||||
<guid isPermaLink="false">https://www.jobindex.dk/14011701</guid>
|
||||
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>Marketing Content Creator, Clerk.IO ApS</title>
|
||||
<link>https://www.jobindex.dk/vis-job/r14011706</link>
|
||||
<description>
|
||||
|
||||
<div class="">
|
||||
<div class="jobsearch-result">
|
||||
<div class="jix_robotjob" data-jobad-tid="r14011706" data-robotjob-abtest-enabled="">
|
||||
|
||||
|
||||
<div class="jix_robotjob-inner">
|
||||
|
||||
|
||||
<h4><A HREF="https://www.jobindex.dk/vis-job/r14011706" rel="noopener" target="_blank">
|
||||
Marketing Content Creator
|
||||
</a></h4>
|
||||
|
||||
<div class="jobad-element-area">
|
||||
<span class="jix_robotjob--area">København</span>
|
||||
|
||||
</div>
|
||||
|
||||
<p>Your marketing degree gave you the foundation.</p>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</description>
|
||||
<category>Studiepraktik</category>
|
||||
<category>Marketing</category>
|
||||
<guid isPermaLink="false">https://www.jobindex.dk/14011706</guid>
|
||||
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"total": {
|
||||
"value": 32
|
||||
},
|
||||
"hits": [
|
||||
{
|
||||
"id": "31457942",
|
||||
"headline": "Postdoctoral Researcher",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31457942",
|
||||
"publication_date": "2026-09-09T15:47:27",
|
||||
"description": {
|
||||
"text": "The European Spallation Source (ESS) is one of the largest."
|
||||
},
|
||||
"employer": {
|
||||
"name": "European Spallation Source Eric"
|
||||
},
|
||||
"workplace_model": {
|
||||
"concept_id": "ZxwM_gtz_n61",
|
||||
"label": "Arbete på plats"
|
||||
},
|
||||
"workplace_address": {
|
||||
"municipality": "Lund",
|
||||
"region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU",
|
||||
"country": "Sverige"
|
||||
},
|
||||
"workplace_addresses": [
|
||||
{
|
||||
"municipality": "Lund",
|
||||
"region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU",
|
||||
"country": "Sverige"
|
||||
},
|
||||
{
|
||||
"municipality": "Stockholm",
|
||||
"region": "Stockholms län",
|
||||
"region_concept_id": "CifL_Rzy_Mku",
|
||||
"country": "Sverige"
|
||||
},
|
||||
{
|
||||
"municipality": null,
|
||||
"region": "Västra Götalands län",
|
||||
"region_concept_id": "zdoY_6u5_Krt",
|
||||
"country": "Sverige"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "31435504",
|
||||
"headline": "Postdoctoral Fellow in Robotics and Computer Vision",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31435504",
|
||||
"publication_date": "2026-09-03T15:46:06",
|
||||
"description": {
|
||||
"text": "Robotics at ESS."
|
||||
},
|
||||
"employer": {
|
||||
"name": "European Spallation Source Eric"
|
||||
},
|
||||
"workplace_model": {
|
||||
"concept_id": null,
|
||||
"label": null
|
||||
},
|
||||
"workplace_address": {
|
||||
"municipality": null,
|
||||
"region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU",
|
||||
"country": "Sverige"
|
||||
},
|
||||
"workplace_addresses": [
|
||||
{
|
||||
"municipality": null,
|
||||
"region": "Skåne län",
|
||||
"region_concept_id": "CaRE_1nn_cSU",
|
||||
"country": "Sverige"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "31400001",
|
||||
"headline": "Data scientist, distansarbete möjligt",
|
||||
"webpage_url": "javascript:alert(1)",
|
||||
"workplace_address": {},
|
||||
"workplace_addresses": [
|
||||
{}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "31400002",
|
||||
"headline": "Data scientist, distansarbete möjligt",
|
||||
"webpage_url": "https://arbetsformedlingen.se/platsbanken/annonser/31400002",
|
||||
"employer": {
|
||||
"name": null
|
||||
},
|
||||
"workplace_address": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
"""The Adzuna reader, built against Adzuna's documented response (no key to
|
||||
run it live). No network: `get` is a stub."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, adzuna # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
|
||||
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
|
||||
"key": {"app_id": "id1", "app_key": "key1"}})
|
||||
|
||||
|
||||
def _rows(ask: Ask = ASK) -> list[dict[str, Any]]:
|
||||
decoded, _results, _total, _cursor = adzuna.page(lambda url, **kw: BODY, "", None, ask)
|
||||
return list(adzuna.rows(decoded, ""))
|
||||
|
||||
|
||||
def test_it_declares_the_key_and_states_the_robots_finding() -> None:
|
||||
assert readers.PROVIDERS["adzuna"] is adzuna
|
||||
reads = adzuna.READS
|
||||
assert reads.key is not None and reads.key.name == "adzuna"
|
||||
assert (reads.key.fields, reads.key.env) == (
|
||||
("app_id", "app_key"), ("ADZUNA_APP_ID", "ADZUNA_APP_KEY"))
|
||||
assert (reads.text, reads.complete, reads.page_size) == ("excerpt", "stated", 50)
|
||||
assert "disallows /" in reads.robots and "2026-09-30" in reads.robots
|
||||
|
||||
|
||||
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
|
||||
first, predicted = _rows() # javascript: refused
|
||||
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
|
||||
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
||||
assert first["teaser"] == "Build our backend services in Python & Go."
|
||||
assert "description" not in first
|
||||
assert first["company"] == "Acme GmbH"
|
||||
assert first["places"] == [{"text": "Berlin, Deutschland", "remote": None}]
|
||||
assert first["country"] == "DE" and first["remote"] == "remote"
|
||||
assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR")
|
||||
assert (predicted["salary_min"], predicted["currency"]) == (None, None)
|
||||
|
||||
|
||||
def test_it_sends_the_key_the_country_and_the_query() -> None:
|
||||
asked: list[str] = []
|
||||
|
||||
def get(url: str, **kwargs: Any) -> bytes:
|
||||
asked.append(url)
|
||||
return BODY
|
||||
|
||||
adzuna.page(get, "", None, ASK)
|
||||
parts = urlsplit(asked[0])
|
||||
assert (parts.netloc, parts.path) == ("api.adzuna.com", "/v1/api/jobs/de/search/1")
|
||||
assert parse_qs(parts.query) == {
|
||||
"app_id": ["id1"], "app_key": ["key1"], "results_per_page": ["50"],
|
||||
"sort_by": ["date"], "what": ["backend"], "where": ["Berlin"]}
|
||||
|
||||
|
||||
def test_it_refuses_an_unknown_country() -> None:
|
||||
with pytest.raises(ValueError, match="country"):
|
||||
adzuna.page(lambda url, **kw: BODY, "", None,
|
||||
Ask(params={"q": "x", "country": "dk"}))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("results", "count", "after"), [(50, 120, 2), (49, 49, None)])
|
||||
def test_it_pages_on_a_full_page_and_states_the_count(
|
||||
results: int, count: int, after: int | None) -> None:
|
||||
body = json.dumps({"count": count, "results": [
|
||||
{"id": str(n), "title": "T", "redirect_url": f"https://x.example/{n}"}
|
||||
for n in range(results)]}).encode()
|
||||
_decoded, found, total, nxt = adzuna.page(lambda url, **kw: body, "", None, ASK)
|
||||
assert (len(found), total, nxt) == (results, count, after)
|
||||
|
||||
|
||||
def test_saved_params_need_a_known_country() -> None:
|
||||
assert adzuna.clean_params({"q": "x", "country": " DE ", "where": " Berlin "}) == {
|
||||
"q": "x", "country": "de", "where": "Berlin"}
|
||||
assert adzuna.clean_params({"q": "x", "country": "gb", "where": ""}) == {
|
||||
"q": "x", "country": "gb"}
|
||||
with pytest.raises(ValueError, match="country"):
|
||||
adzuna.clean_params({"q": "x", "country": "dk"})
|
||||
@@ -0,0 +1,111 @@
|
||||
"""The Jobindex reader: the RSS twin of a search, 20 results a page, 200 at
|
||||
most. No network: `get` is a stub."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, jobindex # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
|
||||
|
||||
|
||||
def _rows(body: bytes = BODY) -> list[dict[str, Any]]:
|
||||
decoded, _results, _total, _cursor = jobindex.page(lambda url, **kw: body, "", None, Ask())
|
||||
return list(jobindex.rows(decoded, ""))
|
||||
|
||||
|
||||
def _feed(items: int) -> bytes:
|
||||
body = "".join(f"<item><title>Role {n}, Acme</title>"
|
||||
f"<link>https://www.jobindex.dk/vis-job/r{n}</link></item>"
|
||||
for n in range(items))
|
||||
return (f"<rss><channel><title>Jobindex</title>{body}</channel></rss>").encode()
|
||||
|
||||
|
||||
def test_it_is_a_search_reader_with_the_owners_robots_decision() -> None:
|
||||
assert readers.PROVIDERS["jobindex"] is jobindex
|
||||
reads = jobindex.READS
|
||||
assert (reads.scope, reads.page_size, reads.ceiling, reads.text) == (
|
||||
"search", 20, 200, "excerpt")
|
||||
assert "owner, 2026-09-30" in reads.robots and "geoareaid" in reads.robots
|
||||
assert jobindex.allowed("")("www.jobindex.dk") and not jobindex.allowed("")("jobindex.dk")
|
||||
|
||||
|
||||
def test_it_reads_title_company_town_date_and_teaser() -> None:
|
||||
paid, bare, robot = _rows()
|
||||
assert [r["source_id"] for r in (paid, bare, robot)] == ["h1701165", "r14011701", "r14011706"]
|
||||
assert paid["company"] == "DSV"
|
||||
assert paid["title"].startswith("Dangerous Goods Compliance Manager")
|
||||
assert paid["places"] == [{"text": "Hedehusene", "remote": None}]
|
||||
assert paid["location"] == "Hedehusene" and paid["country"] == "DK"
|
||||
assert paid["posted"].date().isoformat() == "2026-09-23"
|
||||
assert paid["teaser"].startswith("Do you want to minimise")
|
||||
assert "description" not in paid
|
||||
assert bare["company"] == "" and bare["places"][0]["text"] == "København"
|
||||
assert (robot["title"], robot["company"]) == ("Marketing Content Creator", "Clerk.IO ApS")
|
||||
|
||||
|
||||
def test_it_reads_hybrid_and_remote_from_the_town_line() -> None:
|
||||
from plugins.readers._words import remote_word
|
||||
|
||||
assert remote_word("Birkerød, Hybrid position") == "hybrid"
|
||||
assert remote_word("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
|
||||
assert remote_word("Remote, Danmark") == "remote"
|
||||
assert remote_word("Hedehusene") is None
|
||||
|
||||
|
||||
def test_it_asks_for_the_query_the_area_and_the_page() -> None:
|
||||
asked: list[str] = []
|
||||
|
||||
def get(url: str, **kwargs: Any) -> bytes:
|
||||
asked.append(url)
|
||||
return _feed(3)
|
||||
|
||||
jobindex.page(get, "", None, Ask(params={"q": "backend developer", "area": 1}))
|
||||
parts = urlsplit(asked[0])
|
||||
assert (parts.scheme, parts.netloc, parts.path) == (
|
||||
"https", "www.jobindex.dk", "/jobsoegning.rss")
|
||||
assert parse_qs(parts.query) == {"q": ["backend developer"], "sort": ["date"],
|
||||
"geoareaid": ["1"], "page": ["1"]}
|
||||
jobindex.page(get, "", 4, Ask(params={"q": "x"}))
|
||||
assert "geoareaid" not in asked[1] and "page=4" in asked[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("items", "cursor", "after"), [
|
||||
(20, None, 2), # a full page has a next
|
||||
(19, None, None), # a short page is the end
|
||||
(20, 10, None), # the tenth page reaches the 200 ceiling
|
||||
(20, 9, 10),
|
||||
])
|
||||
def test_it_pages_until_a_short_page_or_the_ceiling(
|
||||
items: int, cursor: int | None, after: int | None) -> None:
|
||||
_decoded, results, total, nxt = jobindex.page(
|
||||
lambda url, **kw: _feed(items), "", cursor, Ask(params={"q": "x"}))
|
||||
assert (len(results), total, nxt) == (items, None, after)
|
||||
|
||||
|
||||
def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
||||
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
|
||||
assert len(results) == 20 and nxt == 2
|
||||
assert len(jobindex.rows(decoded, "")) == 19
|
||||
|
||||
|
||||
def test_an_answer_without_a_channel_raises() -> None:
|
||||
with pytest.raises(ValueError, match="channel"):
|
||||
jobindex.page(lambda url, **kw: b"<rss/>", "", None, Ask())
|
||||
|
||||
|
||||
def test_saved_params_keep_the_query_and_a_real_area() -> None:
|
||||
assert jobindex.clean_params({"q": "x", "area": "3"}) == {"q": "x", "area": 3}
|
||||
assert jobindex.clean_params({"q": "x", "area": 0}) == {"q": "x"}
|
||||
@@ -0,0 +1,111 @@
|
||||
"""The Platsbanken reader (JobTech search API): 100 hits a page, offsets to
|
||||
2000. No network: `get` is a stub."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, platsbanken # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
|
||||
|
||||
|
||||
def _stock(total: int | None, stock: int, asked: list[str]) -> Any:
|
||||
def get(url: str, **kwargs: Any) -> bytes:
|
||||
asked.append(url)
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
start, limit = int(query["offset"][0]), int(query["limit"][0])
|
||||
body: dict[str, Any] = {"hits": [
|
||||
{"id": str(n), "headline": f"Role {n}", "webpage_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + limit, stock))]}
|
||||
if total is not None:
|
||||
body["total"] = {"value": total}
|
||||
return json.dumps(body).encode()
|
||||
return get
|
||||
|
||||
|
||||
def test_it_is_a_search_reader_with_a_404_robots_txt_recorded() -> None:
|
||||
assert readers.PROVIDERS["platsbanken"] is platsbanken
|
||||
reads = platsbanken.READS
|
||||
assert (reads.scope, reads.page_size, reads.ceiling, reads.places) == (
|
||||
"search", 100, 2000, "many")
|
||||
assert "404" in reads.robots and "2026-09-30" in reads.robots
|
||||
|
||||
|
||||
def test_it_keeps_every_workplace_of_an_ad() -> None:
|
||||
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||
rows = platsbanken.rows(decoded, "")
|
||||
assert total == 32
|
||||
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
||||
first, no_town, bare = rows
|
||||
assert [p["text"] for p in first["places"]] == [
|
||||
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
|
||||
assert first["location"] == "Lund"
|
||||
assert no_town["places"] == [{"text": "Skåne län", "remote": None}]
|
||||
assert bare["places"] == []
|
||||
|
||||
|
||||
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
||||
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||
first, _no_town, remote = platsbanken.rows(decoded, "")
|
||||
assert first["title"] == "Postdoctoral Researcher"
|
||||
assert first["company"] == "European Spallation Source Eric"
|
||||
assert first["country"] == "SE" and first["remote"] is None
|
||||
assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942"
|
||||
assert first["posted"].date().isoformat() == "2026-09-09"
|
||||
assert first["description"].startswith("The European Spallation Source")
|
||||
assert "teaser" not in first
|
||||
assert (remote["remote"], remote["company"], remote["posted"]) == ("remote", "", None)
|
||||
|
||||
|
||||
def test_it_sends_the_region_the_offset_and_no_region_for_all_of_sweden() -> None:
|
||||
asked: list[str] = []
|
||||
get = _stock(250, 250, asked)
|
||||
platsbanken.page(get, "", None, Ask(params={"q": "data", "region": "CaRE_1nn_cSU"}))
|
||||
platsbanken.page(get, "", 100, Ask(params={"q": "data", "region": ""}))
|
||||
assert parse_qs(urlsplit(asked[0]).query) == {
|
||||
"q": ["data"], "sort": ["pubdate-desc"], "limit": ["100"], "offset": ["0"],
|
||||
"region": ["CaRE_1nn_cSU"]}
|
||||
assert "region" not in urlsplit(asked[1]).query and "offset=100" in asked[1]
|
||||
assert urlsplit(asked[0]).netloc == "jobsearch.api.jobtechdev.se"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("cursor", "stock", "after"), [
|
||||
(None, 250, 100), (200, 250, None), # a short page is the end
|
||||
(1900, 2500, 2000), (2000, 2500, None), # the offset ceiling
|
||||
])
|
||||
def test_it_pages_by_offset_up_to_the_ceiling(
|
||||
cursor: int | None, stock: int, after: int | None) -> None:
|
||||
_decoded, _found, _total, nxt = platsbanken.page(
|
||||
_stock(None, stock, []), "", cursor, Ask(params={"q": "x"}))
|
||||
assert nxt == after
|
||||
|
||||
|
||||
def test_it_refuses_an_unknown_region() -> None:
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
platsbanken.page(lambda url, **kw: b"{}", "", None,
|
||||
Ask(params={"q": "x", "region": "zYpL_aGk_QY8"})) # Midland
|
||||
|
||||
|
||||
def test_the_regions_are_the_21_lan() -> None:
|
||||
assert len(platsbanken.REGIONS) == 21
|
||||
assert all(name.endswith(" län") for name in platsbanken.REGIONS.values())
|
||||
|
||||
|
||||
def test_saved_params_keep_a_län_or_none() -> None:
|
||||
assert platsbanken.clean_params({"q": "x", "region": " CaRE_1nn_cSU "}) == {
|
||||
"q": "x", "region": "CaRE_1nn_cSU"}
|
||||
assert platsbanken.clean_params({"q": "x", "region": ""}) == {"q": "x", "region": ""}
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
platsbanken.clean_params({"q": "x", "region": "zYpL_aGk_QY8"})
|
||||
@@ -573,6 +573,17 @@ def test_search_params_keeps_only_what_the_provider_reads() -> None:
|
||||
"q": "data", "region": "CaRE_1nn_cSU", "pages": 3}
|
||||
|
||||
|
||||
def test_the_providers_and_their_params_follow_the_reader_declarations() -> None:
|
||||
from plugins import readers
|
||||
|
||||
assert sources.PROVIDERS == tuple(readers.PROVIDERS) == ("jobindex", "adzuna", "platsbanken")
|
||||
assert sources.search_params("adzuna", " backend ", country=" DE ", where=" Berlin ",
|
||||
area=1, region="x", pages=2) == {
|
||||
"q": "backend", "country": "de", "where": "Berlin", "pages": 2}
|
||||
with pytest.raises(ValueError, match="country"):
|
||||
sources.search_params("adzuna", "backend", country="dk")
|
||||
|
||||
|
||||
def test_search_params_refuses_a_region_that_is_no_swedish_lan() -> None:
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
sources.search_params("platsbanken", "data", region="zYpL_aGk_QY8") # Midland
|
||||
|
||||
Reference in new issue
Block a user