Files
jobboard/plugins/readers/jobindex.py
T

150 lines
6.1 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Jobindex: Denmark's largest job board, searched through the RSS twin of
its search page, `/jobsoegning.rss`; no key.
Checked live on 2026-09-23 and moved here from `plugins/search.py`. `q` takes
Jobindex's own syntax: words are OR, `"a b"` a phrase, `+word` required.
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region Sjælland, 2
Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. 20 results a page, and no
page past the tenth answers anything, so a search reads 200 results at most.
The feed states no total, so a search is never complete.
An item is `<title>` "Role, Company", a `<link>` to Jobindex's own page for it
and a `<description>` of HTML: the `<h4>` is the title alone, a paid ad's logo
has the company as its `alt`, the `jix_robotjob--area` span is the town and
the paragraphs are an excerpt of the ad, handed over as `teaser`. A crawled ad
("robot job") has no logo, so its company is what follows the title's last
comma, or '' when there is none. An answer with no `<channel>` is not a search
result and raises, since an empty list would read as a search that ran out.
"""
from __future__ import annotations
import html
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable
from typing import Any
from urllib.parse import urlencode
from desk.sources import strip_html
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._contract import Ask, Fields, Filters, Place, Reads
NAME = "jobindex"
LABEL = "Jobindex"
RANK = 10
READS = Reads(
scope="search", listing="paged", page_size=20, ceiling=200,
text="excerpt", complete="never",
filters=Filters(query="one", place="one"),
places="one",
fields=Fields(posted="trusted", country="stated"),
robots="read though robots.txt disallows sort=, page= and geoareaid=: owner, 2026-09-30")
#: Saved param -> the filter dimension it fills.
PARAMS = {"q": "query", "area": "place"}
RSS = "https://www.jobindex.dk/jobsoegning.rss"
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
#: against the titles its feeds answer with. 0 is all of Denmark.
AREAS: dict[int, str] = {
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
8: "Aarhus", 7: "Aalborg",
}
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
def allowed(slug: str) -> Callable[[str], bool]:
return only("www.jobindex.dk")
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
"""The saved params of a search: `q`, and `area` when it is a number
other than 0."""
out: dict[str, Any] = {"q": params["q"]}
if params.get("area"):
out["area"] = int(params["area"])
return out
def _text(fragment: str) -> str:
"""HTML to plain text. Jobindex breaks titles after a slash with a
zero-width space, which would otherwise end up in `title_key`."""
return html.unescape(strip_html(fragment)).replace("​", "").strip()
def _query(ask: Ask, number: int) -> str:
query: dict[str, Any] = {"q": ask.query or str(ask.params.get("q") or ""),
"sort": "date"}
area = ask.places[0] if ask.places else ask.params.get("area")
if area:
query["geoareaid"] = area
return RSS + "?" + urlencode(query | {"page": number})
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search, newest first; the cursor is the page number.
The next page follows a full page (counted before any item is dropped, so
a dropped item does not pass for the end) while the 200-result ceiling is
not reached."""
number = int(cursor or 1)
root = ET.fromstring(get(_query(ask, number)))
channel = root.find("channel")
if channel is None:
raise ValueError("jobindex answered without an RSS channel")
match = _SEARCH_AREA.search(channel.findtext("title") or "")
items = [{
"id": (item.findtext("link") or "").strip().rstrip("/").rsplit("/", 1)[-1],
"link": (item.findtext("link") or "").strip(),
"title": (item.findtext("title") or "").strip(),
"description": item.findtext("description") or "",
"published": item.findtext("pubDate"),
} for item in channel.findall("item")]
decoded = {"area": match.group(1).strip() if match else "", "items": items}
size, ceiling = READS.page_size or 20, READS.ceiling or 200
more = len(items) >= size and number * size < ceiling
return decoded, [{"id": item["id"]} for item in items], None, number + 1 if more else None
def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = []
for item in decoded["items"]:
link, full_title = item["link"], item["title"]
if not full_title or not is_web_url(link):
continue
description = item["description"]
h4 = _H4.search(description)
title = _text(h4.group(1)) if h4 else full_title
alt = _ALT.search(description)
company = html.unescape(alt.group(1)).strip() if alt else ""
if not company and full_title.startswith(title + ", "):
company = full_title[len(title) + 2:].strip()
area = _AREA.search(description)
town = _text(area.group(1)) if area else decoded["area"]
remote = remote_word(f"{town} {title}")
places: list[Place] = [{"text": town, "remote": True if remote == "remote" else None}] \
if town else []
out.append({
"source_id": item["id"],
"title": title or full_title,
"company": company,
"url": link,
"places": places,
"location": town,
"country": "DK",
"remote": remote,
"posted": parse_dt(item["published"]),
"teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
})
return out