170 lines
7.1 KiB
Python
170 lines
7.1 KiB
Python
"""Jobindex: Denmark's largest job board, searched through the RSS twin of
|
||
its search page, `/jobsoegning.rss`; no key.
|
||
|
||
Checked live on 2026-09-23 and moved here from `plugins/search.py`. `q` takes
|
||
Jobindex's own syntax: words are OR, `"a b"` a phrase, `+word` required.
|
||
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region Sjælland, 2
|
||
Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. 20 results a page, and no
|
||
page past the tenth answers anything, so a search reads 200 results at most.
|
||
The feed states no total, so a search is never complete.
|
||
|
||
An item is `<title>` "Role, Company", a `<link>` to Jobindex's own page for it
|
||
and a `<description>` of HTML: the `<h4>` is the title alone, a paid ad's logo
|
||
has the company as its `alt`, the `jix_robotjob--area` span is the town and
|
||
the paragraphs are an excerpt of the ad, handed over as `teaser`. A crawled ad
|
||
("robot job") has no logo, so its company is what follows the title's last
|
||
comma, or '' when there is none. An answer with no `<channel>` is not a search
|
||
result and raises, since an empty list would read as a search that ran out.
|
||
|
||
The `<link>` opens Jobindex's `vis-job` page, and `posting_link` reads the
|
||
"Se jobbet" button on it: the employer's site, The Hub or Jobindex's own
|
||
`jobannonce`, checked on 2026-09-30. It is not part of the search: fetching
|
||
that page counts as a click in Jobindex's statistics, so `app.jobs.posting`
|
||
does it only for a role somebody opened. The page's "other ads" cards carry
|
||
the same words on `/c?t=` redirects, which are not this ad and are skipped.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import html
|
||
import re
|
||
import xml.etree.ElementTree as ET
|
||
from collections.abc import Callable
|
||
from typing import Any
|
||
from urllib.parse import urlencode, urljoin, urlsplit
|
||
|
||
from desk.sources import strip_html
|
||
|
||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, place
|
||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||
|
||
NAME = "jobindex"
|
||
LABEL = "Jobindex"
|
||
RANK = 10
|
||
|
||
READS = Reads(
|
||
scope="search", listing="paged", page_size=20, ceiling=200,
|
||
text="excerpt", complete="never",
|
||
filters=Filters(query="one", place="one"),
|
||
places="one",
|
||
fields=Fields(posted="trusted", country="stated"),
|
||
robots="read though robots.txt disallows sort=, page= and geoareaid=: owner, 2026-09-30")
|
||
|
||
#: Saved param -> the filter dimension it fills.
|
||
PARAMS = {"q": "query", "area": "place"}
|
||
|
||
RSS = "https://www.jobindex.dk/jobsoegning.rss"
|
||
|
||
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
|
||
#: against the titles its feeds answer with. 0 is all of Denmark.
|
||
AREAS: dict[int, str] = {
|
||
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
|
||
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
|
||
8: "Aarhus", 7: "Aalborg",
|
||
}
|
||
|
||
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
|
||
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
|
||
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
|
||
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
|
||
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
|
||
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
||
_SE_JOBBET = re.compile(r"<a\b([^>]*)>\s*Se jobbet\s*</a>", re.IGNORECASE)
|
||
_HREF = re.compile(r'\bhref\s*=\s*"([^"]*)"', re.IGNORECASE)
|
||
|
||
|
||
def allowed(slug: str) -> Callable[[str], bool]:
|
||
return only("www.jobindex.dk")
|
||
|
||
|
||
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
||
"""The saved params of a search: `q`, and `area` when it is a number
|
||
other than 0."""
|
||
out: dict[str, Any] = {"q": params["q"]}
|
||
if params.get("area"):
|
||
out["area"] = int(params["area"])
|
||
return out
|
||
|
||
|
||
def posting_link(page_html: str) -> str:
|
||
"""The "Se jobbet" link of a `vis-job` page: where the ad itself is, as an
|
||
absolute URL, or '' when the page has none. A `/c?t=` link is another ad's
|
||
redirect and is skipped."""
|
||
for tag in _SE_JOBBET.finditer(page_html):
|
||
found = _HREF.search(tag.group(1))
|
||
if not found:
|
||
continue
|
||
url = urljoin("https://www.jobindex.dk/", html.unescape(found.group(1)).strip())
|
||
if is_web_url(url) and urlsplit(url).path != "/c":
|
||
return url
|
||
return ""
|
||
|
||
|
||
def _text(fragment: str) -> str:
|
||
"""HTML to plain text. Jobindex breaks titles after a slash with a
|
||
zero-width space, which would otherwise end up in `title_key`."""
|
||
return html.unescape(strip_html(fragment)).replace("", "").strip()
|
||
|
||
|
||
def _query(ask: Ask, number: int) -> str:
|
||
query: dict[str, Any] = {"q": ask.query or str(ask.params.get("q") or ""),
|
||
"sort": "date"}
|
||
area = ask.places[0] if ask.places else ask.params.get("area")
|
||
if area:
|
||
query["geoareaid"] = area
|
||
return RSS + "?" + urlencode(query | {"page": number})
|
||
|
||
|
||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||
"""One page of a search, newest first; the cursor is the page number.
|
||
The next page follows a full page (counted before any item is dropped, so
|
||
a dropped item does not pass for the end) while the 200-result ceiling is
|
||
not reached."""
|
||
number = int(cursor or 1)
|
||
root = ET.fromstring(get(_query(ask, number)))
|
||
channel = root.find("channel")
|
||
if channel is None:
|
||
raise ValueError("jobindex answered without an RSS channel")
|
||
match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
||
items = [{
|
||
"id": (item.findtext("link") or "").strip().rstrip("/").rsplit("/", 1)[-1],
|
||
"link": (item.findtext("link") or "").strip(),
|
||
"title": (item.findtext("title") or "").strip(),
|
||
"description": item.findtext("description") or "",
|
||
"published": item.findtext("pubDate"),
|
||
} for item in channel.findall("item")]
|
||
decoded = {"area": match.group(1).strip() if match else "", "items": items}
|
||
size, ceiling = READS.page_size or 20, READS.ceiling or 200
|
||
more = len(items) >= size and number * size < ceiling
|
||
return decoded, [{"id": item["id"]} for item in items], None, number + 1 if more else None
|
||
|
||
|
||
def rows(decoded: Any, slug: str) -> list[Row]:
|
||
out: list[Row] = []
|
||
for item in decoded["items"]:
|
||
link, full_title = item["link"], item["title"]
|
||
if not full_title or not is_web_url(link):
|
||
continue
|
||
description = item["description"]
|
||
h4 = _H4.search(description)
|
||
title = _text(h4.group(1)) if h4 else full_title
|
||
alt = _ALT.search(description)
|
||
company = html.unescape(alt.group(1)).strip() if alt else ""
|
||
if not company and full_title.startswith(title + ", "):
|
||
company = full_title[len(title) + 2:].strip()
|
||
area = _AREA.search(description)
|
||
town = _text(area.group(1)) if area else decoded["area"]
|
||
places: list[Place] = [place(town)] if town else []
|
||
out.append({
|
||
"source_id": item["id"],
|
||
"title": title or full_title,
|
||
"company": company,
|
||
"url": link,
|
||
"places": places,
|
||
"location": town,
|
||
"country": "DK",
|
||
"posted": parse_dt(item["published"]),
|
||
"teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
||
})
|
||
return out
|