Add the freehire search reader: tech roles of many countries with full text, yearly pay and work mode

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:19:15 +02:00
1 parent 018a1ad895
commit 59e43f18b7
3 files changed
+862

No files matched your search

+190
View File
@@ -0,0 +1,190 @@
"""freehire: an open aggregator of IT jobs crawled from about 50 hosts (company
boards, Jobnet, The Hub and others), through its public JSON API at
freehire.me/api/v1; no key.
Checked live on 2026-10-03. The search is `/agent/jobs/search`, whose hits
carry the full description (`description_format=html`), at 300 requests a
minute; `limit` is at most 100 and `offset + limit` at most 10,000, so a
search reads 10,000 results at most. `meta.total` is the count. `countries`
(lower-case ISO codes) and `work_mode` repeat in the query; several
countries are one request, so `country` is a `many` filter. `q` is noisy:
unquoted words are OR over title, company, text and place, so a search is
narrowed by `country` and a few distinctive words. A hit's `source` names
where freehire crawled it (`greenhouse`, `jobnet`, `whatjobs-dk`); it is kept
in `raw`, since a role freehire took from a board roledesk reads is a copy
of that board's role. `url` carries `utm_source=freehire.me`, which is
removed.
A hit states `cities` (canonical names, at times those of other sites of
the employer) and `location`, the posting's own words; every one of them is
a place of the role. `enrichment` is model-extracted and may state
`salary_min`, `salary_max`, `salary_currency` and `salary_period`; only a
yearly figure goes to the pay columns, the other periods are said in
`salary_text`. `enrichment` also has `seniority` and `employment_type`,
which this reader does not hand on yet: a reader that states them needs a
table of the host's words in `app/prep/hosts/freehire.py`.
"""
from __future__ import annotations
import json
import re
from collections.abc import Callable
from typing import Any
from urllib.parse import urlencode, urlsplit, urlunsplit
from desk.sources import USER_AGENT
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line,
yearly_amount)
NAME = "freehire"
LABEL = "freehire (tech, many countries)"
RANK = 40
READS = Reads(
scope="search", listing="paged", page_size=100, ceiling=10000,
text="listing", complete="stated",
filters=Filters(query="one", country="many", remote="one"),
places="many",
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated"),
robots="read: robots.txt allows / for every agent; its Disallow of /api/ is for "
"Googlebot and Bingbot only, and its comments invite API use with a "
"descriptive User-Agent (checked 2026-10-03)")
#: Saved param -> the filter dimension it fills.
PARAMS = {"q": "query", "country": "country", "remote": "remote"}
SEARCH = "https://freehire.me/api/v1/agent/jobs/search"
#: `work_mode` of freehire's facets, for the form: value -> label. "" is any.
MODES: dict[str, str] = {"remote": "Remote", "hybrid": "Hybrid", "onsite": "On site"}
_CODE = re.compile(r"[a-z]{2}")
def allowed(slug: str) -> Callable[[str], bool]:
return only("freehire.me")
def _codes(value: Any) -> list[str]:
"""Country codes from a comma-separated string or a list, lower-case, one
of each in the order given; a ValueError for anything but two letters."""
items = value if isinstance(value, list | tuple | set) else [value]
out: list[str] = []
for item in items:
for code in str(item or "").replace(" ", ",").lower().split(","):
if code and code not in out:
if not _CODE.fullmatch(code):
raise ValueError(f"country is empty or ISO codes like dk,se, not {code!r}")
out.append(code)
return out
def _mode(value: Any) -> str:
mode = str(value or "").strip().lower()
if mode and mode not in MODES:
raise ValueError(f"remote is empty or one of {', '.join(MODES)}")
return mode
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
"""The saved params of a search: `q`, `country` as a comma-separated list
of ISO codes (empty for all countries) and `remote`, one of `MODES` or
empty for any."""
return {"q": params["q"], "country": ",".join(_codes(params.get("country"))),
"remote": _mode(params.get("remote"))}
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search, newest first; the cursor is the offset. The next
page follows a full page while its end is within the ceiling. The
countries are the ask's or the saved param's (a comma-separated string
reaches `Ask.countries` as one value), the work mode the saved param's or
`remote` of the ask."""
countries = _codes(list(ask.countries) or ask.params.get("country"))
mode = _mode(ask.params.get("remote")) or ("remote" if ask.remote else "")
size, ceiling = READS.page_size or 100, READS.ceiling or 10000
offset = int(cursor or 0)
query: list[tuple[str, Any]] = [
("limit", size), ("offset", offset), ("sort", "posted_at"), ("order", "desc"),
("description_format", "html")]
if text := ask.query or str(ask.params.get("q") or "").strip():
query.append(("q", text))
query += [("countries", code) for code in countries]
if mode:
query.append(("work_mode", mode))
body = get(SEARCH + "?" + urlencode(query),
headers={"User-Agent": f"roledesk/jobboard ({USER_AGENT})"})
data = json.loads(body)
hits = [hit for hit in data.get("data") or [] if isinstance(hit, dict)]
total = (data.get("meta") or {}).get("total")
more = len(hits) >= size and offset + 2 * size <= ceiling
return (data, [{"id": str(hit.get("public_slug") or "")} for hit in hits],
total if isinstance(total, int) and not isinstance(total, bool) else None,
offset + size if more else None)
def _url(url: str) -> str:
"""`url` without freehire's `utm_source` parameter, the rest verbatim."""
parts = urlsplit(url.strip())
query = "&".join(item for item in parts.query.split("&")
if item and not item.startswith("utm_source="))
return urlunsplit(parts._replace(query=query))
def _places(hit: dict[str, Any], mode: str) -> list[Place]:
"""The posting's own `location`, then each canonical city it does not
already name. A remote role's places are remote, an on-site one's offices;
a hybrid role's say nothing."""
remote = {"remote": True, "onsite": False}.get(mode)
location = str(hit.get("location") or "").strip()
found = [location] if location else []
for city in hit.get("cities") or []:
city = str(city or "").strip()
if city and not any(city.casefold() in text.casefold() for text in found):
found.append(city)
return [{"text": text, "remote": remote} for text in found]
def _pay(enrichment: dict[str, Any]) -> dict[str, Any]:
"""Pay from the model-extracted `enrichment`: yearly figures when
`salary_period` is a year, whatever the period a line that says it."""
low, high = num(enrichment.get("salary_min")), num(enrichment.get("salary_max"))
currency = enrichment.get("salary_currency")
period = str(enrichment.get("salary_period") or "").strip().lower()
out: dict[str, Any] = {}
if period and (line := pay_line(low, high, currency, period)):
out["salary_text"] = line
lo, hi, code = yearly_amount(low, high, currency, period, {"year": 1})
if code:
out |= {"salary_min": lo, "salary_max": hi, "currency": code}
return out
def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = []
for hit in decoded.get("data") or []:
title = " ".join(str(hit.get("title") or "").split())
source_id = str(hit.get("public_slug") or "").strip()
url = _url(str(hit.get("url") or ""))
if not title or not source_id or not is_web_url(url):
continue
mode = str(hit.get("work_mode") or "").strip()
places = _places(hit, mode)
countries = [str(c) for c in hit.get("countries") or [] if c]
out.append({
"source_id": source_id,
"title": title,
"company": str(hit.get("company") or "").strip(),
"url": url,
"places": places,
"location": places[0]["text"] if places else "",
"country": countries[0].upper() if len(countries) == 1 else "",
"remote": mode or None,
"posted": parse_dt(hit.get("posted_at")),
"description": html_text(hit.get("description")),
"source": str(hit.get("source") or ""),
} | _pay(hit.get("enrichment") or {}))
return out