Add the freehire search reader: tech roles of many countries with full text, yearly pay and work mode
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
018a1ad895
commit
59e43f18b7
3 files changed
+862
No files matched your search
@@ -0,0 +1,190 @@
|
||||
"""freehire: an open aggregator of IT jobs crawled from about 50 hosts (company
|
||||
boards, Jobnet, The Hub and others), through its public JSON API at
|
||||
freehire.me/api/v1; no key.
|
||||
|
||||
Checked live on 2026-10-03. The search is `/agent/jobs/search`, whose hits
|
||||
carry the full description (`description_format=html`), at 300 requests a
|
||||
minute; `limit` is at most 100 and `offset + limit` at most 10,000, so a
|
||||
search reads 10,000 results at most. `meta.total` is the count. `countries`
|
||||
(lower-case ISO codes) and `work_mode` repeat in the query; several
|
||||
countries are one request, so `country` is a `many` filter. `q` is noisy:
|
||||
unquoted words are OR over title, company, text and place, so a search is
|
||||
narrowed by `country` and a few distinctive words. A hit's `source` names
|
||||
where freehire crawled it (`greenhouse`, `jobnet`, `whatjobs-dk`); it is kept
|
||||
in `raw`, since a role freehire took from a board roledesk reads is a copy
|
||||
of that board's role. `url` carries `utm_source=freehire.me`, which is
|
||||
removed.
|
||||
|
||||
A hit states `cities` (canonical names, at times those of other sites of
|
||||
the employer) and `location`, the posting's own words; every one of them is
|
||||
a place of the role. `enrichment` is model-extracted and may state
|
||||
`salary_min`, `salary_max`, `salary_currency` and `salary_period`; only a
|
||||
yearly figure goes to the pay columns, the other periods are said in
|
||||
`salary_text`. `enrichment` also has `seniority` and `employment_type`,
|
||||
which this reader does not hand on yet: a reader that states them needs a
|
||||
table of the host's words in `app/prep/hosts/freehire.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode, urlsplit, urlunsplit
|
||||
|
||||
from desk.sources import USER_AGENT
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line,
|
||||
yearly_amount)
|
||||
|
||||
NAME = "freehire"
|
||||
LABEL = "freehire (tech, many countries)"
|
||||
RANK = 40
|
||||
|
||||
READS = Reads(
|
||||
scope="search", listing="paged", page_size=100, ceiling=10000,
|
||||
text="listing", complete="stated",
|
||||
filters=Filters(query="one", country="many", remote="one"),
|
||||
places="many",
|
||||
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated"),
|
||||
robots="read: robots.txt allows / for every agent; its Disallow of /api/ is for "
|
||||
"Googlebot and Bingbot only, and its comments invite API use with a "
|
||||
"descriptive User-Agent (checked 2026-10-03)")
|
||||
|
||||
#: Saved param -> the filter dimension it fills.
|
||||
PARAMS = {"q": "query", "country": "country", "remote": "remote"}
|
||||
|
||||
SEARCH = "https://freehire.me/api/v1/agent/jobs/search"
|
||||
|
||||
#: `work_mode` of freehire's facets, for the form: value -> label. "" is any.
|
||||
MODES: dict[str, str] = {"remote": "Remote", "hybrid": "Hybrid", "onsite": "On site"}
|
||||
|
||||
_CODE = re.compile(r"[a-z]{2}")
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only("freehire.me")
|
||||
|
||||
|
||||
def _codes(value: Any) -> list[str]:
|
||||
"""Country codes from a comma-separated string or a list, lower-case, one
|
||||
of each in the order given; a ValueError for anything but two letters."""
|
||||
items = value if isinstance(value, list | tuple | set) else [value]
|
||||
out: list[str] = []
|
||||
for item in items:
|
||||
for code in str(item or "").replace(" ", ",").lower().split(","):
|
||||
if code and code not in out:
|
||||
if not _CODE.fullmatch(code):
|
||||
raise ValueError(f"country is empty or ISO codes like dk,se, not {code!r}")
|
||||
out.append(code)
|
||||
return out
|
||||
|
||||
|
||||
def _mode(value: Any) -> str:
|
||||
mode = str(value or "").strip().lower()
|
||||
if mode and mode not in MODES:
|
||||
raise ValueError(f"remote is empty or one of {', '.join(MODES)}")
|
||||
return mode
|
||||
|
||||
|
||||
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The saved params of a search: `q`, `country` as a comma-separated list
|
||||
of ISO codes (empty for all countries) and `remote`, one of `MODES` or
|
||||
empty for any."""
|
||||
return {"q": params["q"], "country": ",".join(_codes(params.get("country"))),
|
||||
"remote": _mode(params.get("remote"))}
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of a search, newest first; the cursor is the offset. The next
|
||||
page follows a full page while its end is within the ceiling. The
|
||||
countries are the ask's or the saved param's (a comma-separated string
|
||||
reaches `Ask.countries` as one value), the work mode the saved param's or
|
||||
`remote` of the ask."""
|
||||
countries = _codes(list(ask.countries) or ask.params.get("country"))
|
||||
mode = _mode(ask.params.get("remote")) or ("remote" if ask.remote else "")
|
||||
size, ceiling = READS.page_size or 100, READS.ceiling or 10000
|
||||
offset = int(cursor or 0)
|
||||
query: list[tuple[str, Any]] = [
|
||||
("limit", size), ("offset", offset), ("sort", "posted_at"), ("order", "desc"),
|
||||
("description_format", "html")]
|
||||
if text := ask.query or str(ask.params.get("q") or "").strip():
|
||||
query.append(("q", text))
|
||||
query += [("countries", code) for code in countries]
|
||||
if mode:
|
||||
query.append(("work_mode", mode))
|
||||
body = get(SEARCH + "?" + urlencode(query),
|
||||
headers={"User-Agent": f"roledesk/jobboard ({USER_AGENT})"})
|
||||
data = json.loads(body)
|
||||
hits = [hit for hit in data.get("data") or [] if isinstance(hit, dict)]
|
||||
total = (data.get("meta") or {}).get("total")
|
||||
more = len(hits) >= size and offset + 2 * size <= ceiling
|
||||
return (data, [{"id": str(hit.get("public_slug") or "")} for hit in hits],
|
||||
total if isinstance(total, int) and not isinstance(total, bool) else None,
|
||||
offset + size if more else None)
|
||||
|
||||
|
||||
def _url(url: str) -> str:
|
||||
"""`url` without freehire's `utm_source` parameter, the rest verbatim."""
|
||||
parts = urlsplit(url.strip())
|
||||
query = "&".join(item for item in parts.query.split("&")
|
||||
if item and not item.startswith("utm_source="))
|
||||
return urlunsplit(parts._replace(query=query))
|
||||
|
||||
|
||||
def _places(hit: dict[str, Any], mode: str) -> list[Place]:
|
||||
"""The posting's own `location`, then each canonical city it does not
|
||||
already name. A remote role's places are remote, an on-site one's offices;
|
||||
a hybrid role's say nothing."""
|
||||
remote = {"remote": True, "onsite": False}.get(mode)
|
||||
location = str(hit.get("location") or "").strip()
|
||||
found = [location] if location else []
|
||||
for city in hit.get("cities") or []:
|
||||
city = str(city or "").strip()
|
||||
if city and not any(city.casefold() in text.casefold() for text in found):
|
||||
found.append(city)
|
||||
return [{"text": text, "remote": remote} for text in found]
|
||||
|
||||
|
||||
def _pay(enrichment: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Pay from the model-extracted `enrichment`: yearly figures when
|
||||
`salary_period` is a year, whatever the period a line that says it."""
|
||||
low, high = num(enrichment.get("salary_min")), num(enrichment.get("salary_max"))
|
||||
currency = enrichment.get("salary_currency")
|
||||
period = str(enrichment.get("salary_period") or "").strip().lower()
|
||||
out: dict[str, Any] = {}
|
||||
if period and (line := pay_line(low, high, currency, period)):
|
||||
out["salary_text"] = line
|
||||
lo, hi, code = yearly_amount(low, high, currency, period, {"year": 1})
|
||||
if code:
|
||||
out |= {"salary_min": lo, "salary_max": hi, "currency": code}
|
||||
return out
|
||||
|
||||
|
||||
def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
out: list[Row] = []
|
||||
for hit in decoded.get("data") or []:
|
||||
title = " ".join(str(hit.get("title") or "").split())
|
||||
source_id = str(hit.get("public_slug") or "").strip()
|
||||
url = _url(str(hit.get("url") or ""))
|
||||
if not title or not source_id or not is_web_url(url):
|
||||
continue
|
||||
mode = str(hit.get("work_mode") or "").strip()
|
||||
places = _places(hit, mode)
|
||||
countries = [str(c) for c in hit.get("countries") or [] if c]
|
||||
out.append({
|
||||
"source_id": source_id,
|
||||
"title": title,
|
||||
"company": str(hit.get("company") or "").strip(),
|
||||
"url": url,
|
||||
"places": places,
|
||||
"location": places[0]["text"] if places else "",
|
||||
"country": countries[0].upper() if len(countries) == 1 else "",
|
||||
"remote": mode or None,
|
||||
"posted": parse_dt(hit.get("posted_at")),
|
||||
"description": html_text(hit.get("description")),
|
||||
"source": str(hit.get("source") or ""),
|
||||
} | _pay(hit.get("enrichment") or {}))
|
||||
return out
|
||||
Reference in new issue
Block a user