191 lines
8.4 KiB
Python
191 lines
8.4 KiB
Python
"""freehire: an open aggregator of IT jobs crawled from about 50 hosts (company
|
|
boards, Jobnet, The Hub and others), through its public JSON API at
|
|
freehire.me/api/v1; no key.
|
|
|
|
Checked live on 2026-10-03. The search is `/agent/jobs/search`, whose hits
|
|
carry the full description (`description_format=html`), at 300 requests a
|
|
minute; `limit` is at most 100 and `offset + limit` at most 10,000, so a
|
|
search reads 10,000 results at most. `meta.total` is the count. `countries`
|
|
(lower-case ISO codes) and `work_mode` repeat in the query; several
|
|
countries are one request, so `country` is a `many` filter. `q` is noisy:
|
|
unquoted words are OR over title, company, text and place, so a search is
|
|
narrowed by `country` and a few distinctive words. A hit's `source` names
|
|
where freehire crawled it (`greenhouse`, `jobnet`, `whatjobs-dk`); it is kept
|
|
in `raw`, since a role freehire took from a board roledesk reads is a copy
|
|
of that board's role. `url` carries `utm_source=freehire.me`, which is
|
|
removed.
|
|
|
|
A hit states `cities` (canonical names, at times those of other sites of
|
|
the employer) and `location`, the posting's own words; every one of them is
|
|
a place of the role. `enrichment` is model-extracted and may state
|
|
`salary_min`, `salary_max`, `salary_currency` and `salary_period`; only a
|
|
yearly figure goes to the pay columns, the other periods are said in
|
|
`salary_text`. `enrichment` also has `seniority` and `employment_type`,
|
|
which this reader does not hand on yet: a reader that states them needs a
|
|
table of the host's words in `app/prep/hosts/freehire.py`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
from urllib.parse import urlencode, urlsplit, urlunsplit
|
|
|
|
from desk.sources import USER_AGENT
|
|
|
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
|
from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line,
|
|
yearly_amount)
|
|
|
|
NAME = "freehire"
|
|
LABEL = "freehire (tech, many countries)"
|
|
RANK = 40
|
|
|
|
READS = Reads(
|
|
scope="search", listing="paged", page_size=100, ceiling=10000,
|
|
text="listing", complete="stated",
|
|
filters=Filters(query="one", country="many", remote="one"),
|
|
places="many",
|
|
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated"),
|
|
robots="read: robots.txt allows / for every agent; its Disallow of /api/ is for "
|
|
"Googlebot and Bingbot only, and its comments invite API use with a "
|
|
"descriptive User-Agent (checked 2026-10-03)")
|
|
|
|
#: Saved param -> the filter dimension it fills.
|
|
PARAMS = {"q": "query", "country": "country", "remote": "remote"}
|
|
|
|
SEARCH = "https://freehire.me/api/v1/agent/jobs/search"
|
|
|
|
#: `work_mode` of freehire's facets, for the form: value -> label. "" is any.
|
|
MODES: dict[str, str] = {"remote": "Remote", "hybrid": "Hybrid", "onsite": "On site"}
|
|
|
|
_CODE = re.compile(r"[a-z]{2}")
|
|
|
|
|
|
def allowed(slug: str) -> Callable[[str], bool]:
|
|
return only("freehire.me")
|
|
|
|
|
|
def _codes(value: Any) -> list[str]:
|
|
"""Country codes from a comma-separated string or a list, lower-case, one
|
|
of each in the order given; a ValueError for anything but two letters."""
|
|
items = value if isinstance(value, list | tuple | set) else [value]
|
|
out: list[str] = []
|
|
for item in items:
|
|
for code in str(item or "").replace(" ", ",").lower().split(","):
|
|
if code and code not in out:
|
|
if not _CODE.fullmatch(code):
|
|
raise ValueError(f"country is empty or ISO codes like dk,se, not {code!r}")
|
|
out.append(code)
|
|
return out
|
|
|
|
|
|
def _mode(value: Any) -> str:
|
|
mode = str(value or "").strip().lower()
|
|
if mode and mode not in MODES:
|
|
raise ValueError(f"remote is empty or one of {', '.join(MODES)}")
|
|
return mode
|
|
|
|
|
|
def clean_params(params: dict[str, Any]) -> dict[str, Any]:
|
|
"""The saved params of a search: `q`, `country` as a comma-separated list
|
|
of ISO codes (empty for all countries) and `remote`, one of `MODES` or
|
|
empty for any."""
|
|
return {"q": params["q"], "country": ",".join(_codes(params.get("country"))),
|
|
"remote": _mode(params.get("remote"))}
|
|
|
|
|
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
|
"""One page of a search, newest first; the cursor is the offset. The next
|
|
page follows a full page while its end is within the ceiling. The
|
|
countries are the ask's or the saved param's (a comma-separated string
|
|
reaches `Ask.countries` as one value), the work mode the saved param's or
|
|
`remote` of the ask."""
|
|
countries = _codes(list(ask.countries) or ask.params.get("country"))
|
|
mode = _mode(ask.params.get("remote")) or ("remote" if ask.remote else "")
|
|
size, ceiling = READS.page_size or 100, READS.ceiling or 10000
|
|
offset = int(cursor or 0)
|
|
query: list[tuple[str, Any]] = [
|
|
("limit", size), ("offset", offset), ("sort", "posted_at"), ("order", "desc"),
|
|
("description_format", "html")]
|
|
if text := ask.query or str(ask.params.get("q") or "").strip():
|
|
query.append(("q", text))
|
|
query += [("countries", code) for code in countries]
|
|
if mode:
|
|
query.append(("work_mode", mode))
|
|
body = get(SEARCH + "?" + urlencode(query),
|
|
headers={"User-Agent": f"roledesk/jobboard ({USER_AGENT})"})
|
|
data = json.loads(body)
|
|
hits = [hit for hit in data.get("data") or [] if isinstance(hit, dict)]
|
|
total = (data.get("meta") or {}).get("total")
|
|
more = len(hits) >= size and offset + 2 * size <= ceiling
|
|
return (data, [{"id": str(hit.get("public_slug") or "")} for hit in hits],
|
|
total if isinstance(total, int) and not isinstance(total, bool) else None,
|
|
offset + size if more else None)
|
|
|
|
|
|
def _url(url: str) -> str:
|
|
"""`url` without freehire's `utm_source` parameter, the rest verbatim."""
|
|
parts = urlsplit(url.strip())
|
|
query = "&".join(item for item in parts.query.split("&")
|
|
if item and not item.startswith("utm_source="))
|
|
return urlunsplit(parts._replace(query=query))
|
|
|
|
|
|
def _places(hit: dict[str, Any], mode: str) -> list[Place]:
|
|
"""The posting's own `location`, then each canonical city it does not
|
|
already name. A remote role's places are remote, an on-site one's offices;
|
|
a hybrid role's say nothing."""
|
|
remote = {"remote": True, "onsite": False}.get(mode)
|
|
location = str(hit.get("location") or "").strip()
|
|
found = [location] if location else []
|
|
for city in hit.get("cities") or []:
|
|
city = str(city or "").strip()
|
|
if city and not any(city.casefold() in text.casefold() for text in found):
|
|
found.append(city)
|
|
return [{"text": text, "remote": remote} for text in found]
|
|
|
|
|
|
def _pay(enrichment: dict[str, Any]) -> dict[str, Any]:
|
|
"""Pay from the model-extracted `enrichment`: yearly figures when
|
|
`salary_period` is a year, whatever the period a line that says it."""
|
|
low, high = num(enrichment.get("salary_min")), num(enrichment.get("salary_max"))
|
|
currency = enrichment.get("salary_currency")
|
|
period = str(enrichment.get("salary_period") or "").strip().lower()
|
|
out: dict[str, Any] = {}
|
|
if period and (line := pay_line(low, high, currency, period)):
|
|
out["salary_text"] = line
|
|
lo, hi, code = yearly_amount(low, high, currency, period, {"year": 1})
|
|
if code:
|
|
out |= {"salary_min": lo, "salary_max": hi, "currency": code}
|
|
return out
|
|
|
|
|
|
def rows(decoded: Any, slug: str) -> list[Row]:
|
|
out: list[Row] = []
|
|
for hit in decoded.get("data") or []:
|
|
title = " ".join(str(hit.get("title") or "").split())
|
|
source_id = str(hit.get("public_slug") or "").strip()
|
|
url = _url(str(hit.get("url") or ""))
|
|
if not title or not source_id or not is_web_url(url):
|
|
continue
|
|
mode = str(hit.get("work_mode") or "").strip()
|
|
places = _places(hit, mode)
|
|
countries = [str(c) for c in hit.get("countries") or [] if c]
|
|
out.append({
|
|
"source_id": source_id,
|
|
"title": title,
|
|
"company": str(hit.get("company") or "").strip(),
|
|
"url": url,
|
|
"places": places,
|
|
"location": places[0]["text"] if places else "",
|
|
"country": countries[0].upper() if len(countries) == 1 else "",
|
|
"remote": mode or None,
|
|
"posted": parse_dt(hit.get("posted_at")),
|
|
"description": html_text(hit.get("description")),
|
|
"source": str(hit.get("source") or ""),
|
|
} | _pay(hit.get("enrichment") or {}))
|
|
return out
|