218 lines
10 KiB
Python
218 lines
10 KiB
Python
"""The freehire reader (public JSON API, `/agent/jobs/search`): 100 hits a
|
|
page, offsets to 10,000. No network: `get` is a stub."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from plugins import readers # noqa: E402
|
|
from plugins.readers import Ask, freehire # noqa: E402
|
|
|
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
|
|
|
|
|
|
def _rows() -> dict[str, dict[str, Any]]:
|
|
decoded, *_ = freehire.page(lambda url, **kw: BODY, "", None, Ask())
|
|
return {row["company"]: row for row in freehire.rows(decoded, "")}
|
|
|
|
|
|
def _stock(total: int | None, stock: int, asked: list[str]) -> Any:
|
|
def get(url: str, **kwargs: Any) -> bytes:
|
|
asked.append(url)
|
|
query = parse_qs(urlsplit(url).query)
|
|
start, limit = int(query["offset"][0]), int(query["limit"][0])
|
|
body: dict[str, Any] = {"data": [
|
|
{"public_slug": f"role-{n}", "title": f"Role {n}", "url": f"https://x.example/{n}"}
|
|
for n in range(start, min(start + limit, stock))], "meta": {}}
|
|
if total is not None:
|
|
body["meta"]["total"] = total
|
|
return json.dumps(body).encode()
|
|
return get
|
|
|
|
|
|
def test_it_is_a_search_reader_whose_robots_txt_allows_the_api() -> None:
|
|
assert readers.PROVIDERS["freehire"] is freehire
|
|
reads = freehire.READS
|
|
assert (reads.scope, reads.page_size, reads.ceiling, reads.places) == (
|
|
"search", 100, 10000, "many")
|
|
assert reads.filters.declared() == {"query": "one", "country": "many", "remote": "one"}
|
|
assert "Googlebot" in reads.robots and "2026-10-03" in reads.robots
|
|
assert freehire.allowed("")("freehire.me") and not freehire.allowed("")("example.com")
|
|
|
|
|
|
def test_it_reads_the_full_text_company_date_and_source() -> None:
|
|
rows = _rows()
|
|
first = rows["Zeta Global"]
|
|
assert first["title"] == "Data Engineer"
|
|
assert first["posted"].isoformat().startswith("2026-")
|
|
assert first["description"] and "<" not in first["description"]
|
|
assert first["source"] == "greenhouse" and "teaser" not in first
|
|
assert rows["DTU Compute - Bygning 322"]["source"] == "jobnet"
|
|
|
|
|
|
def test_it_removes_utm_source_and_nothing_else_from_the_link() -> None:
|
|
assert _rows()["Zeta Global"]["url"].startswith("https://") and "utm_source" not in _rows()["Zeta Global"]["url"]
|
|
assert freehire._url("https://x.example/j?id=7&utm_source=freehire.me&ref=a") == (
|
|
"https://x.example/j?id=7&ref=a")
|
|
assert freehire._url("https://x.example/j?utm_source=freehire.me") == "https://x.example/j"
|
|
assert freehire._url("https://x.example/j/") == "https://x.example/j/"
|
|
|
|
|
|
def test_it_keeps_the_location_and_every_city_it_does_not_name() -> None:
|
|
rows = _rows()
|
|
assert [p["text"] for p in rows["Zeta Global"]["places"]] == [
|
|
"Copenhagen, Denmark", "Berlin", "London"] # Copenhagen is named already
|
|
assert rows["Zeta Global"]["location"] == "Copenhagen, Denmark"
|
|
assert [p["text"] for p in rows["DataDome"]["places"]] == ["France - Remote", "Paris"]
|
|
assert [p["text"] for p in rows["jobgether"]["places"]] == ["India", "Luxembourg", "Mexico City"]
|
|
assert [p["text"] for p in rows["DTU Compute - Bygning 322"]["places"]] == ["Kongens Lyngby, Danmark"]
|
|
|
|
|
|
def test_a_role_without_a_location_has_its_cities() -> None:
|
|
novo = _rows()["Novo Nordisk"]
|
|
assert [p["text"] for p in novo["places"]] == ["Måløv"]
|
|
|
|
|
|
def test_work_mode_is_the_remote_word_and_marks_the_places() -> None:
|
|
rows = _rows()
|
|
assert rows["jobgether"]["remote"] == "remote"
|
|
assert all(p["remote"] is True for p in rows["jobgether"]["places"])
|
|
assert rows["Zeta Global"]["remote"] is None
|
|
assert all(p["remote"] is None for p in rows["Zeta Global"]["places"])
|
|
hit = {"location": "Aarhus", "cities": []}
|
|
assert freehire._places(hit, "onsite") == [{"text": "Aarhus", "remote": False}]
|
|
assert freehire._places(hit, "hybrid") == [{"text": "Aarhus", "remote": None}]
|
|
|
|
|
|
def test_country_is_stated_only_for_a_role_in_exactly_one() -> None:
|
|
rows = _rows()
|
|
assert rows["DTU Compute - Bygning 322"]["country"] == "DK"
|
|
assert rows["Zeta Global"]["country"] == "" and rows["jobgether"]["country"] == "" # 3 and 36
|
|
|
|
|
|
def test_a_yearly_salary_goes_to_the_columns_and_says_its_period() -> None:
|
|
novo = _rows()["Novo Nordisk"]
|
|
assert (novo["salary_min"], novo["salary_max"], novo["currency"]) == (651000, 956900, "DKK")
|
|
assert novo["salary_text"] == "DKK 651,000 to 956,900 per year"
|
|
assert "salary_min" not in _rows()["Zeta Global"]
|
|
|
|
|
|
def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None:
|
|
month = {"salary_min": 40000, "salary_max": 50000, "salary_currency": "dkk",
|
|
"salary_period": "month"}
|
|
assert freehire._pay(month) == {"salary_text": "DKK 40,000 to 50,000 per month"}
|
|
assert freehire._pay({"salary_min": 40000, "salary_currency": "EUR"}) == {} # no period
|
|
assert freehire._pay({"salary_period": "year"}) == {}
|
|
assert freehire._pay({"salary_min": 0, "salary_max": 0, "salary_period": "year",
|
|
"salary_currency": "EUR"}) == {}
|
|
year = freehire._pay({"salary_min": 60000, "salary_currency": "eur", "salary_period": "year"})
|
|
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
|
|
|
|
|
|
def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None:
|
|
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
|
|
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
|
|
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
|
|
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
|
|
found = freehire.rows({"data": hits}, "")
|
|
assert [row["source_id"] for row in found] == ["d"]
|
|
assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == []
|
|
|
|
|
|
def test_it_asks_for_countries_work_mode_and_the_text_in_html() -> None:
|
|
asked: list[str] = []
|
|
freehire.page(_stock(250, 250, asked), "", None,
|
|
Ask(params={"q": "head of data", "country": "dk,SE", "remote": "hybrid"}))
|
|
url = urlsplit(asked[0])
|
|
assert (url.scheme, url.netloc, url.path) == ("https", "freehire.me", "/api/v1/agent/jobs/search")
|
|
assert parse_qs(url.query) == {
|
|
"q": ["head of data"], "limit": ["100"], "offset": ["0"], "sort": ["posted_at"],
|
|
"order": ["desc"], "description_format": ["html"], "countries": ["dk", "se"],
|
|
"work_mode": ["hybrid"]}
|
|
assert asked[0].count("countries=") == 2
|
|
|
|
|
|
def test_it_asks_nothing_it_was_not_given() -> None:
|
|
asked: list[str] = []
|
|
freehire.page(_stock(None, 5, asked), "", None, Ask(params={"q": "x", "country": "",
|
|
"remote": ""}))
|
|
freehire.page(_stock(None, 5, asked), "", 100, Ask())
|
|
assert "countries" not in asked[0] and "work_mode" not in asked[0]
|
|
assert "q=" not in asked[1] and "offset=100" in asked[1]
|
|
|
|
|
|
def test_the_ask_wins_over_the_saved_params_where_it_is_set() -> None:
|
|
asked: list[str] = []
|
|
freehire.page(_stock(None, 5, asked), "", None,
|
|
Ask(countries=("DK", "NO"), query="data", remote=True,
|
|
params={"q": "other", "country": "se"}))
|
|
query = parse_qs(urlsplit(asked[0]).query)
|
|
assert (query["q"], query["countries"], query["work_mode"]) == (
|
|
["data"], ["dk", "no"], ["remote"])
|
|
freehire.page(_stock(None, 5, asked), "", None, Ask(countries=("DK,SE",))) # one value
|
|
assert parse_qs(urlsplit(asked[1]).query)["countries"] == ["dk", "se"]
|
|
|
|
|
|
def test_it_names_itself_in_the_user_agent() -> None:
|
|
seen: list[dict[str, str]] = []
|
|
|
|
def get(url: str, **kwargs: Any) -> bytes:
|
|
seen.append(kwargs["headers"])
|
|
return b'{"data": [], "meta": {"total": 0}}'
|
|
|
|
freehire.page(get, "", None, Ask())
|
|
assert seen[0]["User-Agent"].startswith("roledesk/jobboard (")
|
|
|
|
|
|
def test_the_total_is_the_metas_and_a_missing_one_is_none() -> None:
|
|
_decoded, found, total, _cursor = freehire.page(lambda url, **kw: BODY, "", None, Ask())
|
|
assert total == 1391 and [f["id"] for f in found] == [
|
|
hit["public_slug"] for hit in json.loads(BODY)["data"]]
|
|
assert freehire.page(_stock(None, 5, []), "", None, Ask())[2] is None
|
|
|
|
|
|
@pytest.mark.parametrize(("cursor", "stock", "after"), [
|
|
(None, 250, 100), (200, 250, None), # a short page is the end
|
|
(9800, 20000, 9900), (9900, 20000, None), # offset + limit stays within 10,000
|
|
])
|
|
def test_it_pages_by_offset_up_to_the_ceiling(
|
|
cursor: int | None, stock: int, after: int | None) -> None:
|
|
asked: list[str] = []
|
|
_decoded, _found, _total, nxt = freehire.page(
|
|
_stock(None, stock, asked), "", cursor, Ask(params={"q": "x"}))
|
|
assert nxt == after
|
|
query = parse_qs(urlsplit(asked[0]).query)
|
|
assert int(query["offset"][0]) + int(query["limit"][0]) <= 10000
|
|
|
|
|
|
def test_it_refuses_a_country_or_a_work_mode_it_cannot_send() -> None:
|
|
for params in ({"country": "denmark"}, {"remote": "anywhere"}):
|
|
with pytest.raises(ValueError):
|
|
freehire.page(lambda url, **kw: b"{}", "", None, Ask(params=params))
|
|
|
|
|
|
def test_the_modes_are_freehires_work_modes() -> None:
|
|
assert list(freehire.MODES) == ["remote", "hybrid", "onsite"]
|
|
assert all(label for label in freehire.MODES.values())
|
|
|
|
|
|
def test_saved_params_keep_codes_in_lower_case_and_a_known_mode() -> None:
|
|
assert freehire.clean_params({"q": "x", "country": " DK, se ,dk", "remote": "Hybrid"}) == {
|
|
"q": "x", "country": "dk,se", "remote": "hybrid"}
|
|
assert freehire.clean_params({"q": "x", "country": ["NO", "fi"]})["country"] == "no,fi"
|
|
assert freehire.clean_params({"q": "x"}) == {"q": "x", "country": "", "remote": ""}
|
|
with pytest.raises(ValueError, match="country"):
|
|
freehire.clean_params({"q": "x", "country": "denmark"})
|
|
with pytest.raises(ValueError, match="remote"):
|
|
freehire.clean_params({"q": "x", "remote": "anywhere"})
|