diff --git a/plugins/readers/ashby.py b/plugins/readers/ashby.py index 0513477..0e7b12c 100644 --- a/plugins/readers/ashby.py +++ b/plugins/readers/ashby.py @@ -8,12 +8,17 @@ The field map is ported from `openings.go` in from __future__ import annotations import json +import logging +import re from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount +from ._contract import Ask, Fields, Place, Reads, Row +from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount + +log = logging.getLogger(__name__) NAME = "ashby" LABEL = "Ashby" @@ -23,8 +28,23 @@ PATTERNS = ( r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)", ) +READS = Reads( + scope="board", listing="whole", text="listing", complete="whole", + places="many", empty="suspect", + fields=Fields(pay="figures", posted="trusted", remote="stated", employment="stated"), + robots="api: no robots.txt check needed (public board API), 2026-09-30", +) + _API = "api.ashbyhq.com" +#: `workplaceType` -> `store._remote`'s words. `isRemote` is true for hybrid +#: roles too, so it is only the fallback when the role has no `workplaceType`. +_WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"} + +#: `employmentType` -> the app's employment words. +_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "intern", + "contract": "contract", "temporary": "temporary"} + def slug_ok(slug: str) -> bool: return path_slug_ok(slug) @@ -38,7 +58,7 @@ def careers_url(slug: str) -> str: return f"https://jobs.ashbyhq.com/{slug}" -def feed_url(slug: str) -> str: +def feed_url(slug: str, ask: Ask) -> str: return f"https://{_API}/posting-api/job-board/{slug}?includeCompensation=true" @@ -94,6 +114,32 @@ def _salary_text(compensation: dict[str, Any]) -> str: or compensation.get("scrapeableCompensationSalarySummary") or "") +def _places(j: dict[str, Any], workplace: str | None) -> list[Place]: + """The primary `location`, then each of `secondaryLocations`, in the host's + words. The primary is remote or an office as `workplaceType` says; a + place that names itself remote is remote.""" + primary = (j.get("location") or "").strip() + named = [(primary, {"remote": True, "hybrid": False, "onsite": False}.get(workplace or ""))] + named += [(str((s or {}).get("location") or "").strip(), None) + for s in j.get("secondaryLocations") or []] + out: list[Place] = [] + for text, remote in named: + if text and text not in (p["text"] for p in out): + out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) + else remote}) + return out + + +def _employment(value: Any) -> str | None: + word = str(value or "").strip() + if not word: + return None + found = _EMPLOYMENT.get(word.lower()) + if found is None: + log.debug("ashby: unknown employmentType %r", word) + return found + + def parse(body: bytes, slug: str) -> list[Row]: data = json.loads(body) rows = [] @@ -105,18 +151,24 @@ def parse(body: bytes, slug: str) -> list[Row]: description = j.get("descriptionHtml") or j.get("descriptionPlain") or "" compensation = j.get("compensation") or {} salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or []) - rows.append({ + workplace = _WORKPLACE.get(str(j.get("workplaceType") or "").strip().lower()) + places = _places(j, workplace) + row: Row = { "source_id": str(j.get("id") or ""), "title": title, - "location": (j.get("location") or "").strip(), + "places": places, + "location": "; ".join(p["text"] for p in places), "url": url, "posted": parse_dt(j.get("publishedAt")), "department": (j.get("department") or j.get("team") or "").strip(), "description": strip_html(description), - "remote": j.get("isRemote"), + "remote": workplace or j.get("isRemote"), "salary_text": _salary_text(compensation), "salary_min": salary_min, "salary_max": salary_max, "currency": currency, - }) + } + if employment := _employment(j.get("employmentType")): + row["employment"] = employment + rows.append(row) return rows diff --git a/plugins/readers/bamboohr.py b/plugins/readers/bamboohr.py index 0a558a2..ab3fa3a 100644 --- a/plugins/readers/bamboohr.py +++ b/plugins/readers/bamboohr.py @@ -16,12 +16,31 @@ from __future__ import annotations import html import json +import logging +import re from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Get, Row, label_slug_ok, only, parse_dt +from ._contract import Ask, Fields, Place, Reads, Row +from ._shared import Get, label_slug_ok, only, parse_dt + +log = logging.getLogger(__name__) + +READS = Reads( + scope="board", listing="whole", text="detail", complete="whole", + places="one", + fields=Fields(posted="trusted", remote="stated", employment="stated"), + robots="read: robots.txt allows /careers (checked 2026-09-29)", +) + +#: `employmentStatusLabel` -> the app's employment words. BambooHR's labels are +#: the company's own ("Full-Time", "Part-Time", "Contract", "Intern"). +_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", + "part time": "part_time", "contract": "contract", "contractor": "contract", + "temporary": "temporary", "intern": "intern", "internship": "intern", + "freelance": "freelance"} NAME = "bamboohr" LABEL = "BambooHR" @@ -54,11 +73,7 @@ def careers_url(slug: str) -> str: return f"https://{host(slug)}/careers" -def robots_page(slug: str) -> str: - return careers_url(slug) - - -def feed_url(slug: str) -> str: +def feed_url(slug: str, ask: Ask) -> str: return f"https://{host(slug)}/careers/list" @@ -79,6 +94,23 @@ def _place(j: dict[str, Any]) -> str: return ", ".join(str(p).strip() for p in parts if p and str(p).strip()) +def _places(j: dict[str, Any]) -> list[Place]: + """The one place BambooHR states, if it states any.""" + text = _place(j) + return [{"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}] \ + if text else [] + + +def _employment(j: dict[str, Any]) -> str | None: + label = str(j.get("employmentStatusLabel") or "").strip() + if not label: + return None + found = _EMPLOYMENT.get(label.lower()) + if found is None: + log.debug("bamboohr: unknown employmentStatusLabel %r", label) + return found + + def parse(body: bytes, slug: str) -> list[Row]: """`/careers/list` -> `{"meta": {"totalCount": n}, "result": [...]}`. A role has `id`, `jobOpeningName`, `departmentLabel`, `location` (`city`, @@ -99,9 +131,11 @@ def parse(body: bytes, slug: str) -> list[Row]: source_id = str(j.get("id") or "") if not title or not source_id: continue - out.append({ + places = _places(j) + row: Row = { "source_id": source_id, "title": title, + "places": places, "location": _place(j), "url": f"https://{host(slug)}/careers/{source_id}", "posted": None, @@ -109,7 +143,10 @@ def parse(body: bytes, slug: str) -> list[Row]: "description": "", "remote": _remote(j), "salary_text": "", # the list states no pay; the detail's `compensation` was null - }) + } + if employment := _employment(j): + row["employment"] = employment + out.append(row) return out @@ -130,6 +167,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: } if place := _place(opening): out["location"] = place + out["places"] = _places(opening) if remote := _remote(opening): out["remote"] = remote return out diff --git a/plugins/readers/deel.py b/plugins/readers/deel.py index f0b6291..521276e 100644 --- a/plugins/readers/deel.py +++ b/plugins/readers/deel.py @@ -16,13 +16,25 @@ from __future__ import annotations import html import json +import logging import re from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Row, num, only, parse_dt, pay_line +from ._contract import Ask, Fields, Reads, Row +from ._shared import num, only, parse_dt, pay_line + +log = logging.getLogger(__name__) + +READS = Reads( + scope="board", listing="whole", text="listing", complete="whole", + places="many", + fields=Fields(pay="line", posted="listed", remote="stated", employment="stated"), + robots="read: robots.txt allows the board page /; /api/ is disallowed and not read " + "(checked 2026-09-29)", +) NAME = "deel" LABEL = "Deel" @@ -50,6 +62,12 @@ _TEXT = re.compile(rb"T([0-9a-f]+),") #: Deel's `workArrangementEnum` -> `store._remote`'s words. _REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"} +#: The name of a `jobEmploymentTypes` entry -> the app's employment words. +_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", + "part time": "part_time", "contract": "contract", "contractor": "contract", + "consultant": "contract", "temporary": "temporary", "intern": "intern", + "internship": "intern", "freelance": "freelance"} + def slug_ok(slug: str) -> bool: return bool(_SLUG.fullmatch(slug)) @@ -63,11 +81,7 @@ def careers_url(slug: str) -> str: return f"https://{_HOST}/{slug}" -def robots_page(slug: str) -> str: - return careers_url(slug) - - -def feed_url(slug: str) -> str: +def feed_url(slug: str, ask: Ask) -> str: return careers_url(slug) @@ -106,9 +120,21 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]: raise ValueError("not a Deel job board: no jobPostings in the page") +def _employment(job: dict[str, Any]) -> str | None: + """The first `jobEmploymentTypes` entry that names a known type.""" + for entry in job.get("jobEmploymentTypes") or []: + name = str(((entry or {}).get("employmentType") or {}).get("name") or "").strip() + if not name: + continue + if found := _EMPLOYMENT.get(name.lower()): + return found + log.debug("deel: unknown employment type %r", name) + return None + + def parse(body: bytes, slug: str) -> list[Row]: """`jobs.deel.com/` -> one row per posting. `location` joins the - role's places with "; ". `posted` is the day the posting was published + role's places with "; ", and `places` lists each. `posted` is the day the posting was published (its publication's state), not the day the role was drafted. Deel states pay as a currency and two amounts with no period, and the @@ -132,9 +158,12 @@ def parse(body: bytes, slug: str) -> list[Row]: description = p.get("richtextDescription") or "" if description.startswith("$"): description = stream.get(description[1:], "") - out.append({ + arrangement = str(job.get("workArrangementEnum") or "") + remote = {"REMOTE": True, "HYBRID": False, "ON_SITE": False}.get(arrangement) + row: Row = { "source_id": post_id, "title": title, + "places": [{"text": name, "remote": remote} for name in dict.fromkeys(places)], "location": "; ".join(dict.fromkeys(places)), "url": f"https://{_HOST}/{slug}/job-details/{post_id}/overview", "posted": parse_dt(published.get("createdAt") or p.get("createdAt")), @@ -144,5 +173,8 @@ def parse(body: bytes, slug: str) -> list[Row]: "salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")), pay.get("currencyIsoCode"), "") if p.get("isCompensationVisible") else ""), - }) + } + if employment := _employment(job): + row["employment"] = employment + out.append(row) return out diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py index de23555..fe2364f 100644 --- a/plugins/readers/eightfold.py +++ b/plugins/readers/eightfold.py @@ -38,7 +38,15 @@ from typing import Any from desk.sources import strip_html -from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._contract import Ask, Fields, Place, Reads, Row +from ._shared import Get, Page, is_web_url, only, parse_dt + +READS = Reads( + scope="board", listing="paged", page_size=10, text="detail", complete="stated", + places="many", + fields=Fields(posted="listed", remote="stated"), + robots="read: robots.txt allows /careers (checked 2026-09-29)", +) NAME = "eightfold" LABEL = "Eightfold" @@ -85,11 +93,7 @@ def careers_url(slug: str) -> str: return f"https://{parts(slug)[0]}/careers" -def robots_page(slug: str) -> str: - return careers_url(slug) - - -def page(get: Get, slug: str, cursor: Any) -> Page: +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """One page of `/api/pcsx/search` -> `{"status": 200, "data": {"positions": [...], "count": n}}`; the next starts at the offset after this page's positions, and an empty page ends the read.""" @@ -108,25 +112,27 @@ def page(get: Get, slug: str, cursor: Any) -> Page: start + len(positions) if positions else None) -def _place(locations: Any) -> str: - """The role's places, "; " apart. Eightfold writes each as "city,region, - country" with no space and repeats the name when a city is its own region - ("Mexico City,Mexico City,Mexico"), so the repeats go.""" - out: list[str] = [] +def _places(locations: Any) -> list[Place]: + """The role's places. Eightfold writes each as "city,region,country" with + no space and repeats the name when a city is its own region ("Mexico + City,Mexico City,Mexico"), so the repeats go.""" + out: list[Place] = [] for loc in locations if isinstance(locations, list) else []: names: list[str] = [] for name in str(loc).split(","): name = name.strip() if name and name not in names: names.append(name) - if names and ", ".join(names) not in out: - out.append(", ".join(names)) - return "; ".join(out) + text = ", ".join(names) + if text and text not in (p["text"] for p in out): + out.append({"text": text, "remote": True if "remote" in text.lower() else None}) + return out def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]: """A position has `id` (a number, kept as a string), `displayJobId`, - `name`, `locations`, `postedTs` (epoch seconds), `department`, + `name`, `locations`, `creationTs` (epoch seconds; `postedTs` is + refreshed in bulk), `department`, `workLocationOption` and `positionUrl`, the path of the role's page on the tenant's host. The list has no text, and no pay is stated anywhere.""" host = parts(slug)[0] @@ -138,12 +144,16 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]: url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else "" if not title or not source_id or not is_web_url(url): continue + places = _places(p.get("locations") or p.get("standardizedLocations")) out.append({ "source_id": source_id, "title": title, - "location": _place(p.get("locations") or p.get("standardizedLocations")), + "places": places, + "location": "; ".join(pl["text"] for pl in places), "url": url, - "posted": parse_dt(p.get("postedTs")), + # `postedTs` is refreshed in bulk (a whole page says today); + # `creationTs` is the posting date. + "posted": parse_dt(p.get("creationTs") or p.get("postedTs")), "department": str(p.get("department") or "").strip(), "description": "", "remote": _REMOTE.get(str(p.get("workLocationOption") or "").strip().lower()), diff --git a/plugins/readers/gem.py b/plugins/readers/gem.py index e57db05..1d225df 100644 --- a/plugins/readers/gem.py +++ b/plugins/readers/gem.py @@ -19,13 +19,24 @@ from __future__ import annotations import html import json +import logging import re from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Get, Page, Row, only, parse_dt +from ._contract import Ask, Fields, Place, Reads, Row +from ._shared import Get, Page, only, parse_dt + +log = logging.getLogger(__name__) + +READS = Reads( + scope="board", listing="paged", text="detail", complete="end", + places="many", empty="suspect", + fields=Fields(posted="listed", remote="stated", employment="stated"), + robots="read: robots.txt allows / (checked 2026-09-29)", +) NAME = "gem" LABEL = "Gem" @@ -57,6 +68,11 @@ DETAIL_QUERY = ( #: `job.locationType` -> `store._remote`'s words. _REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"} +#: `job.employmentType` -> the app's employment words. +_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACT": "contract", + "CONTRACTOR": "contract", "INTERN": "intern", "INTERNSHIP": "intern", + "TEMPORARY": "temporary", "FREELANCE": "freelance"} + def slug_ok(slug: str) -> bool: return bool(_BOARD.fullmatch(slug)) @@ -70,10 +86,6 @@ def careers_url(slug: str) -> str: return f"https://{_HOST}/{slug}" -def robots_page(slug: str) -> str: - return careers_url(slug) - - def _ask(get: Get, operation: str, query: str, variables: dict[str, str], path: str) -> Any: """One GraphQL call, and the part of its answer at `data.`. An @@ -93,7 +105,7 @@ def _ask(get: Get, operation: str, query: str, variables: dict[str, str], return data[path] -def page(get: Get, slug: str, cursor: Any) -> Page: +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """The one page there is: `data.oatsExternalJobPostings.jobPostings`.""" board = _ask(get, "JobBoardList", LIST_QUERY, {"boardId": slug}, "oatsExternalJobPostings") postings = board.get("jobPostings") if isinstance(board, dict) else None @@ -116,26 +128,34 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]: if not title or not ext_id: continue locations = p.get("locations") or [] - places: list[str] = [] + places: list[Place] = [] for loc in locations: - place = str(loc.get("city") or loc.get("name") or "").strip() - if place and place not in places: - places.append(place) + text = str(loc.get("city") or loc.get("name") or "").strip() + if text and text not in (pl["text"] for pl in places): + places.append({"text": text, "remote": loc["isRemote"] + if isinstance(loc.get("isRemote"), bool) else None}) job = p.get("job") or {} remote = _REMOTE.get(str(job.get("locationType") or "").upper()) if remote is None and locations and all(loc.get("isRemote") for loc in locations): remote = "remote" - out.append({ + row: Row = { "source_id": ext_id, "title": title, - "location": "; ".join(places), + "places": places, + "location": "; ".join(pl["text"] for pl in places), "url": f"{careers_url(slug)}/{ext_id}", "posted": None, # only the role's own query has a date "department": str((job.get("department") or {}).get("name") or "").strip(), "description": "", "remote": remote, "salary_text": "", - }) + } + if word := str(job.get("employmentType") or "").strip().upper(): + if employment := _EMPLOYMENT.get(word): + row["employment"] = employment + else: + log.debug("gem: unknown employmentType %r", word) + out.append(row) return out diff --git a/plugins/readers/greenhouse.py b/plugins/readers/greenhouse.py index ce0ceb6..3753026 100644 --- a/plugins/readers/greenhouse.py +++ b/plugins/readers/greenhouse.py @@ -8,12 +8,14 @@ from __future__ import annotations import html import json +import re from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line +from ._contract import Ask, Fields, Place, Reads, Row +from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line NAME = "greenhouse" LABEL = "Greenhouse" @@ -23,8 +25,22 @@ PATTERNS = ( r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)", ) +READS = Reads( + scope="board", listing="whole", text="listing", complete="whole", + places="many", + fields=Fields(pay="figures", posted="trusted"), + robots="api: no robots.txt check needed (public board API), 2026-09-30", +) + _API = "boards-api.greenhouse.io" +#: What a board writes as `location.name` when it means a way of working, not a +#: place (Cloudflare: "Hybrid", "In-Office", "Distributed; Hybrid"). The real +#: places are then in `offices`. +_PLACEHOLDER = re.compile( + r"(?:(?:hybrid|in[- ]?office|on[- ]?site|remote|distributed|flexible|office)" + r"(?:\s+or\s+|\s*[;/,&]\s*|\s+)?)+", re.I) + def slug_ok(slug: str) -> bool: return path_slug_ok(slug) @@ -38,7 +54,7 @@ def careers_url(slug: str) -> str: return f"https://job-boards.greenhouse.io/{slug}" -def feed_url(slug: str) -> str: +def feed_url(slug: str, ask: Ask) -> str: return f"https://{_API}/v1/boards/{slug}/jobs?content=true&pay_transparency=true" @@ -107,6 +123,30 @@ def _salary_text(ranges: list[dict[str, Any]]) -> str: return "; ".join(lines) +def _places(j: dict[str, Any]) -> list[Place]: + """The role's places: each of `offices` (present with `content=true`), in + the host's words, or, when there is none, `location.name` as stated. A + `location.name` that only names a way of working ("Hybrid") is not a + place: the offices replace it.""" + name = ((j.get("location") or {}).get("name") or "").strip() + out: list[Place] = [] + for office in j.get("offices") or []: + text = str((office or {}).get("location") or (office or {}).get("name") or "").strip() + if text and text not in (p["text"] for p in out): + out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) + else None}) + if not out or not name or _PLACEHOLDER.fullmatch(name): + return out or ([{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) + else None}] if name else []) + # A name that says something the offices do not (a "Remote India" beside + # an office in Bangalore) is one more place; one they already contain, + # or that contains them, is the same place said twice. + low = name.lower() + if any(p["text"].lower() in low or low in p["text"].lower() for p in out): + return out + return [{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) else None}, *out] + + def parse(body: bytes, slug: str) -> list[Row]: data = json.loads(body) rows = [] @@ -122,10 +162,12 @@ def parse(body: bytes, slug: str) -> list[Row]: content = html.unescape(j.get("content") or "") ranges = j.get("pay_input_ranges") or [] salary_min, salary_max, currency = _salary(ranges) + places = _places(j) rows.append({ "source_id": str(j.get("id") or ""), "title": title, - "location": ((j.get("location") or {}).get("name") or "").strip(), + "places": places, + "location": "; ".join(p["text"] for p in places), "url": url, "posted": parse_dt(j.get("first_published") or j.get("updated_at")), "department": (departments[0].get("name") if departments else "") or "", diff --git a/plugins/readers/hibob.py b/plugins/readers/hibob.py index 4117d75..fa5349a 100644 --- a/plugins/readers/hibob.py +++ b/plugins/readers/hibob.py @@ -16,12 +16,35 @@ from __future__ import annotations import html import json +import logging from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Get, Page, Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount +from ._contract import Ask, Fields, Reads, Row +from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount + +log = logging.getLogger(__name__) + +# One ad names one site: a company that hires at two sites posts two ads with +# their own ids (Dixa's "Expression of interest" at Copenhagen and London), so +# the host publishes one posting per place. +READS = Reads( + scope="board", listing="paged", text="listing", complete="end", + places="copies", + fields=Fields(pay="figures", posted="listed", remote="stated", employment="stated"), + robots="read: robots.txt allows the careers site; its robots.txt is the single-page " + "app's index page with no rules (checked 2026-09-29)", +) + +#: `employmentType`, the company's own name for it -> the app's employment +#: words. "Permanent" says nothing about hours and stays unmapped. +_EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time", + "full-time": "full_time", "regular part time": "part_time", + "part time": "part_time", "part-time": "part_time", "contract": "contract", + "contractor": "contract", "temporary": "temporary", "fixed term": "temporary", + "intern": "intern", "internship": "intern", "freelance": "freelance"} NAME = "hibob" LABEL = "HiBob" @@ -55,11 +78,7 @@ def careers_url(slug: str) -> str: return f"https://{_host(slug)}/" -def robots_page(slug: str) -> str: - return careers_url(slug) - - -def page(get: Get, slug: str, cursor: Any) -> Page: +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """The one page: every ad, and no total or cursor, so the board is complete when the answer parses. The host's robots.txt is its single-page app's index page (a 200 with no rules), which allows.""" @@ -96,9 +115,12 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: period = _PERIODS.get(str(ad.get("payTransparencySalaryPayPeriod") or "").strip().lower(), "") salary_min, salary_max, code = yearly_amount(lo, hi, currency, period, {"year": 1, "month": 12}) - out.append({ + remote = {"remote": True, "hybrid": False, "on_site": False, "onsite": False}.get( + str(ad.get("workspaceTypeId") or "").strip().lower()) + row: Row = { "source_id": ad_id, "title": title, + "places": [{"text": location, "remote": remote}] if location else [], "location": location, "url": f"https://{_host(slug)}/jobs/{ad_id}", "posted": parse_dt(ad.get("publishedAt")), @@ -109,5 +131,12 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: "salary_min": salary_min, "salary_max": salary_max, "currency": code, - }) + } + kind = str(ad.get("employmentType") or "").strip() + if kind: + if employment := _EMPLOYMENT.get(kind.lower()): + row["employment"] = employment + else: + log.debug("hibob: unknown employmentType %r", kind) + out.append(row) return out diff --git a/plugins/readers/hrmanager.py b/plugins/readers/hrmanager.py index 6bde149..5bf54a8 100644 --- a/plugins/readers/hrmanager.py +++ b/plugins/readers/hrmanager.py @@ -10,12 +10,13 @@ without `take` it answers the first 25 roles whatever the board holds, and `flyingtiger` does). robots.txt: the portal's own pages on candidate.hr-manager.net disallow -everything for `*` except the application form, and so `robots_page` is the -feed's URL, on the API host, which has no robots.txt (a 404, which allows). -The feed is the interface HR Manager publishes for readers like this one; the -portal's HTML list, which the robots.txt does shut out, is never read. The -role's link is the application form the robots.txt names as allowed, and is -only shown, not fetched. +everything for `*` except the application form. The feed is on the API host, +which has no robots.txt (a 404, which allows); it is the interface HR Manager +publishes for readers like this one, and the portal's HTML list, which the +robots.txt does shut out, is never read. The role's link is the application +form the robots.txt names as allowed, and is only shown, not fetched. The +declaration says "disallowed" all the same, and the owner decides whether the +feed is read. """ from __future__ import annotations @@ -28,7 +29,15 @@ from typing import Any from desk.sources import strip_html -from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._contract import Ask, Fields, Reads, Row +from ._shared import Get, Page, is_web_url, only, parse_dt + +READS = Reads( + scope="board", listing="paged", page_size=100, text="listing", complete="end", + places="one", + fields=Fields(posted="listed"), + robots="disallowed by robots.txt; owner to decide", +) NAME = "hrmanager" LABEL = "HR Manager" @@ -64,11 +73,7 @@ def _feed(slug: str, skip: int) -> str: f"?incads=true&skip={skip}&take={PAGE_SIZE}") -def robots_page(slug: str) -> str: - return _feed(slug, 0) - - -def page(get: Get, slug: str, cursor: Any) -> Page: +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: skip = int(cursor or 0) items = _items(get(_feed(slug, skip), headers={"Accept": "application/rss+xml"})) results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items] @@ -103,6 +108,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]: out.append({ "source_id": (item.findtext("guid") or url).strip(), "title": title, + "places": [], "location": "", "url": url, "posted": parse_dt(item.findtext("pubDate")), diff --git a/plugins/readers/jibe.py b/plugins/readers/jibe.py index b285e6a..9314675 100644 --- a/plugins/readers/jibe.py +++ b/plugins/readers/jibe.py @@ -30,13 +30,29 @@ from __future__ import annotations import html import json +import logging import re from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Get, Page, Row, only, parse_dt +from ._contract import Ask, Fields, Place, Reads, Row +from ._shared import Get, Page, only, parse_dt + +log = logging.getLogger(__name__) + +READS = Reads( + scope="board", listing="paged", page_size=100, text="listing", complete="stated", + places="many", + fields=Fields(posted="listed", country="stated", employment="stated"), + robots="read: robots.txt allows /jobs, with crawl-delay 5 not honoured (checked 2026-09-29)", +) + +#: `employment_type` -> the app's employment words. +_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACTOR": "contract", + "CONTRACT": "contract", "TEMPORARY": "temporary", "INTERN": "intern", + "INTERNSHIP": "intern", "FREELANCE": "freelance"} NAME = "jibe" LABEL = "Jibe" @@ -73,11 +89,7 @@ def careers_url(slug: str) -> str: return f"https://{slug.lower()}/jobs" -def robots_page(slug: str) -> str: - return careers_url(slug) - - -def page(get: Get, slug: str, cursor: Any) -> Page: +def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """One page of `/api/jobs`; the next is the following number, and an empty page ends the read.""" number = int(cursor or 1) @@ -99,6 +111,21 @@ def _plain(markup: Any) -> str: return " ".join(html.unescape(strip_html(str(markup or ""))).split()) +def _places(j: dict[str, Any]) -> list[Place]: + """`full_location` names every place of the role, "; " apart, each as + "City, Country"; the site's `city` and `country` stand in when it is + missing.""" + line = str(j.get("full_location") or "").strip() \ + or ", ".join(p for p in (j.get("city"), j.get("country")) if p) + out: list[Place] = [] + for part in line.split(";"): + text = part.strip() + if text and text not in (p["text"] for p in out): + out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) + else None}) + return out + + def rows(data: dict[str, Any], slug: str) -> list[Row]: """One page: each `jobs[].data` has `slug` (equal to `req_id`), `title`, the HTML `description` and, on some, `qualifications`, `full_location` @@ -117,16 +144,29 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: categories = j.get("categories") or [] text = "\n\n".join(p for p in (_plain(j.get("description")), _plain(j.get("qualifications"))) if p) - out.append({ + places = _places(j) + row: Row = { "source_id": role, "title": title, - "location": (j.get("full_location") - or ", ".join(p for p in (j.get("city"), j.get("country")) if p)).strip(), + "places": places, + "location": "; ".join(p["text"] for p in places), "url": f"https://{slug.lower()}/jobs/{role}", "posted": parse_dt(j.get("posted_date")), "department": str((categories[0] if categories else {}).get("name") or "").strip(), "description": text, "remote": None, "salary_text": "", - }) + } + # One code, for a role at one place: over several countries it would + # gate the others by the site's own. + code = str(j.get("country_code") or "").strip().upper() + if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code): + row["country"] = code + kind = str(j.get("employment_type") or "").strip() + if kind: + if employment := _EMPLOYMENT.get(kind.upper()): + row["employment"] = employment + else: + log.debug("jibe: unknown employment_type %r", kind) + out.append(row) return out diff --git a/plugins/readers/jobylon.py b/plugins/readers/jobylon.py index 64ba2b0..a631107 100644 --- a/plugins/readers/jobylon.py +++ b/plugins/readers/jobylon.py @@ -18,13 +18,24 @@ from __future__ import annotations import html import json +import logging import re from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Get, Row, parse_dt +from ._contract import Ask, Fields, Place, Reads, Row +from ._shared import Get, parse_dt + +log = logging.getLogger(__name__) + +READS = Reads( + scope="board", listing="whole", text="detail", teaser=True, complete="whole", + places="many", + fields=Fields(posted="trusted", remote="stated", seniority="stated", employment="stated"), + robots="read: robots.txt allows /jobs/companies/ (checked 2026-09-29)", +) NAME = "jobylon" LABEL = "Jobylon" @@ -47,6 +58,21 @@ _SLUG = re.compile(r"[0-9]{1,12}") #: from. _ROLE_PATH = re.compile(r"/jobs/[0-9]+(?:-[A-Za-z0-9_-]*)?/?") +#: The embed's `experience` -> `app.models.SENIORITIES`. Only the coarse levels +#: are mapped; "Experienced" sits between "Mid level" and the managerial +#: levels and says too little to place, so it stays unmapped and is logged. +_SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern", + "entry level": "junior", "entry-level": "junior", "junior": "junior", + "graduate": "junior", "mid level": "mid", "mid-level": "mid", "mid": "mid", + "senior": "senior", "lead": "lead", "manager": "manager", "director": "director", + "executive": "exec", "not applicable": ""} + +#: The embed's `employment_type` -> the app's employment words. +_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", + "part time": "part_time", "contract": "contract", "consultant": "contract", + "temporary": "temporary", "internship": "intern", "intern": "intern", + "freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"} + _ASSIGNMENT = "JBL.embed_v2['jobs'] = " _KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _TRAILING_COMMA = re.compile(r",(\s*[\]}])") @@ -74,17 +100,13 @@ def careers_url(slug: str) -> str: return f"https://{_EMP}/companies/{slug}/" -def feed_url(slug: str) -> str: +def feed_url(slug: str, ask: Ask) -> str: + """Both hosts serve the same robots.txt (compared on 2026-09-29), so the + roles' pages on emp.jobylon.com are covered by what was checked for this + one.""" return f"https://{_CDN}/jobs/companies/{slug}/embed/v2/" -def robots_page(slug: str) -> str: - """The page the board is read from. Both hosts serve the same robots.txt - (compared on 2026-09-29), so the roles' pages on emp.jobylon.com are - covered by the check on this one.""" - return feed_url(slug) - - def _json(text: str) -> Any: """A JavaScript array or object literal as JSON: keys quoted, strings single-quoted with `\\uXXXX` escapes, trailing commas.""" @@ -157,6 +179,22 @@ def _published(value: Any) -> Any: return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}") +def _mapped(table: dict[str, str], value: Any, what: str) -> str | None: + """`table`'s word for the host's `value`; None when the host states none + or the table does not know it (logged, so the table can grow).""" + word = str(value or "").strip() + if not word: + return None + found = table.get(word.lower()) + if found is None: + log.debug("jobylon: unknown %s %r", what, word) + return found or None + + +def _place(text: str) -> Place: + return {"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None} + + def parse(body: bytes, slug: str) -> list[Row]: """The embed page -> one row per role. `posted` is the day `published_date` names, which `detail` replaces with the exact time. `location` joins the @@ -172,18 +210,30 @@ def parse(body: bytes, slug: str) -> list[Row]: if not title or not job_id or not _ROLE_PATH.fullmatch(path): continue function = str(j.get("function") or "").strip() - out.append({ + places = [_place(name) for name in dict.fromkeys( + str(name).strip() for name in j.get("locations") or [] if str(name).strip())] + row: Row = { "source_id": job_id, "title": title, - "location": "; ".join(dict.fromkeys( - str(name).strip() for name in j.get("locations") or [] if str(name).strip())), + "places": places, + "location": "; ".join(pl["text"] for pl in places), "url": f"https://{_EMP}{path}", "posted": _published(j.get("published_date")), "department": "" if function == "All functions" else function, "description": "", "remote": _REMOTE.get(str(j.get("workspace") or "").strip().lower()), "salary_text": "", # the embed states no pay - }) + } + # `summary` is the text the company gave the listing, the string + # "None" when it gave none. + summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split()) + if summary and summary != "None": + row["teaser"] = summary + if seniority := _mapped(_SENIORITY, j.get("experience"), "experience"): + row["seniority"] = seniority + if employment := _mapped(_EMPLOYMENT, j.get("employment_type"), "employment_type"): + row["employment"] = employment + out.append(row) return out @@ -213,4 +263,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: } if places: found["location"] = "; ".join(places) + found["places"] = [_place(name) for name in places] return found diff --git a/plugins/readers/lever.py b/plugins/readers/lever.py index f988ce6..f2bca21 100644 --- a/plugins/readers/lever.py +++ b/plugins/readers/lever.py @@ -16,12 +16,32 @@ The field map is ported from `openings.go` in from __future__ import annotations import json +import logging +import re from collections.abc import Callable from typing import Any from desk.sources import strip_html -from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount +from ._contract import Ask, Fields, Place, Reads, Row +from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount + +log = logging.getLogger(__name__) + +READS = Reads( + scope="board", listing="whole", text="listing", complete="whole", + places="many", + fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated", + employment="stated"), + robots="api: no robots.txt check needed (public board API), 2026-09-30", +) + +#: `categories.commitment` -> the app's employment words. A commitment is free +#: text on Lever ("Permanent" is seen too and says nothing about hours). +_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time", + "part-time": "part_time", "part time": "part_time", "parttime": "part_time", + "intern": "intern", "internship": "intern", "contract": "contract", + "contractor": "contract", "temporary": "temporary", "freelance": "freelance"} NAME = "lever" LABEL = "Lever" @@ -60,7 +80,7 @@ def careers_url(slug: str) -> str: return f"https://jobs.{infix}lever.co/{site}" -def feed_url(slug: str) -> str: +def feed_url(slug: str, ask: Ask) -> str: infix, site = _split(slug) return f"https://api.{infix}lever.co/v0/postings/{site}?mode=json" @@ -88,6 +108,19 @@ def _salary_text(salary_range: dict[str, Any]) -> str: salary_range.get("currency"), period) +def _places(categories: dict[str, Any]) -> list[Place]: + """`categories.allLocations` (every place of the role, the first being + `location`), or `location` alone when the list is missing.""" + names = [categories.get("location"), *(categories.get("allLocations") or [])] + out: list[Place] = [] + for name in names: + text = str(name or "").strip() + if text and text not in (p["text"] for p in out): + out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) + else None}) + return out + + def parse(body: bytes, slug: str) -> list[Row]: data = json.loads(body) if not isinstance(data, list): @@ -104,10 +137,12 @@ def parse(body: bytes, slug: str) -> list[Row]: salary_min, salary_max, currency = yearly_amount( salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"), salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12}) - rows.append({ + places = _places(categories) + row: Row = { "source_id": str(j.get("id") or ""), "title": title, - "location": (categories.get("location") or "").strip(), + "places": places, + "location": "; ".join(p["text"] for p in places), "url": url, "posted": parse_dt(j.get("createdAt")), "department": (categories.get("department") or categories.get("team") or "").strip(), @@ -117,5 +152,17 @@ def parse(body: bytes, slug: str) -> list[Row]: "salary_min": salary_min, "salary_max": salary_max, "currency": currency, - }) + } + # `country` is one code, for a role at one place; over several it would + # gate the others by the first's country. + country = str(j.get("country") or "").strip().upper() + if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country): + row["country"] = country + commitment = str(categories.get("commitment") or "").strip() + if commitment: + if employment := _EMPLOYMENT.get(commitment.lower()): + row["employment"] = employment + else: + log.debug("lever: unknown commitment %r", commitment) + rows.append(row) return rows diff --git a/tests/fixtures/ats/ashby-places.json b/tests/fixtures/ats/ashby-places.json new file mode 100644 index 0000000..32482f5 --- /dev/null +++ b/tests/fixtures/ats/ashby-places.json @@ -0,0 +1,38 @@ +{ + "jobs": [ + { + "id": "a1", + "title": "Research Engineer", + "department": "Research", + "team": "Safety", + "location": "San Francisco", + "secondaryLocations": [ + { + "location": "London, UK" + }, + { + "location": "Remote - US" + } + ], + "employmentType": "FullTime", + "workplaceType": "Hybrid", + "isRemote": true, + "publishedAt": "2026-09-01T00:00:00Z", + "jobUrl": "https://ashby.example/a1", + "descriptionHtml": "

Do research.

" + }, + { + "id": "a2", + "title": "Recruiter", + "department": "People", + "location": "Dublin", + "secondaryLocations": [], + "employmentType": "Contract", + "workplaceType": "Remote", + "isRemote": true, + "publishedAt": "2026-09-02T00:00:00Z", + "jobUrl": "https://ashby.example/a2", + "descriptionHtml": "" + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/ats/bamboohr-places.json b/tests/fixtures/ats/bamboohr-places.json new file mode 100644 index 0000000..a0d0562 --- /dev/null +++ b/tests/fixtures/ats/bamboohr-places.json @@ -0,0 +1,24 @@ +{ + "meta": { + "totalCount": 1 + }, + "result": [ + { + "id": "9", + "jobOpeningName": "Analyst", + "departmentLabel": "Finance", + "employmentStatusLabel": "Part-Time", + "location": { + "city": "Aarhus", + "state": null + }, + "atsLocation": { + "country": "Denmark", + "state": null, + "city": null + }, + "isRemote": null, + "locationType": "0" + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/ats/deel-places.html b/tests/fixtures/ats/deel-places.html new file mode 100644 index 0000000..0855ffa --- /dev/null +++ b/tests/fixtures/ats/deel-places.html @@ -0,0 +1,6 @@ +Klarna Careers
+ + + + + diff --git a/tests/fixtures/ats/eightfold-places.json b/tests/fixtures/ats/eightfold-places.json new file mode 100644 index 0000000..84e36b4 --- /dev/null +++ b/tests/fixtures/ats/eightfold-places.json @@ -0,0 +1,33 @@ +{ + "status": 200, + "data": { + "count": 2, + "positions": [ + { + "id": 9001, + "name": "Field Engineer", + "locations": [ + "Stockholm,Stockholm,Sweden", + "Gothenburg,Vastra Gotaland,Sweden" + ], + "postedTs": 1790703782, + "creationTs": 1785822052, + "department": "Service", + "workLocationOption": "hybrid", + "positionUrl": "/careers/job/9001" + }, + { + "id": 9002, + "name": "Buyer", + "locations": [ + "Lund,Skane,Sweden" + ], + "postedTs": 1790703782, + "creationTs": 1785822052, + "department": "Supply", + "workLocationOption": "onsite", + "positionUrl": "/careers/job/9002" + } + ] + } +} \ No newline at end of file diff --git a/tests/fixtures/ats/gem-places.json b/tests/fixtures/ats/gem-places.json new file mode 100644 index 0000000..b7347cb --- /dev/null +++ b/tests/fixtures/ats/gem-places.json @@ -0,0 +1 @@ +{"data":{"oatsExternalJobPostings":{"jobPostings":[{"id":"T2F0c0pvYlBvc3Q6MTA0NDAzNA==","extId":"8080046002","title":"Staff AI Product Engineer (Prague/Brno/Remote EU)","locations":[{"id":"38771","name":"Prague (Hybrid)","city":"Prague","isoCountry":"CZE","isRemote":false,"extId":"4029634002"}],"job":{"id":"T2F0c0pvYjo2MDYxMDY=","department":{"id":"37338","name":"Product Engineering","extId":"4065161002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6NTY0ODcxMw==","extId":"am9icG9zdDpS7XehojHQK_NnncgSKWoQ","title":"Staff Full-stack Product Engineer (Prague/Brno/Remote EU)","locations":[{"id":"38771","name":"Prague (Hybrid)","city":"Prague","isoCountry":"CZE","isRemote":false,"extId":"4029634002"}],"job":{"id":"T2F0c0pvYjo2NjEyMTk=","department":{"id":"37338","name":"Product Engineering","extId":"4065161002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6NTY0ODcxNg==","extId":"am9icG9zdDqWC1V_RM7U6ORvTd1Iu64r","title":"Staff Backend Product Engineer (Prague/Brno/Remote EU)","locations":[{"id":"38771","name":"Prague (Hybrid)","city":"Prague","isoCountry":"CZE","isRemote":false,"extId":"4029634002"},{"id":"38772","name":"Remote Europe","city":"Remote Europe","isoCountry":"CZE","isRemote":true,"extId":"4030754002"},{"id":"38781","name":"Brno (Hybrid)","city":"Brno","isoCountry":"CZE","isRemote":false,"extId":"4051723002"}],"job":{"id":"T2F0c0pvYjoyMDg5NTcx","department":{"id":"37338","name":"Product Engineering","extId":"4065161002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6NzU2OTA5MQ==","extId":"am9icG9zdDpfq1dHcevndHGSzK_aYE1g","title":"Marketing Systems Operations Lead","locations":[{"id":"38780","name":"San Francisco","city":"San Francisco","isoCountry":"USA","isRemote":false,"extId":"4030598002"}],"job":{"id":"T2F0c0pvYjo2NTkyMzYy","department":{"id":"37379","name":"GTM Ops","extId":"4068217002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6NzU5NjA4Mg==","extId":"am9icG9zdDq5-IOakBvxHzWhMPnTzoPp","title":"Staff Fullstack DevX Engineer (Prague/Brno/Remote EU)","locations":[{"id":"38771","name":"Prague (Hybrid)","city":"Prague","isoCountry":"CZE","isRemote":false,"extId":"4029634002"},{"id":"38772","name":"Remote Europe","city":"Remote Europe","isoCountry":"CZE","isRemote":true,"extId":"4030754002"},{"id":"38781","name":"Brno (Hybrid)","city":"Brno","isoCountry":"CZE","isRemote":false,"extId":"4051723002"}],"job":{"id":"T2F0c0pvYjo2NjA4ODU3","department":{"id":"37338","name":"Product Engineering","extId":"4065161002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6NzYwMjE1NA==","extId":"am9icG9zdDoaMPkCxlUbsYDApcWCdyCP","title":"Staff Frontend Product Engineer (Prague/Brno/Remote EU)","locations":[{"id":"38771","name":"Prague (Hybrid)","city":"Prague","isoCountry":"CZE","isRemote":false,"extId":"4029634002"}],"job":{"id":"T2F0c0pvYjo2NjA4ODU4","department":{"id":"37338","name":"Product Engineering","extId":"4065161002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6NzYwMzc4Mg==","extId":"am9icG9zdDpGGIKW-IlxhJ2EnwVAtLRB","title":"Staff Full-stack Product Engineer, Growth (Prague/Brno/Remote EU) ","locations":[{"id":"38771","name":"Prague (Hybrid)","city":"Prague","isoCountry":"CZE","isRemote":false,"extId":"4029634002"}],"job":{"id":"T2F0c0pvYjo2NjEyMTk=","department":{"id":"37338","name":"Product Engineering","extId":"4065161002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6NzcyNzc0Ng==","extId":"am9icG9zdDq9ZHg3x3sdBTy6feA4MZ9f","title":"Digital Marketing Manager","locations":[{"id":"38780","name":"San Francisco","city":"San Francisco","isoCountry":"USA","isRemote":false,"extId":"4030598002"}],"job":{"id":"T2F0c0pvYjo2NjkyOTAw","department":{"id":"37358","name":"Digital Marketing","extId":"4076635002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6Nzk0NDYyMA==","extId":"am9icG9zdDp906RvIzQmjA9funJO3zB1","title":"AI Data Engineer","locations":[{"id":"38771","name":"Prague (Hybrid)","city":"Prague","isoCountry":"CZE","isRemote":false,"extId":"4029634002"},{"id":"38781","name":"Brno (Hybrid)","city":"Brno","isoCountry":"CZE","isRemote":false,"extId":"4051723002"}],"job":{"id":"T2F0c0pvYjo2ODE0MzAz","department":{"id":"37352","name":"Data","extId":"4076631002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6ODAzMTI0MQ==","extId":"am9icG9zdDqzuJaiLLHinwNQLMZuqlux","title":"Digital Marketing Specialist ","locations":[{"id":"38780","name":"San Francisco","city":"San Francisco","isoCountry":"USA","isRemote":false,"extId":"4030598002"}],"job":{"id":"T2F0c0pvYjo2NjkyOTAw","department":{"id":"37358","name":"Digital Marketing","extId":"4076635002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}},{"id":"T2F0c0pvYlBvc3Q6ODA1NTY3NQ==","extId":"am9icG9zdDrYRR2Ri8264a07xF_f9eI3","title":"AI Customer Success Engineer ","locations":[{"id":"38780","name":"San Francisco","city":"San Francisco","isoCountry":"USA","isRemote":false,"extId":"4030598002"}],"job":{"id":"T2F0c0pvYjo2ODc3MjM2","department":{"id":"37347","name":"Customer Success","extId":"4071103002"},"locationType":"HYBRID","employmentType":"FULL_TIME"}}]}}} \ No newline at end of file diff --git a/tests/fixtures/ats/greenhouse-places.json b/tests/fixtures/ats/greenhouse-places.json new file mode 100644 index 0000000..7ea3fa6 --- /dev/null +++ b/tests/fixtures/ats/greenhouse-places.json @@ -0,0 +1,68 @@ +{ + "jobs": [ + { + "id": 7001, + "title": "Solutions Engineer", + "location": { + "name": "Hybrid" + }, + "absolute_url": "https://gh.example/7001", + "first_published": "2026-09-09T08:00:00-04:00", + "departments": [ + { + "id": 1, + "name": "Sales" + } + ], + "content": "", + "offices": [ + { + "id": 1, + "name": "Austin, TX", + "location": "Austin, TX, United States" + }, + { + "id": 2, + "name": "New York, NY", + "location": "New York, New York, United States" + } + ] + }, + { + "id": 7002, + "title": "Support Engineer", + "location": { + "name": "Remote India" + }, + "absolute_url": "https://gh.example/7002", + "first_published": "2026-09-09T08:00:00-04:00", + "departments": [], + "content": "", + "offices": [ + { + "id": 3, + "name": "Bangalore", + "location": "Bangalore, India" + } + ] + }, + { + "id": 7003, + "title": "Designer", + "location": { + "name": "Lisbon, Portugal" + }, + "absolute_url": "https://gh.example/7003", + "first_published": "2026-09-09T08:00:00-04:00", + "departments": [], + "content": "", + "offices": [ + { + "id": 4, + "name": "Lisbon", + "location": "Lisbon, Portugal" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/ats/hibob-places.json b/tests/fixtures/ats/hibob-places.json new file mode 100644 index 0000000..6948a15 --- /dev/null +++ b/tests/fixtures/ats/hibob-places.json @@ -0,0 +1 @@ +{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]} \ No newline at end of file diff --git a/tests/fixtures/ats/hrmanager-places.xml b/tests/fixtures/ats/hrmanager-places.xml new file mode 100644 index 0000000..08883bb --- /dev/null +++ b/tests/fixtures/ats/hrmanager-places.xml @@ -0,0 +1,91 @@ +HR Manager Talent Recruiter - Position Listhttps://candidate.hr-manager.net/vacancies/list.aspx?customer=stsgroup_trHR Manager Talent Recruiter - Position ListTue, 29 Sep 2026 20:11:15 +0200143857https://candidate.hr-manager.net/ApplicationInit.aspx?cid=2523&ProjectId=143857&DepartmentId=18958&MediaId=5Commissioning Engineer- T&D<p><strong>About the Position</strong></p> + +<p style="text-align:justify">We have an exciting opportunity for a Commissioning Engineer, to join STS Group T&D Division.</p> + +<p style="text-align:justify">The position will be located in Ireland.</p> + +<p style="text-align:justify"><br /> +STS Group, is dedicated to being the leading and most cost-efficient provider of electrical engineering, instrumentation, design and commissioning services across Ireland, UK, Europe and the Middle East. Since its formation, STS Group has broadened its scope to become a full-service provider to world renowned brand names in the Pharmaceutical, Data Centre, Utilities, Oil, Gas & Chemicals, Food & Beverage, Transport and other industry sectors.</p> + +<p style="text-align:justify">Headquartered in Waterford, Ireland, STS Group is one of three business units within Dussmann Technical Solutions (DTS) and is a proud member of the Dussmann Group. STS Group employs the best people, invests heavily in training and education ensuring that the highest standards of health, safety and governance are applied throughout the organisation</p> + +<p><br /> +<strong>What you will be doing?</strong></p> + +<ul> + <li>Tue, 18 Mar 2025 17:31:36 +01002025-04-02T12:30:03+02:00143844https://candidate.hr-manager.net/ApplicationInit.aspx?cid=2523&ProjectId=143844&DepartmentId=18958&MediaId=5Electrical Tester- Dublin<p>The Company</p> + +<p>STS Group, is dedicated to being the leading and most cost-efficient provider of electrical engineering, instrumentation, design and commissioning services across Ireland, UK, Europe and the Middle East. Since its formation, STS Group has broadened its scope to become a full-service provider to world renowned brand names in the Pharmaceutical, Data Centre, Utilities, Oil, Gas & Chemicals, Food & Beverage, Transport and other industry sectors.</p> + +<p>Headquartered in Waterford, Ireland, STS Group is one of three business units within Dussmann Technical Solutions (DTS) and is a proud member of the Dussmann Group. STS Group employs the best people, invests heavily in training and education ensuring that the highest standards of health, safety and governance are applied throughout the organisation.</p> + +<p> </p> + +<p> </p> + +<p>*This company is an equal opportunities employer*</p> + +<p> </p> + +<p>Job description</p> + +<p> </p> + +<p> </p> + +<p>Duties & Responsibilities:</p> + +<ul> + <li>Fully qualified Electrician with current <strong>Safe Electric (RECI) Certification</strong>.</li>Tue, 18 Feb 2025 11:02:55 +01002025-02-18T11:05:30+01:00143854https://candidate.hr-manager.net/ApplicationInit.aspx?cid=2523&ProjectId=143854&DepartmentId=18958&MediaId=5Pre Apprentice- Dublin 15<p>Are you a pre Apprentice? Join Our Apprenticeship Program – Your Future Starts Here!</p> + +<p>Are you ready to jumpstart your career in Electrical & Instrumentation? We're on the lookout for passionate pre-apprentices to join our team. Here’s why this opportunity is a game-changer:</p> + +<p>What’s in It for You?</p> + +<p>· Get Rewarded: Signing bonus plus a €1,000 Exceptional Performance Award for every successful apprenticeship year.</p> + +<p>· Real-World Training: Learn by doing, with hands-on experience from day one.</p> + +<p>· Globally Recognised Certification: Open doors anywhere in the world with industry-approved qualifications.</p> + +<p>· Secure Your Future: Enjoy job stability and countless opportunities for growth.</p> + +<p>· Climb the Ladder: Our "Beyond Apprentice" program fast-tracks your career progression.</p> + +<p>· Work Variety: Gain experience in diverse environments and departments.</p> + +<p>· Stay Ahead: Master cutting-edge technologies and stay at the forefront of innovation.</p> + +<p>What You’ll Be Doing</p> + +<p>Tue, 11 Mar 2025 16:30:53 +01002025-03-11T16:32:11+01:00143849https://candidate.hr-manager.net/ApplicationInit.aspx?cid=2523&ProjectId=143849&DepartmentId=18958&MediaId=5QS- Cork<p><strong>The Company</strong></p> + +<p>STS Group, is dedicated to being the leading and most cost-efficient provider of electrical engineering, instrumentation, design and commissioning services across Ireland, UK, Europe and the Middle East. Since its formation, STS Group has broadened its scope to become a full-service provider to world renowned brand names in the Pharmaceutical, Data Centre, Utilities, Oil, Gas & Chemicals, Food & Beverage, Transport and other industry sectors.</p> + +<p>Headquartered in Waterford, Ireland, STS Group is one of three business units within Dussmann Technical Solutions (DTS) and is a proud member of the Dussmann Group. STS Group employs the best people, invests heavily in training and education ensuring that the highest standards of health, safety and governance are applied throughout the organisation</p> + +<p>*This company is an equal opportunities employer*</p> + +<p> </p> + +<p><strong>Quantity Surveyor Criteria:</strong></p> + +<ul> + <li>2+ years Post-Graduate experience with a Main Contractor/ Developer.</li> + <li>Qualification in Quantity Surveying or similar area</li> + <li>Thu, 06 Mar 2025 15:30:00 +01002025-03-06T15:33:46+01:00143848https://candidate.hr-manager.net/ApplicationInit.aspx?cid=2523&ProjectId=143848&DepartmentId=18958&MediaId=5QS- Dublin<p><strong>The Company</strong></p> + +<p>STS Group, is dedicated to being the leading and most cost-efficient provider of electrical engineering, instrumentation, design and commissioning services across Ireland, UK, Europe and the Middle East. Since its formation, STS Group has broadened its scope to become a full-service provider to world renowned brand names in the Pharmaceutical, Data Centre, Utilities, Oil, Gas & Chemicals, Food & Beverage, Transport and other industry sectors.</p> + +<p>Headquartered in Waterford, Ireland, STS Group is one of three business units within Dussmann Technical Solutions (DTS) and is a proud member of the Dussmann Group. STS Group employs the best people, invests heavily in training and education ensuring that the highest standards of health, safety and governance are applied throughout the organisation</p> + +<p>*This company is an equal opportunities employer*</p> + +<p> </p> + +<p><strong>Senior Quantity Surveyor Criteria:</strong></p> + +<ul> + <li>2 + years Post-Graduate experience with a Main Contractor/ Developer.</li> + <li>Qualification in Quantity Surveying or similar area</li> + <li>Thu, 06 Mar 2025 15:11:42 +01002025-03-06T15:29:29+01:00 \ No newline at end of file diff --git a/tests/fixtures/ats/jibe-places.json b/tests/fixtures/ats/jibe-places.json new file mode 100644 index 0000000..4fffe1a --- /dev/null +++ b/tests/fixtures/ats/jibe-places.json @@ -0,0 +1,41 @@ +{ + "jobs": [ + { + "data": { + "slug": "1", + "req_id": "1", + "title": "Analyst", + "description": "

x

", + "city": "Amsterdam", + "country": "Netherlands", + "country_code": "NL", + "full_location": "Amsterdam, Netherlands; Manchester, United Kingdom", + "multipleLocations": true, + "employment_type": "FULL_TIME", + "categories": [ + { + "name": "Finance" + } + ], + "posted_date": "2026-09-29T12:50:00+0000" + } + }, + { + "data": { + "slug": "2", + "req_id": "2", + "title": "Clerk", + "description": "

y

", + "city": "Oslo", + "country": "Norway", + "country_code": "NO", + "full_location": "Oslo, Norway", + "multipleLocations": false, + "employment_type": "PART_TIME", + "categories": [], + "posted_date": "2026-09-29T12:50:00+0000" + } + } + ], + "totalCount": 2 +} \ No newline at end of file diff --git a/tests/fixtures/ats/jobylon-places.html b/tests/fixtures/ats/jobylon-places.html new file mode 100644 index 0000000..bb1f81c --- /dev/null +++ b/tests/fixtures/ats/jobylon-places.html @@ -0,0 +1,33 @@ + diff --git a/tests/fixtures/ats/jobylon-teaser.html b/tests/fixtures/ats/jobylon-teaser.html new file mode 100644 index 0000000..0281dc8 --- /dev/null +++ b/tests/fixtures/ats/jobylon-teaser.html @@ -0,0 +1,18 @@ + diff --git a/tests/fixtures/ats/lever-places.json b/tests/fixtures/ats/lever-places.json new file mode 100644 index 0000000..e61cfa5 --- /dev/null +++ b/tests/fixtures/ats/lever-places.json @@ -0,0 +1,36 @@ +[ + { + "id": "l1", + "text": "Data Scientist", + "hostedUrl": "https://lever.example/l1", + "createdAt": 1789430400000, + "categories": { + "location": "London", + "department": "Data", + "commitment": "Full-time", + "allLocations": [ + "London", + "Stockholm" + ] + }, + "country": "GB", + "workplaceType": "hybrid", + "description": "

Data.

" + }, + { + "id": "l2", + "text": "Designer", + "hostedUrl": "https://lever.example/l2", + "createdAt": 1789430400000, + "categories": { + "location": "Berlin", + "commitment": "Permanent", + "allLocations": [ + "Berlin" + ] + }, + "country": "DE", + "workplaceType": "on-site", + "description": "" + } +] \ No newline at end of file diff --git a/tests/test_ats_ashby.py b/tests/test_ats_ashby.py new file mode 100644 index 0000000..49cd788 --- /dev/null +++ b/tests/test_ats_ashby.py @@ -0,0 +1,44 @@ +"""The Ashby host module's places, work arrangement and employment, against a +small saved board. No network. The pay and the shape of a plain row are in +`test_ats.py`.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins.readers import Ask, ashby # noqa: E402 + +BOARD = (ROOT / "tests" / "fixtures" / "ats" / "ashby-places.json").read_bytes() + + +def test_the_primary_and_every_secondary_location_are_places() -> None: + first, second = ashby.parse(BOARD, "acme") + assert first["places"] == [{"text": "San Francisco", "remote": False}, + {"text": "London, UK", "remote": None}, + {"text": "Remote - US", "remote": True}] + assert first["location"] == "San Francisco; London, UK; Remote - US" + assert second["places"] == [{"text": "Dublin", "remote": True}] + + +def test_a_hybrid_role_is_not_remote_although_the_host_flags_it() -> None: + first, second = ashby.parse(BOARD, "acme") + assert first["remote"] == "hybrid" and second["remote"] == "remote" + + +def test_the_employment_type_is_mapped() -> None: + first, second = ashby.parse(BOARD, "acme") + assert (first["employment"], second["employment"]) == ("full_time", "contract") + + +def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None: + reads = ashby.READS + assert (reads.listing, reads.text, reads.places, reads.empty) == ( + "whole", "listing", "many", "suspect") + assert reads.fields.posted == "trusted" and reads.fields.pay == "figures" + assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30" + assert ashby.feed_url("acme", Ask()).endswith("/job-board/acme?includeCompensation=true") diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py index be8ca53..f97357e 100644 --- a/tests/test_ats_bamboohr.py +++ b/tests/test_ats_bamboohr.py @@ -9,7 +9,6 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any -import httpx import pytest ROOT = Path(__file__).resolve().parents[1] @@ -18,7 +17,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import bamboohr # noqa: E402 +from plugins.readers import Ask, bamboohr # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -99,9 +98,10 @@ def test_detail_reads_the_text_the_date_and_the_country() -> None: found = bamboohr.detail(_get_of({url: _read("bamboohr-detail-37.json")}, asked), "configit", row) assert asked == [(url, {"Accept": "application/json"})] - assert set(found) <= set(ats.DETAIL_FIELDS) + assert set(found) <= set(ats.DETAIL_FIELDS) | {"places"} assert found["posted"] == datetime(2026, 9, 23, tzinfo=UTC) assert found["location"] == "København Ø, Denmark" + assert found["places"] == [{"text": "København Ø, Denmark", "remote": None}] assert found["remote"] == "hybrid" text = found["description"] assert text.startswith("Our Global Marketing Team is looking for") @@ -125,38 +125,44 @@ def test_detail_refuses_an_answer_that_is_not_a_role() -> None: bamboohr.detail(_get_of({url: body}, []), "configit", row) -def test_the_board_and_its_details_are_read_through_the_app( - monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(ats, "PAGE_DELAY", 0) +def test_the_board_and_its_details_are_read_with_the_json_accept_on_the_detail() -> None: base = "https://configit.bamboohr.com/careers" - seen: list[str] = [] + asked: list[tuple[str, Any]] = [] + answers: dict[str, bytes | Exception] = { + f"{base}/list": _read("bamboohr.json"), + f"{base}/37/detail": _read("bamboohr-detail-37.json")} + get = _get_of(answers, asked) + rows = bamboohr.parse(get(bamboohr.feed_url("configit", Ask())), "configit") + assert len(rows) == 2 + found = bamboohr.detail(get, "configit", rows[1]) + assert [url for url, _ in asked] == [f"{base}/list", f"{base}/37/detail"] + assert asked[1][1] == {"Accept": "application/json"} + assert found["location"] == "København Ø, Denmark" + assert found["posted"] == datetime(2026, 9, 23, tzinfo=UTC) - def handler(request: httpx.Request) -> httpx.Response: - seen.append(str(request.url)) - if request.url.path == "/careers/list": - return httpx.Response(200, content=_read("bamboohr.json")) - if request.url.path == "/careers/26/detail": - return httpx.Response(404) - assert request.headers["accept"] == "application/json" - return httpx.Response(200, content=_read("bamboohr-detail-37.json")) - with httpx.Client(transport=httpx.MockTransport(handler)) as client: - rows, complete = ats._fetch_rows(client, "bamboohr:configit") - assert complete and len(rows) == 2 and seen == [f"{base}/list"] - filled, done, failed = ats._read_details(client, "bamboohr:configit", rows, complete, - set(), set()) - assert seen[1:] == [f"{base}/26/detail", f"{base}/37/detail"] - # A 404 counts as read; the other role is filled from its own page. - assert done == {"bamboohr:configit:26", "bamboohr:configit:37"} and failed == set() - assert filled[0]["description"] == "" and filled[1]["location"] == "København Ø, Denmark" - assert filled[1]["posted"] == datetime(2026, 9, 23, tzinfo=UTC) +def test_the_role_states_one_place_and_the_employment() -> None: + rows = bamboohr.parse(_read("bamboohr.json"), "configit") + assert rows[1]["places"] == [{"text": "København Ø", "remote": None}] + assert rows[1]["employment"] == "full_time" + body = json.loads(_read("bamboohr.json")) + body["result"][0] |= {"employmentStatusLabel": "Seasonal", "location": {"city": None}} + first = bamboohr.parse(json.dumps(body).encode(), "configit")[0] + assert first["places"] == [] and "employment" not in first + + +def test_it_declares_what_it_reads() -> None: + reads = bamboohr.READS + assert (reads.listing, reads.text, reads.places, reads.fields.posted) == ( + "whole", "detail", "one", "trusted") + assert reads.robots.startswith("read: robots.txt allows /careers") + assert not hasattr(bamboohr, "robots_page") def test_the_slug_the_urls_and_the_allow_list() -> None: assert bamboohr.slug_ok("configit") and bamboohr.slug_ok("Configit") assert bamboohr.careers_url("configit") == "https://configit.bamboohr.com/careers" - assert bamboohr.robots_page("configit") == "https://configit.bamboohr.com/careers" - assert bamboohr.feed_url("Configit") == "https://configit.bamboohr.com/careers/list" + assert bamboohr.feed_url("Configit", Ask()) == "https://configit.bamboohr.com/careers/list" allow = bamboohr.allowed("Configit") assert allow("configit.bamboohr.com") and allow("CONFIGIT.bamboohr.com") assert not allow("bamboohr.com") and not allow("other.bamboohr.com") diff --git a/tests/test_ats_deel.py b/tests/test_ats_deel.py index 26acff2..aa27f07 100644 --- a/tests/test_ats_deel.py +++ b/tests/test_ats_deel.py @@ -17,7 +17,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import deel # noqa: E402 +from plugins.readers import Ask, deel # noqa: E402 PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes() URL = "https://jobs.deel.com/klarna" @@ -46,6 +46,13 @@ def test_several_locations_are_joined_and_remote_uses_the_stores_words() -> None assert rows["a8be45ac"]["location"] == "Stockholm; London; Milan; Warsaw" assert rows["f96bb9d4"]["remote"] == "remote" assert rows["f96bb9d4"]["location"] == "Ohio" + # Every place is kept, in the host's order; the work arrangement is the + # places' remote flag. + assert rows["a8be45ac"]["places"] == [ + {"text": name, "remote": False} for name in ("Stockholm", "London", "Milan", "Warsaw")] + assert rows["f96bb9d4"]["places"] == [{"text": "Ohio", "remote": True}] + assert rows["f96bb9d4"]["employment"] == "contract" # "Consultant" + assert rows["21584046"]["employment"] == "full_time" def test_pay_is_only_text_since_the_period_is_not_stated() -> None: @@ -90,37 +97,23 @@ def test_a_posting_without_a_title_or_id_is_skipped() -> None: # ------------------------------------------------------------ the fetch -@pytest.fixture -def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: - answers: dict[str, Any] = {} +def test_the_board_is_one_request() -> None: asked: list[str] = [] - def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, - www_twin=False): + def get(url: str, **kwargs: Any) -> bytes: asked.append(url) - if not ats._allowed_host(host, slug)(httpx.URL(url).host): - raise ats._Refused(url) - answer = answers[url] - if isinstance(answer, BaseException): - raise answer - return answer + return PAGE - monkeypatch.setattr(ats, "_get", fake_get) - return {"answers": answers, "asked": asked} + rows = deel.parse(get(deel.feed_url("klarna", Ask())), "klarna") + assert len(rows) == 5 and asked == [URL] -def test_the_board_is_one_request_and_complete(web: dict[str, Any]) -> None: - web["answers"][URL] = PAGE - with httpx.Client() as client: - rows, complete = ats._fetch_rows(client, "deel:klarna") - assert len(rows) == 5 and complete - assert web["asked"] == [URL] - - -def test_a_page_that_cannot_be_read_fails_the_board(web: dict[str, Any]) -> None: - web["answers"][URL] = b"Job Board Not Found" - with httpx.Client() as client, pytest.raises(ValueError): - ats._fetch_rows(client, "deel:klarna") +def test_it_declares_a_whole_listing_read_from_the_page() -> None: + reads = deel.READS + assert (reads.listing, reads.text, reads.places, reads.complete) == ( + "whole", "listing", "many", "whole") + assert reads.fields.pay == "line" and "/api/ is disallowed and not read" in reads.robots + assert not hasattr(deel, "robots_page") # ------------------------------------------------------- slugs and hosts @@ -131,10 +124,9 @@ def test_the_module_is_registered_as_a_feed_host() -> None: assert not hasattr(deel, "page") and not hasattr(deel, "detail") -def test_the_urls_and_the_robots_page_are_the_boards_own() -> None: +def test_the_urls_are_the_boards_own() -> None: assert deel.careers_url("klarna") == URL - assert deel.feed_url("klarna") == URL - assert deel.robots_page("klarna") == URL + assert deel.feed_url("klarna", Ask()) == URL assert deel.allowed("klarna")("jobs.deel.com") and deel.allowed("klarna")("JOBS.DEEL.COM") for host in ("deel.com", "api.deel.com", "jobs.deel.com.evil.example", "evil.example"): assert not deel.allowed("klarna")(host) @@ -159,7 +151,7 @@ def test_no_slug_reaches_another_host() -> None: """The host is fixed in every URL the module builds; a slug only ever lands in the path.""" for slug in ("klarna", "Klarna", "a-b_c"): - for url in (deel.careers_url(slug), deel.feed_url(slug), deel.robots_page(slug)): + for url in (deel.careers_url(slug), deel.feed_url(slug, Ask())): assert httpx.URL(url).host == "jobs.deel.com" diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py index 093a5d6..ed5e5f0 100644 --- a/tests/test_ats_eightfold.py +++ b/tests/test_ats_eightfold.py @@ -18,7 +18,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import eightfold # noqa: E402 +from plugins.readers import Ask, eightfold # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SLUG = "jobs.ericsson.com/ericsson.com" @@ -47,6 +47,26 @@ def _search(total: int, seen: list[str] | None = None) -> httpx.MockTransport: return httpx.MockTransport(handler) +def _crawl(slug: str, transport: httpx.MockTransport) -> tuple[list[dict[str, Any]], bool]: + """What the runner does with a paged reader: pages until the cursor ends, + complete when as many distinct ids were read as the host stated.""" + rows: list[dict[str, Any]] = [] + ids: set[str] = set() + total: Any = None + with httpx.Client(transport=transport) as client: + def get(url: str, **_: Any) -> bytes: + return client.get(url).content + + cursor: Any = None + while True: + decoded, results, total, cursor = eightfold.page(get, slug, cursor, Ask()) + rows += eightfold.rows(decoded, slug) + ids |= {r["id"] for r in results} + if cursor is None or (total is not None and len(ids) >= total): + break + return rows, total is not None and len(ids) >= total + + def test_a_page_becomes_rows() -> None: rows = eightfold.rows(_saved()["data"]["positions"], SLUG) assert len(rows) == 5 @@ -56,11 +76,14 @@ def test_a_page_becomes_rows() -> None: assert first["url"] == "https://jobs.ericsson.com/careers/job/563121777024688" assert first["location"] == "Stockholm, Sweden" # the city is its own region assert first["department"] == "76 Product Development" - assert first["posted"] == datetime.fromtimestamp(1790703782, tz=UTC) + # `creationTs`, the posting date; `postedTs` (1790703782) is refreshed in bulk. + assert first["posted"] == datetime.fromtimestamp(1785822052, tz=UTC) + assert first["places"] == [{"text": "Stockholm, Sweden", "remote": None}] assert first["remote"] == "onsite" and (first["description"], first["salary_text"]) == ("", "") assert mexico["location"] == "Mexico City, Mexico" assert many["location"] == ("Tecnoparque, Mexico City, Mexico; Santa Fe, Mexico City, Mexico; " "Mexico City, Mexico; Tlalnepantla, Estado de México, Mexico") + assert [p["text"] for p in many["places"]] == many["location"].split("; ") @pytest.mark.parametrize(("option", "remote"), [ @@ -89,36 +112,23 @@ def test_a_page_asks_for_the_newest_first_from_its_offset() -> None: asked.append(url) return json.dumps(_saved()).encode() - decoded, results, total, cursor = eightfold.page(get, SLUG, None) + decoded, results, total, cursor = eightfold.page(get, SLUG, None, Ask()) assert asked == ["https://jobs.ericsson.com/api/pcsx/search?domain=ericsson.com" "&sort_by=timestamp&start=0"] assert (len(decoded), len(results), total, cursor) == (5, 5, 637, 5) - assert eightfold.page(get, SLUG, 630)[3] == 635 + assert eightfold.page(get, SLUG, 630, Ask())[3] == 635 assert asked[-1].endswith("&start=630") -def test_a_board_is_read_ten_at_a_time_to_its_total(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(ats, "PAGE_DELAY", 0) +def test_a_board_is_read_ten_at_a_time_to_its_total() -> None: seen: list[str] = [] - with httpx.Client(transport=_search(25, seen)) as client: - rows, complete = ats._fetch_rows(client, f"eightfold:{SLUG}") + rows, complete = _crawl(SLUG, _search(25, seen)) assert (len(rows), complete) == (25, True) assert [u.rsplit("=", 1)[1] for u in seen] == ["0", "10", "20"] -def test_a_board_over_the_page_cap_is_incomplete(monkeypatch: pytest.MonkeyPatch) -> None: - """The cap is 100 pages of ten: a board of 1,292 (Infineon's) or 2,319 - (Microsoft's) is read as its first thousand and never as the whole.""" - monkeypatch.setattr(ats, "PAGE_DELAY", 0) - seen: list[str] = [] - with httpx.Client(transport=_search(1292, seen)) as client: - rows, complete = ats._fetch_rows(client, "eightfold:infineon.eightfold.ai/infineon.com") - assert (len(rows), complete) == (1000, False) and len(seen) == ats.MAX_PAGES == 100 - - -def test_an_empty_board_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None: - with httpx.Client(transport=_search(0)) as client: - assert ats._fetch_rows(client, f"eightfold:{SLUG}") == ([], True) +def test_an_empty_board_is_complete_at_zero() -> None: + assert _crawl(SLUG, _search(0)) == ([], True) @pytest.mark.parametrize("body", [ @@ -126,7 +136,7 @@ def test_an_empty_board_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> b'{"data": {"positions": null}}']) def test_an_answer_that_is_not_a_search_raises(body: bytes) -> None: with pytest.raises(ValueError): - eightfold.page(lambda url, **_: body, SLUG, None) + eightfold.page(lambda url, **_: body, SLUG, None, Ask()) def test_detail_reads_the_text_of_the_ad() -> None: @@ -164,7 +174,6 @@ def test_the_slug_the_urls_and_the_allow_list() -> None: for slug in (SLUG, "infineon.eightfold.ai/infineon.com", "Apply.Careers.Microsoft.com/microsoft.com"): assert eightfold.slug_ok(slug) and ats.is_known_target(f"eightfold:{slug}") assert eightfold.careers_url("Jobs.Ericsson.com/ericsson.com") == "https://jobs.ericsson.com/careers" - assert eightfold.robots_page(SLUG) == "https://jobs.ericsson.com/careers" allow = eightfold.allowed("Jobs.Ericsson.com/ericsson.com") assert allow("jobs.ericsson.com") and allow("JOBS.ERICSSON.COM") assert not allow("ericsson.com") and not allow("evil.example") @@ -193,3 +202,12 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found assert ("eightfold", "") in found assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} + + +def test_it_declares_a_paged_read_of_ten_with_a_total_and_no_filter_yet() -> None: + reads = eightfold.READS + assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == ( + "paged", 10, "detail", "stated", "many") + assert reads.filters.declared() == {} and reads.fields.posted == "listed" + assert reads.robots.startswith("read: robots.txt allows /careers") + assert not hasattr(eightfold, "robots_page") diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py index 99cd1fe..eeed4e5 100644 --- a/tests/test_ats_gem.py +++ b/tests/test_ats_gem.py @@ -3,13 +3,11 @@ answers. No network.""" from __future__ import annotations -import json import sys from datetime import UTC, datetime from pathlib import Path from typing import Any -import httpx import pytest ROOT = Path(__file__).resolve().parents[1] @@ -18,7 +16,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import gem # noqa: E402 +from plugins.readers import Ask, gem # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" API = "https://jobs.gem.com/api/public/graphql" @@ -47,7 +45,7 @@ def _listing() -> dict[str, bytes | Exception]: def test_a_listing_becomes_rows() -> None: asked: list[dict[str, Any]] = [] - decoded, results, total, cursor = gem.page(_get_of(_listing(), asked), "productboard", None) + decoded, results, total, cursor = gem.page(_get_of(_listing(), asked), "productboard", None, Ask()) assert asked == [{"operationName": "JobBoardList", "variables": {"boardId": "productboard"}, "query": gem.LIST_QUERY, "headers": None}] assert (total, cursor) == (None, None) and len(results) == 11 @@ -62,6 +60,11 @@ def test_a_listing_becomes_rows() -> None: assert (first["posted"], first["description"], first["salary_text"]) == (None, "", "") assert backend["source_id"] == "am9icG9zdDqWC1V_RM7U6ORvTd1Iu64r" # opaque assert backend["location"] == "Prague; Remote Europe; Brno" + # Every place is kept, and the host's own flag says which is remote. + assert backend["places"] == [{"text": "Prague", "remote": False}, + {"text": "Remote Europe", "remote": True}, + {"text": "Brno", "remote": False}] + assert first["employment"] == "full_time" assert len({r["source_id"] for r in rows}) == 11 @@ -80,28 +83,28 @@ def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None: assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == [] -def test_the_board_is_read_through_the_app_without_origin_or_referer() -> None: - seen: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - seen.append(request) - return httpx.Response(200, content=_read("gem.json")) - - with httpx.Client(transport=httpx.MockTransport(handler)) as client: - rows, complete = ats._fetch_rows(client, "gem:productboard") - assert len(rows) == 11 and complete - (request,) = seen - assert str(request.url) == API and request.method == "POST" - assert json.loads(request.content)["variables"] == {"boardId": "productboard"} - assert "origin" not in request.headers and "referer" not in request.headers - assert request.headers["content-type"] == "application/json" +def test_the_board_is_one_post_without_origin_or_referer() -> None: + asked: list[dict[str, Any]] = [] + decoded, _results, total, cursor = gem.page(_get_of(_listing(), asked), "productboard", + None, Ask()) + (call,) = asked + assert call["variables"] == {"boardId": "productboard"} and call["headers"] is None + assert len(gem.rows(decoded, "productboard")) == 11 and (total, cursor) == (None, None) -def test_an_empty_board_is_complete_at_zero() -> None: +def test_an_empty_board_is_no_rows() -> None: body = b'{"data":{"oatsExternalJobPostings":{"jobPostings":[]}}}' - with httpx.Client(transport=httpx.MockTransport( - lambda request: httpx.Response(200, content=body))) as client: - assert ats._fetch_rows(client, "gem:nobody") == ([], True) + decoded, results, _total, cursor = gem.page( + _get_of({"JobBoardList": body}, []), "nobody", None, Ask()) + assert gem.rows(decoded, "nobody") == [] and results == [] and cursor is None + + +def test_it_declares_one_page_that_may_be_empty_by_mistake() -> None: + reads = gem.READS + assert (reads.listing, reads.text, reads.places, reads.empty) == ( + "paged", "detail", "many", "suspect") + assert reads.robots.startswith("read: robots.txt allows /") + assert not hasattr(gem, "robots_page") @pytest.mark.parametrize(("body", "message"), [ @@ -114,7 +117,7 @@ def test_an_empty_board_is_complete_at_zero() -> None: ]) def test_an_answer_of_another_shape_fails_with_a_clear_error(body: bytes, message: str) -> None: with pytest.raises(ValueError, match=message): - gem.page(_get_of({"JobBoardList": body}, []), "productboard", None) + gem.page(_get_of({"JobBoardList": body}, []), "productboard", None, Ask()) def test_detail_reads_the_text_and_the_date() -> None: @@ -149,7 +152,6 @@ def test_the_slug_the_urls_and_the_allow_list() -> None: for slug in ("productboard", "Productboard", "senndertechnologies-gmbh", "a_b", "1"): assert gem.slug_ok(slug) assert gem.careers_url("Productboard") == "https://jobs.gem.com/Productboard" - assert gem.robots_page("productboard") == "https://jobs.gem.com/productboard" allow = gem.allowed("productboard") assert allow("jobs.gem.com") and allow("JOBS.gem.com") assert not allow("gem.com") and not allow("productboard.gem.com") diff --git a/tests/test_ats_greenhouse.py b/tests/test_ats_greenhouse.py new file mode 100644 index 0000000..08594de --- /dev/null +++ b/tests/test_ats_greenhouse.py @@ -0,0 +1,52 @@ +"""The Greenhouse host module's places, against a small saved board with +`offices`. No network. The pay and the plain row are in `test_ats.py`.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins.readers import Ask, greenhouse # noqa: E402 + +BOARD = (ROOT / "tests" / "fixtures" / "ats" / "greenhouse-places.json").read_bytes() + + +def test_a_placeholder_location_is_replaced_by_the_offices() -> None: + hybrid = greenhouse.parse(BOARD, "acme")[0] + assert hybrid["places"] == [ + {"text": "Austin, TX, United States", "remote": None}, + {"text": "New York, New York, United States", "remote": None}] + assert hybrid["location"] == "Austin, TX, United States; New York, New York, United States" + + +def test_a_name_the_offices_do_not_contain_is_one_more_place() -> None: + remote = greenhouse.parse(BOARD, "acme")[1] + assert [p["text"] for p in remote["places"]] == ["Remote India", "Bangalore, India"] + assert remote["places"][0]["remote"] is True + + +def test_a_name_an_office_repeats_is_one_place() -> None: + lisbon = greenhouse.parse(BOARD, "acme")[2] + assert lisbon["places"] == [{"text": "Lisbon, Portugal", "remote": None}] + + +def test_without_offices_the_name_is_the_place() -> None: + body = b'{"jobs": [{"id": 1, "title": "T", "absolute_url": "https://gh.example/1", ' \ + b'"location": {"name": "Hybrid"}}, {"id": 2, "title": "U", ' \ + b'"absolute_url": "https://gh.example/2", "location": {"name": ""}}]}' + hybrid, nothing = greenhouse.parse(body, "acme") + assert hybrid["places"] == [{"text": "Hybrid", "remote": None}] + assert nothing["places"] == [] and nothing["location"] == "" + + +def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None: + reads = greenhouse.READS + assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many") + assert reads.fields.posted == "trusted" and reads.fields.pay == "figures" + assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30" + assert greenhouse.feed_url("acme", Ask()).endswith("/boards/acme/jobs?content=true&pay_transparency=true") + assert greenhouse.fallback_url("acme").endswith("/boards/acme/jobs") diff --git a/tests/test_ats_hibob.py b/tests/test_ats_hibob.py index 9df2125..02d03c2 100644 --- a/tests/test_ats_hibob.py +++ b/tests/test_ats_hibob.py @@ -17,7 +17,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import hibob # noqa: E402 +from plugins.readers import Ask, hibob # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "hibob.json").read_bytes() @@ -104,7 +104,7 @@ def test_the_page_sends_the_company_header_and_states_no_total() -> None: seen.append((url, headers)) return BOARD - data, results, total, cursor = hibob.page(get, "Liquidia", None) + data, results, total, cursor = hibob.page(get, "Liquidia", None, Ask()) assert seen == [(API, {"companyIdentifier": "liquidia", "Accept": "application/json"})] assert len(results) == 5 and all(r["id"] for r in results) assert total is None and cursor is None @@ -113,59 +113,58 @@ def test_the_page_sends_the_company_header_and_states_no_total() -> None: def test_a_body_without_the_ads_is_not_a_board() -> None: with pytest.raises(ValueError, match="not a HiBob board"): - hibob.page(lambda url, **kw: b'{"error": "unauthorized"}', "acme", None) + hibob.page(lambda url, **kw: b'{"error": "unauthorized"}', "acme", None, Ask()) -@pytest.fixture -def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: - answers: dict[str, Any] = {} +def test_the_board_is_one_request_with_the_company_header() -> None: seen: list[tuple[str, dict[str, str] | None]] = [] - def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, - www_twin=False): + def get(url: str, *, method: str = "GET", payload: Any = None, + headers: dict[str, str] | None = None) -> bytes: seen.append((url, headers)) - if not ats._allowed_host(host, slug)(httpx.URL(url).host): - raise ats._Refused(url) - answer = answers[url] - if isinstance(answer, BaseException): - raise answer - return answer + return BOARD - monkeypatch.setattr(ats, "_get", fake_get) - return {"answers": answers, "seen": seen} + data, _results, _total, cursor = hibob.page(get, "liquidia", None, Ask()) + assert len(hibob.rows(data, "liquidia")) == 5 and cursor is None + assert [url for url, _ in seen] == [API] + assert seen[0][1] is not None and seen[0][1]["companyIdentifier"] == "liquidia" -def test_the_board_is_one_request_and_complete(web: dict[str, Any]) -> None: - web["answers"][API] = BOARD - with httpx.Client() as client: - rows, complete = ats._fetch_rows(client, "hibob:liquidia") - assert len(rows) == 5 and complete - assert [url for url, _ in web["seen"]] == [API] - assert web["seen"][0][1]["companyIdentifier"] == "liquidia" - - -def test_an_empty_board_is_complete_and_a_401_is_a_failure(web: dict[str, Any]) -> None: - web["answers"]["https://acme.careers.hibob.com/api/job-ad"] = b'{"jobAdDetails": []}' +def test_an_empty_board_is_no_rows_and_a_401_reaches_the_caller() -> None: + data, results, _total, _cursor = hibob.page( + lambda url, **kw: b'{"jobAdDetails": []}', "acme", None, Ask()) + assert hibob.rows(data, "acme") == [] and results == [] request = httpx.Request("GET", API) - web["answers"][API] = httpx.HTTPStatusError( - "401", request=request, response=httpx.Response(401, request=request)) - with httpx.Client() as client: - assert ats._fetch_rows(client, "hibob:acme") == ([], True) - with pytest.raises(httpx.HTTPStatusError): - ats._fetch_rows(client, "hibob:liquidia") + + def refused(url: str, **kw: Any) -> bytes: + raise httpx.HTTPStatusError("401", request=request, + response=httpx.Response(401, request=request)) + + with pytest.raises(httpx.HTTPStatusError): + hibob.page(refused, "liquidia", None, Ask()) -def test_the_robots_txt_that_is_the_apps_own_page_allows(web: dict[str, Any]) -> None: - """The host answers /robots.txt with its single-page app's HTML, a 200 - with no rules.""" - web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \ - b'Careers' - with httpx.Client() as client: - ats._check_robots(client, "hibob:dixa") - web["answers"]["https://dixa.careers.hibob.com/robots.txt"] = \ - b"User-agent: *\nDisallow: /\n" - with pytest.raises(ats._Disallowed): - ats._check_robots(client, "hibob:dixa") +def test_each_ad_names_one_site_and_the_host_lists_a_copy_per_site() -> None: + dixa = hibob.parse(DIXA, "dixa") + assert [r["places"] for r in dixa] == [ + [{"text": "Copenhagen, Denmark", "remote": False}], + [{"text": "London, United Kingdom", "remote": False}]] + assert {r["title"] for r in dixa} == {"Expression of interest"} + assert len({r["source_id"] for r in dixa}) == 2 + + +def test_the_employment_type_is_mapped_and_a_permanent_one_is_left_unmapped() -> None: + rows = hibob.parse(BOARD, "liquidia") + assert rows[0]["employment"] == "full_time" # "Regular Full Time" + assert "employment" not in hibob.parse(DIXA, "dixa")[0] # "Permanent" + + +def test_it_declares_copies_with_a_page_and_a_robots_decision() -> None: + reads = hibob.READS + assert (reads.listing, reads.text, reads.places, reads.complete) == ( + "paged", "listing", "copies", "end") + assert reads.robots.startswith("read: robots.txt allows") + assert not hasattr(hibob, "robots_page") # ------------------------------------------------------- slugs and hosts @@ -178,7 +177,6 @@ def test_the_module_is_registered_as_a_paged_host() -> None: def test_the_urls_and_the_allowed_host_follow_the_slug() -> None: assert hibob.careers_url("dixa") == "https://dixa.careers.hibob.com/" - assert hibob.robots_page("dixa") == "https://dixa.careers.hibob.com/" allow = hibob.allowed("Dixa") assert allow("dixa.careers.hibob.com") and allow("DIXA.CAREERS.HIBOB.COM") for host in ("careers.hibob.com", "other.careers.hibob.com", "dixa.hibob.com", diff --git a/tests/test_ats_hrmanager.py b/tests/test_ats_hrmanager.py index 99b38ff..4b57f93 100644 --- a/tests/test_ats_hrmanager.py +++ b/tests/test_ats_hrmanager.py @@ -16,7 +16,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import hrmanager # noqa: E402 +from plugins.readers import Ask, hrmanager # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "hrmanager.xml").read_bytes() @@ -73,83 +73,58 @@ def _paged(answers: dict[str, bytes], asked: list[str]) -> Any: def test_the_page_moves_on_by_what_it_returned_and_ends_on_an_empty_one() -> None: asked: list[str] = [] get = _paged({_feed(0): BOARD, _feed(5): EMPTY}, asked) - data, results, total, cursor = hrmanager.page(get, "stsgroup_tr", None) + data, results, total, cursor = hrmanager.page(get, "stsgroup_tr", None, Ask()) assert len(results) == 5 and total is None and cursor == 5 assert len(hrmanager.rows(data, "stsgroup_tr")) == 5 - _, results, total, cursor = hrmanager.page(get, "stsgroup_tr", cursor) + _, results, total, cursor = hrmanager.page(get, "stsgroup_tr", cursor, Ask()) assert results == [] and total is None and cursor is None assert asked == [_feed(0), _feed(5)] -@pytest.fixture -def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: - answers: dict[str, Any] = {} +def _crawl(get: Any, slug: str) -> list[dict[str, Any]]: + """Pages until the cursor ends, as the runner does, giving the rows.""" + out: list[dict[str, Any]] = [] + cursor: Any = None + while True: + data, _results, _total, cursor = hrmanager.page(get, slug, cursor, Ask()) + out += hrmanager.rows(data, slug) + if cursor is None: + return out + + +def test_a_board_is_read_to_an_empty_page() -> None: asked: list[str] = [] - - def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, - www_twin=False): - asked.append(url) - if not ats._allowed_host(host, slug)(httpx.URL(url).host): - raise ats._Refused(url) - answer = answers[url] - if isinstance(answer, BaseException): - raise answer - return answer - - monkeypatch.setattr(ats, "_get", fake_get) - monkeypatch.setattr(ats, "PAGE_DELAY", 0) - return {"answers": answers, "asked": asked} + rows = _crawl(_paged({_feed(0): BOARD, _feed(5): EMPTY}, asked), "stsgroup_tr") + assert len(rows) == 5 and asked == [_feed(0), _feed(5)] -def test_a_board_is_complete_when_the_read_ends_on_an_empty_page(web: dict[str, Any]) -> None: - web["answers"] |= {_feed(0): BOARD, _feed(5): EMPTY} - with httpx.Client() as client: - rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr") - assert len(rows) == 5 and complete - assert web["asked"] == [_feed(0), _feed(5)] +def test_a_board_with_no_roles_is_empty() -> None: + asked: list[str] = [] + assert _crawl(_paged({_feed(0, "seaborg_tr"): EMPTY}, asked), "seaborg_tr") == [] + assert asked == [_feed(0, "seaborg_tr")] -def test_a_board_with_no_roles_is_complete_and_empty(web: dict[str, Any]) -> None: - web["answers"][_feed(0, "seaborg_tr")] = EMPTY - with httpx.Client() as client: - assert ats._fetch_rows(client, "hrmanager:seaborg_tr") == ([], True) - assert web["asked"] == [_feed(0, "seaborg_tr")] - - -def test_a_page_the_host_repeats_leaves_the_board_incomplete(web: dict[str, Any]) -> None: - """A host that ignored `skip` would answer the same roles again.""" - web["answers"] |= {_feed(0): BOARD, _feed(5): BOARD} - with httpx.Client() as client: - rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr") - assert len(rows) == 5 and not complete - - -def test_a_429_after_the_first_page_ends_the_read_incomplete(web: dict[str, Any]) -> None: +def test_a_429_after_the_first_page_reaches_the_caller() -> None: request = httpx.Request("GET", _feed(5)) - web["answers"] |= {_feed(0): BOARD, _feed(5): httpx.HTTPStatusError( - "429", request=request, response=httpx.Response(429, request=request))} - with httpx.Client() as client: - rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr") - assert len(rows) == 5 and not complete + + def get(url: str, **kwargs: Any) -> bytes: + if url == _feed(5): + raise httpx.HTTPStatusError("429", request=request, + response=httpx.Response(429, request=request)) + return BOARD + + with pytest.raises(httpx.HTTPStatusError): + _crawl(get, "stsgroup_tr") -def test_robots_are_read_from_the_api_host_whose_404_allows(web: dict[str, Any]) -> None: - request = httpx.Request("GET", "https://api.hr-manager.net/robots.txt") - web["answers"]["https://api.hr-manager.net/robots.txt"] = httpx.HTTPStatusError( - "404", request=request, response=httpx.Response(404, request=request)) - state: dict[str, str] = {} - with httpx.Client() as client: - ats._check_robots(client, "hrmanager:stsgroup_tr", state) - assert '"allowed": true' in state["ats:robots:hrmanager:stsgroup_tr"] - assert web["asked"] == ["https://api.hr-manager.net/robots.txt"] - - -def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide( - web: dict[str, Any]) -> None: - web["answers"]["https://api.hr-manager.net/robots.txt"] = \ - b"User-agent: *\nDisallow: /JobPortal.svc/\n" - with httpx.Client() as client, pytest.raises(ats._Disallowed): - ats._check_robots(client, "hrmanager:stsgroup_tr") +def test_the_feed_states_no_place_and_the_robots_decision_is_the_owners() -> None: + rows = hrmanager.parse(BOARD, "stsgroup_tr") + assert all(r["places"] == [] for r in rows) + reads = hrmanager.READS + assert (reads.listing, reads.page_size, reads.text, reads.places) == ( + "paged", 100, "listing", "one") + assert reads.robots == "disallowed by robots.txt; owner to decide" + assert not hasattr(hrmanager, "robots_page") # ------------------------------------------------------- slugs and hosts @@ -163,7 +138,6 @@ def test_the_module_is_registered_as_a_paged_host() -> None: def test_the_urls_and_the_allowed_host() -> None: assert hrmanager.careers_url("seaborg_tr") == \ "https://candidate.hr-manager.net/vacancies/list.aspx?customer=seaborg_tr" - assert hrmanager.robots_page("stsgroup_tr") == _feed(0) allow = hrmanager.allowed("stsgroup_tr") assert allow("api.hr-manager.net") and allow("API.HR-MANAGER.NET") for host in ("candidate.hr-manager.net", "hr-manager.net", "api.hr-manager.net.evil.example", diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py index 27d0863..4bb7999 100644 --- a/tests/test_ats_jibe.py +++ b/tests/test_ats_jibe.py @@ -18,7 +18,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import jibe # noqa: E402 +from plugins.readers import Ask, jibe # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SITE = "jobs.booking.com" @@ -52,6 +52,12 @@ def test_a_page_becomes_rows() -> None: assert first["posted"] == datetime(2026, 9, 29, 12, 55, tzinfo=UTC) assert (first["remote"], first["salary_text"]) == (None, "") assert several["location"] == "Amsterdam, Netherlands; Manchester, United Kingdom" + # Every place the site lists is kept; a country only for a role at one place. + assert [p["text"] for p in several["places"]] == ["Amsterdam, Netherlands", + "Manchester, United Kingdom"] + assert "country" not in several + assert first["places"] == [{"text": "Bucharest, Romania", "remote": None}] + assert (first["country"], first["employment"]) == ("RO", "full_time") assert all(r["description"] and "<" not in r["description"] and " " not in r["description"] for r in rows) @@ -79,56 +85,57 @@ def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None: asked.append(url) return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode() - decoded, results, total, cursor = jibe.page(get, SITE, None) + decoded, results, total, cursor = jibe.page(get, SITE, None, Ask()) assert asked == ["https://jobs.booking.com/api/jobs?limit=100&page=1"] assert (len(results), total, cursor) == (100, 145, 2) assert len(jibe.rows(decoded, SITE)) == 100 - assert jibe.page(get, SITE, 2)[2:] == (145, 3) + assert jibe.page(get, SITE, 2, Ask())[2:] == (145, 3) assert asked[-1].endswith("&page=2") -def test_the_board_is_read_to_its_total_through_the_app(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(ats, "PAGE_DELAY", 0) - board = _board(145) +def _crawl(board: dict[int, dict[str, Any]]) -> tuple[list[dict[str, Any]], bool, list[str]]: + """Pages as the runner does: on to the next number until the cursor ends + or as many distinct ids were read as the total; complete in the latter.""" seen: list[str] = [] - def handler(request: httpx.Request) -> httpx.Response: - seen.append(str(request.url)) - return httpx.Response(200, json=board[int(request.url.params["page"])]) + def get(url: str, **_: Any) -> bytes: + seen.append(url) + return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode() - with httpx.Client(transport=httpx.MockTransport(handler)) as client: - rows, complete = ats._fetch_rows(client, f"jibe:{SITE}") + rows: list[dict[str, Any]] = [] + ids: set[str] = set() + cursor: Any = None + while True: + decoded, results, total, cursor = jibe.page(get, SITE, cursor, Ask()) + rows += jibe.rows(decoded, SITE) + ids |= {r["id"] for r in results} + if cursor is None or (total is not None and len(ids) >= total): + return rows, total is not None and len(ids) >= total, seen + + +def test_the_board_is_read_to_its_total() -> None: + rows, complete, seen = _crawl(_board(145)) assert (len(rows), complete) == (145, True) assert seen == [f"https://jobs.booking.com/api/jobs?limit=100&page={n}" for n in (1, 2)] -def test_a_board_that_ends_short_of_its_total_is_incomplete( - monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(ats, "PAGE_DELAY", 0) +def test_a_board_that_ends_short_of_its_total_is_incomplete() -> None: board = _board(145) board[2]["totalCount"] = board[1]["totalCount"] = 150 # five roles the site never sends board[3] = {"jobs": [], "totalCount": 150} - - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json=board[int(request.url.params["page"])]) - - with httpx.Client(transport=httpx.MockTransport(handler)) as client: - rows, complete = ats._fetch_rows(client, f"jibe:{SITE}") + rows, complete, _seen = _crawl(board) assert (len(rows), complete) == (145, False) def test_an_empty_board_is_complete_at_zero() -> None: - body = {"jobs": [], "totalCount": 0} - with httpx.Client(transport=httpx.MockTransport( - lambda request: httpx.Response(200, json=body))) as client: - assert ats._fetch_rows(client, f"jibe:{SITE}") == ([], True) + assert _crawl({1: {"jobs": [], "totalCount": 0}})[:2] == ([], True) @pytest.mark.parametrize("body", [b"Not found", b"[]", b'{"jobs": null}', b'{"error": "no"}']) def test_an_answer_that_is_not_a_listing_raises(body: bytes) -> None: with pytest.raises(ValueError): - jibe.page(lambda url, **_: body, SITE, None) + jibe.page(lambda url, **_: body, SITE, None, Ask()) def test_a_422_and_a_429_reach_the_shared_code_as_status_errors() -> None: @@ -140,14 +147,13 @@ def test_a_422_and_a_429_reach_the_shared_code_as_status_errors() -> None: response=httpx.Response(422, request=request)) with pytest.raises(httpx.HTTPStatusError): - jibe.page(get, SITE, None) + jibe.page(get, SITE, None, Ask()) def test_the_slug_the_urls_and_the_allow_list() -> None: for slug in (SITE, "Jobs.Booking.com", "careers.acme.co.uk", "acme.com", "a-b.c-d.example"): assert jibe.slug_ok(slug) and ats.is_known_target(f"jibe:{slug}") assert jibe.careers_url("Jobs.Booking.com") == "https://jobs.booking.com/jobs" - assert jibe.robots_page(SITE) == "https://jobs.booking.com/jobs" allow = jibe.allowed("Jobs.Booking.com") assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM") assert not allow("booking.com") and not allow("evil.example") @@ -171,3 +177,11 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None: 'apply') assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")] assert "jibe" not in {h for h, _ in discover._UNREADABLE} + + +def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None: + reads = jibe.READS + assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == ( + "paged", 100, "listing", "stated", "many") + assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt") + assert not hasattr(jibe, "robots_page") diff --git a/tests/test_ats_jobylon.py b/tests/test_ats_jobylon.py index 5a9b4e0..6038de1 100644 --- a/tests/test_ats_jobylon.py +++ b/tests/test_ats_jobylon.py @@ -8,16 +8,16 @@ import sys from pathlib import Path from typing import Any -import httpx import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app import models as app_models # noqa: E402 from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import jobylon # noqa: E402 +from plugins.readers import Ask, jobylon # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "jobylon.html").read_bytes() @@ -34,6 +34,7 @@ def test_rows_come_from_the_embeds_literal() -> None: assert first["title"] == "Financial Controller" assert first["url"] == ROLE assert first["location"] == "Stockholm" + assert first["places"] == [{"text": "Stockholm", "remote": None}] assert first["department"] == "Accounting & Finance" # `&` read as `&` assert first["remote"] == "hybrid" assert first["description"] == "" and first["salary_text"] == "" @@ -45,6 +46,7 @@ def test_rows_come_from_the_embeds_literal() -> None: def test_places_department_and_remote_follow_the_embeds_words() -> None: rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")} assert rows["384238"]["location"] == "Houston; Dallas; Atlanta" + assert [p["text"] for p in rows["384238"]["places"]] == ["Houston", "Dallas", "Atlanta"] assert rows["384238"]["department"] == "" # "All functions" names none assert rows["382124"]["remote"] == "remote" assert rows["382860"]["remote"] == "onsite" @@ -90,7 +92,8 @@ def test_detail_reads_the_role_pages_json_ld() -> None: ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()} first = jobylon.detail(_get(pages, asked), "3143", rows["385191"]) assert asked == [ROLE] - assert set(first) <= set(ats.DETAIL_FIELDS) + assert set(first) <= set(ats.DETAIL_FIELDS) | {"places"} + assert first["places"] == [{"text": "Stockholm, Sweden", "remote": None}] assert first["description"].startswith("Want to be part of transforming road freight?") assert "<" not in first["description"] and "world's leading" in first["description"] assert first["posted"] is not None and first["posted"].isoformat() == \ @@ -99,6 +102,7 @@ def test_detail_reads_the_role_pages_json_ld() -> None: # All four places, where the embed named three. second = jobylon.detail(_get(pages), "3143", rows["384238"]) assert second["location"] == "Houston, TX, USA; Dallas, TX, USA; New Jersey, USA; Atlanta, GA, USA" + assert len(second["places"]) == 4 def test_detail_without_a_job_posting_fails_the_read() -> None: @@ -111,48 +115,48 @@ def test_detail_without_a_job_posting_fails_the_read() -> None: jobylon.detail(_get({ROLE: page}), "3143", row) -@pytest.fixture -def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: - answers: dict[str, Any] = {} +def test_the_board_is_one_request_and_each_role_is_read_on_its_own_page() -> None: asked: list[str] = [] - - def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, - www_twin=False): - asked.append(url) - if not ats._allowed_host(host, slug)(httpx.URL(url).host): - raise ats._Refused(url) - answer = answers[url] - if isinstance(answer, BaseException): - raise answer - return answer - - monkeypatch.setattr(ats, "_get", fake_get) - monkeypatch.setattr(ats, "PAGE_DELAY", 0) - return {"answers": answers, "asked": asked} + pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(), + ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()} + get = _get(pages, asked) + rows = jobylon.parse(get(jobylon.feed_url("3143", Ask())), "3143") + assert len(rows) == 5 and asked == [FEED] + found = jobylon.detail(get, "3143", rows[0]) + assert asked == [FEED, ROLE] and found["description"].startswith("Want to be part") -def test_the_board_is_one_request_and_each_role_is_read_once(web: dict[str, Any]) -> None: - web["answers"] |= {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(), - ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()} - with httpx.Client() as client: - rows, complete = ats._fetch_rows(client, "jobylon:3143") - assert len(rows) == 5 and complete and web["asked"] == [FEED] - # The three roles whose pages are not saved answer 404, which counts as read. - for row in rows: - web["answers"].setdefault(row["url"], httpx.HTTPStatusError( - "404", request=httpx.Request("GET", row["url"]), - response=httpx.Response(404, request=httpx.Request("GET", row["url"])))) - filled, done, failed = ats._read_details(client, "jobylon:3143", rows, complete, - set(), set()) - assert done == {f"jobylon:3143:{r['source_id']}" for r in rows} and not failed - assert filled[0]["description"].startswith("Want to be part") and filled[0]["location"] - assert filled[2]["description"] == "" +def test_an_empty_board_is_no_roles() -> None: + assert jobylon.parse(EMPTY, "1056") == [] -def test_an_empty_board_is_complete(web: dict[str, Any]) -> None: - web["answers"]["https://cdn.jobylon.com/jobs/companies/1056/embed/v2/"] = EMPTY - with httpx.Client() as client: - assert ats._fetch_rows(client, "jobylon:1056") == ([], True) +def test_the_experience_is_mapped_and_a_word_the_table_lacks_is_logged( + caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level("DEBUG", logger="plugins.readers.jobylon") + rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")} + assert rows["384238"]["seniority"] == "mid" # "Mid level" + assert "seniority" not in rows["385191"] # "Experienced" says too little + assert "unknown experience 'Experienced'" in caplog.text + assert rows["385191"]["employment"] == "full_time" + assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values()) + rows = {r["source_id"]: r for r in jobylon.parse( + (FIXTURES / "jobylon-places.html").read_bytes(), "3143")} + assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "intern" + + +def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None: + (row,) = jobylon.parse((FIXTURES / "jobylon-teaser.html").read_bytes(), "3143") + assert row["teaser"] == "Lead our Houston depot and its 40 drivers." + assert all("teaser" not in r for r in jobylon.parse(BOARD, "3143")) + + +def test_it_declares_a_whole_listing_with_a_teaser_and_detail() -> None: + reads = jobylon.READS + assert (reads.listing, reads.text, reads.teaser, reads.places) == ( + "whole", "detail", True, "many") + assert (reads.fields.posted, reads.fields.seniority) == ("trusted", "stated") + assert reads.robots.startswith("read: robots.txt allows /jobs/companies/") + assert not hasattr(jobylon, "robots_page") # ------------------------------------------------------- slugs and hosts @@ -163,10 +167,9 @@ def test_the_module_is_registered_as_a_feed_host_with_detail() -> None: assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page") -def test_the_urls_and_the_robots_page() -> None: - assert jobylon.feed_url("3143") == FEED +def test_the_urls() -> None: + assert jobylon.feed_url("3143", Ask()) == FEED assert jobylon.careers_url("3143") == "https://emp.jobylon.com/companies/3143/" - assert jobylon.robots_page("3143") == FEED allow = jobylon.allowed("3143") assert allow("cdn.jobylon.com") and allow("emp.jobylon.com") and allow("EMP.JOBYLON.COM") for host in ("jobylon.com", "feed.jobylon.com", "emp.jobylon.com.evil.example", diff --git a/tests/test_ats_lever.py b/tests/test_ats_lever.py new file mode 100644 index 0000000..d8cbe27 --- /dev/null +++ b/tests/test_ats_lever.py @@ -0,0 +1,41 @@ +"""The Lever host module's places, country and employment, against a small +saved board. No network. The pay and the plain row are in `test_ats.py`.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins.readers import Ask, lever # noqa: E402 + +BOARD = (ROOT / "tests" / "fixtures" / "ats" / "lever-places.json").read_bytes() + + +def test_every_place_of_all_locations_is_kept_once() -> None: + several, one = lever.parse(BOARD, "acme") + assert several["places"] == [{"text": "London", "remote": None}, + {"text": "Stockholm", "remote": None}] + assert several["location"] == "London; Stockholm" + assert one["places"] == [{"text": "Berlin", "remote": None}] + + +def test_the_country_is_only_taken_for_a_role_at_one_place() -> None: + several, one = lever.parse(BOARD, "acme") + assert "country" not in several and one["country"] == "DE" + + +def test_the_commitment_is_mapped_and_a_permanent_one_is_left_unmapped() -> None: + several, one = lever.parse(BOARD, "acme") + assert several["employment"] == "full_time" and "employment" not in one + + +def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None: + reads = lever.READS + assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many") + assert reads.fields.posted == "trusted" and reads.fields.country == "stated" + assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30" + assert lever.feed_url("eu/tomtom", Ask()) == "https://api.eu.lever.co/v0/postings/tomtom?mode=json" diff --git a/tests/test_readers_contract.py b/tests/test_readers_contract.py index da2c8ee..e52b543 100644 --- a/tests/test_readers_contract.py +++ b/tests/test_readers_contract.py @@ -50,8 +50,6 @@ FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" #: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and #: C (the search providers, which arrive declared) land. UNDECLARED = frozenset({ - "ashby", "bamboohr", "deel", "eightfold", "gem", "greenhouse", "hibob", "hrmanager", - "jibe", "jobylon", "lever", "oracle", "personio", "phenom", "pinpoint", "recruitee", "rippling", "smartrecruiters", "successfactors", "teamtailor", "workable", "workday", })