From 6c7ce1090db4ede47f1169348cf18b4d7206ef63 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 10:54:19 +0200 Subject: [PATCH 1/4] Readers ashby to lever hand on the host's words; their word tables move to app/prep/hosts Co-Authored-By: Claude Opus 5.5 --- app/prep/hosts/ashby.py | 8 ++++++ app/prep/hosts/bamboohr.py | 12 +++++++++ app/prep/hosts/deel.py | 11 ++++++++ app/prep/hosts/eightfold.py | 5 ++++ app/prep/hosts/gem.py | 10 +++++++ app/prep/hosts/hibob.py | 13 +++++++++ app/prep/hosts/jibe.py | 7 +++++ app/prep/hosts/jobylon.py | 21 +++++++++++++++ app/prep/hosts/lever.py | 9 +++++++ plugins/readers/ashby.py | 23 +++++----------- plugins/readers/bamboohr.py | 27 +++++-------------- plugins/readers/deel.py | 19 ++++---------- plugins/readers/eightfold.py | 6 +---- plugins/readers/gem.py | 16 +++--------- plugins/readers/hibob.py | 17 +++--------- plugins/readers/jibe.py | 13 +++------ plugins/readers/jobylon.py | 26 +++--------------- plugins/readers/lever.py | 16 +++--------- tests/test_ats_ashby.py | 8 +++--- tests/test_ats_bamboohr.py | 14 +++++----- tests/test_ats_deel.py | 10 +++---- tests/test_ats_eightfold.py | 4 +-- tests/test_ats_gem.py | 12 ++++----- tests/test_ats_hibob.py | 8 +++--- tests/test_ats_jibe.py | 2 +- tests/test_ats_jobylon.py | 23 +++++++--------- tests/test_ats_lever.py | 4 +-- tests/test_prep_hosts_ashby.py | 37 ++++++++++++++++++++++++++ tests/test_prep_hosts_bamboohr.py | 37 ++++++++++++++++++++++++++ tests/test_prep_hosts_deel.py | 37 ++++++++++++++++++++++++++ tests/test_prep_hosts_eightfold.py | 32 +++++++++++++++++++++++ tests/test_prep_hosts_gem.py | 37 ++++++++++++++++++++++++++ tests/test_prep_hosts_hibob.py | 37 ++++++++++++++++++++++++++ tests/test_prep_hosts_jibe.py | 32 +++++++++++++++++++++++ tests/test_prep_hosts_jobylon.py | 42 ++++++++++++++++++++++++++++++ tests/test_prep_hosts_lever.py | 32 +++++++++++++++++++++++ tests/test_readers_shared.py | 8 +++++- 37 files changed, 504 insertions(+), 171 deletions(-) create mode 100644 app/prep/hosts/ashby.py create mode 100644 app/prep/hosts/bamboohr.py create mode 100644 app/prep/hosts/deel.py create mode 100644 app/prep/hosts/eightfold.py create mode 100644 app/prep/hosts/gem.py create mode 100644 app/prep/hosts/hibob.py create mode 100644 app/prep/hosts/jibe.py create mode 100644 app/prep/hosts/jobylon.py create mode 100644 app/prep/hosts/lever.py create mode 100644 tests/test_prep_hosts_ashby.py create mode 100644 tests/test_prep_hosts_bamboohr.py create mode 100644 tests/test_prep_hosts_deel.py create mode 100644 tests/test_prep_hosts_eightfold.py create mode 100644 tests/test_prep_hosts_gem.py create mode 100644 tests/test_prep_hosts_hibob.py create mode 100644 tests/test_prep_hosts_jibe.py create mode 100644 tests/test_prep_hosts_jobylon.py create mode 100644 tests/test_prep_hosts_lever.py diff --git a/app/prep/hosts/ashby.py b/app/prep/hosts/ashby.py new file mode 100644 index 0000000..0d0db59 --- /dev/null +++ b/app/prep/hosts/ashby.py @@ -0,0 +1,8 @@ +"""Ashby's words, mapped to the app's.""" + +#: `workplaceType`. +REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"} + +#: `employmentType`. +EMPLOYMENT: dict[str, str] = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship", + "contract": "contract", "temporary": "temporary"} diff --git a/app/prep/hosts/bamboohr.py b/app/prep/hosts/bamboohr.py new file mode 100644 index 0000000..5e1d774 --- /dev/null +++ b/app/prep/hosts/bamboohr.py @@ -0,0 +1,12 @@ +"""BambooHR's words, mapped to the app's.""" + +#: `employmentStatusLabel`. BambooHR's labels are the company's own ("Full-Time", +#: "Part-Time", "Contract", "Intern"). +EMPLOYMENT: dict[str, str] = { + "full-time": "full_time", "full time": "full_time", "part-time": "part_time", + "part time": "part_time", "contract": "contract", "contractor": "contract", + "temporary": "temporary", "intern": "internship", "internship": "internship", + "freelance": "freelance"} + +#: `locationType`. Anything else says nothing. +REMOTE: dict[str, str] = {"0": "onsite", "1": "remote", "2": "hybrid"} diff --git a/app/prep/hosts/deel.py b/app/prep/hosts/deel.py new file mode 100644 index 0000000..8d289c5 --- /dev/null +++ b/app/prep/hosts/deel.py @@ -0,0 +1,11 @@ +"""Deel's words, mapped to the app's.""" + +#: `workArrangementEnum`. +REMOTE: dict[str, str] = {"on_site": "onsite", "hybrid": "hybrid", "remote": "remote"} + +#: The name of a `jobEmploymentTypes` entry. +EMPLOYMENT: dict[str, str] = { + "full-time": "full_time", "full time": "full_time", "part-time": "part_time", + "part time": "part_time", "contract": "contract", "contractor": "contract", + "consultant": "contract", "temporary": "temporary", "intern": "internship", + "internship": "internship", "freelance": "freelance"} diff --git a/app/prep/hosts/eightfold.py b/app/prep/hosts/eightfold.py new file mode 100644 index 0000000..022d816 --- /dev/null +++ b/app/prep/hosts/eightfold.py @@ -0,0 +1,5 @@ +"""Eightfold's words, mapped to the app's.""" + +#: `workLocationOption`. Only "onsite" and "hybrid" were seen; the other +#: spellings are the obvious ones. +REMOTE: dict[str, str] = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"} diff --git a/app/prep/hosts/gem.py b/app/prep/hosts/gem.py new file mode 100644 index 0000000..77f8100 --- /dev/null +++ b/app/prep/hosts/gem.py @@ -0,0 +1,10 @@ +"""Gem's words, mapped to the app's.""" + +#: `job.locationType`. +REMOTE: dict[str, str] = {"in_office": "onsite", "hybrid": "hybrid", "remote": "remote"} + +#: `job.employmentType`. +EMPLOYMENT: dict[str, str] = { + "full_time": "full_time", "part_time": "part_time", "contract": "contract", + "contractor": "contract", "intern": "internship", "internship": "internship", + "temporary": "temporary", "freelance": "freelance"} diff --git a/app/prep/hosts/hibob.py b/app/prep/hosts/hibob.py new file mode 100644 index 0000000..7f24d35 --- /dev/null +++ b/app/prep/hosts/hibob.py @@ -0,0 +1,13 @@ +"""HiBob's words, mapped to the app's.""" + +#: `employmentType`, the company's own name for it. "Permanent" says nothing +#: about hours and stays unmapped. +EMPLOYMENT: dict[str, str] = { + "regular full time": "full_time", "full time": "full_time", + "full-time": "full_time", "regular part time": "part_time", + "part time": "part_time", "part-time": "part_time", "contract": "contract", + "contractor": "contract", "temporary": "temporary", "fixed term": "temporary", + "intern": "internship", "internship": "internship", "freelance": "freelance"} + +#: `workspaceTypeId`. +REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite", "onsite": "onsite"} diff --git a/app/prep/hosts/jibe.py b/app/prep/hosts/jibe.py new file mode 100644 index 0000000..4f24331 --- /dev/null +++ b/app/prep/hosts/jibe.py @@ -0,0 +1,7 @@ +"""Jibe's words, mapped to the app's.""" + +#: `employment_type`. +EMPLOYMENT: dict[str, str] = { + "full_time": "full_time", "part_time": "part_time", "contractor": "contract", + "contract": "contract", "temporary": "temporary", "intern": "internship", + "internship": "internship", "freelance": "freelance"} diff --git a/app/prep/hosts/jobylon.py b/app/prep/hosts/jobylon.py new file mode 100644 index 0000000..d10518d --- /dev/null +++ b/app/prep/hosts/jobylon.py @@ -0,0 +1,21 @@ +"""Jobylon's words, mapped to the app's.""" + +#: The embed's `experience`. Only the coarse levels are mapped; "Experienced" +#: sits between "Mid level" and the managerial levels and says too little to +#: place, so it stays unmapped. +SENIORITY: dict[str, str] = { + "student": "intern", "internship": "intern", "trainee": "intern", + "entry level": "junior", "entry-level": "junior", "junior": "junior", + "graduate": "junior", "mid level": "mid", "mid-level": "mid", "mid": "mid", + "senior": "senior", "lead": "lead", "manager": "manager", "director": "director", + "executive": "exec", "not applicable": ""} + +#: The embed's `employment_type`. +EMPLOYMENT: dict[str, str] = { + "full-time": "full_time", "full time": "full_time", "part-time": "part_time", + "part time": "part_time", "contract": "contract", "consultant": "contract", + "temporary": "temporary", "internship": "internship", "intern": "internship", + "freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"} + +#: `workspace`. +REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"} diff --git a/app/prep/hosts/lever.py b/app/prep/hosts/lever.py new file mode 100644 index 0000000..1195002 --- /dev/null +++ b/app/prep/hosts/lever.py @@ -0,0 +1,9 @@ +"""Lever's words, mapped to the app's.""" + +#: `categories.commitment`. A commitment is free text on Lever ("Permanent" is +#: seen too and says nothing about hours). +EMPLOYMENT: dict[str, str] = { + "full-time": "full_time", "full time": "full_time", "fulltime": "full_time", + "part-time": "part_time", "part time": "part_time", "parttime": "part_time", + "intern": "internship", "internship": "internship", "contract": "contract", + "contractor": "contract", "temporary": "temporary", "freelance": "freelance"} diff --git a/plugins/readers/ashby.py b/plugins/readers/ashby.py index dd70ae6..c60efce 100644 --- a/plugins/readers/ashby.py +++ b/plugins/readers/ashby.py @@ -15,7 +15,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount +from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount log = logging.getLogger(__name__) @@ -36,13 +36,6 @@ READS = Reads( _API = "api.ashbyhq.com" -#: `workplaceType` -> `store._remote`'s words. `isRemote` is true for hybrid -#: roles too, so it is only the fallback when the role has no `workplaceType`. -_WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"} - -#: `employmentType` -> the app's employment words. -_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship", - "contract": "contract", "temporary": "temporary"} def slug_ok(slug: str) -> bool: @@ -113,12 +106,12 @@ def _salary_text(compensation: dict[str, Any]) -> str: or compensation.get("scrapeableCompensationSalarySummary") or "") -def _places(j: dict[str, Any], workplace: str | None) -> list[Place]: +def _places(j: dict[str, Any], workplace: str) -> list[Place]: """The primary `location`, then each of `secondaryLocations`, in the host's words. The primary is remote or an office as `workplaceType` says; a place that names itself remote is remote.""" primary = (j.get("location") or "").strip() - named = [(primary, {"remote": True, "hybrid": False, "onsite": False}.get(workplace or ""))] + named = [(primary, {"remote": True, "hybrid": False, "onsite": False}.get(workplace.lower()))] named += [(str((s or {}).get("location") or "").strip(), None) for s in j.get("secondaryLocations") or []] out: list[Place] = [] @@ -127,10 +120,6 @@ def _places(j: dict[str, Any], workplace: str | None) -> list[Place]: return out -def _employment(value: Any) -> str | None: - return table_word(_EMPLOYMENT, value, log, "employmentType") - - def parse(body: bytes, slug: str) -> list[Row]: data = json.loads(body) rows = [] @@ -142,7 +131,7 @@ def parse(body: bytes, slug: str) -> list[Row]: description = j.get("descriptionHtml") or j.get("descriptionPlain") or "" compensation = j.get("compensation") or {} salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or []) - workplace = _WORKPLACE.get(str(j.get("workplaceType") or "").strip().lower()) + workplace = str(j.get("workplaceType") or "").strip() places = _places(j, workplace) row: Row = { "source_id": str(j.get("id") or ""), @@ -153,13 +142,13 @@ def parse(body: bytes, slug: str) -> list[Row]: "posted": parse_dt(j.get("publishedAt")), "department": (j.get("department") or j.get("team") or "").strip(), "description": strip_html(description), - "remote": workplace or j.get("isRemote"), + "remote": workplace or j.get("isRemote"), # `isRemote` is true for hybrid too "salary_text": _salary_text(compensation), "salary_min": salary_min, "salary_max": salary_max, "currency": currency, } - if employment := _employment(j.get("employmentType")): + if employment := str(j.get("employmentType") or "").strip(): row["employment"] = employment rows.append(row) return rows diff --git a/plugins/readers/bamboohr.py b/plugins/readers/bamboohr.py index 95de9e9..dc9d031 100644 --- a/plugins/readers/bamboohr.py +++ b/plugins/readers/bamboohr.py @@ -23,7 +23,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, label_slug_ok, only, parse_dt, place, table_word +from ._shared import Get, label_slug_ok, only, parse_dt, place log = logging.getLogger(__name__) @@ -34,13 +34,6 @@ READS = Reads( robots="read: robots.txt allows /careers (checked 2026-09-29)", ) -#: `employmentStatusLabel` -> the app's employment words. BambooHR's labels are -#: the company's own ("Full-Time", "Part-Time", "Contract", "Intern"). -_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", - "part time": "part_time", "contract": "contract", "contractor": "contract", - "temporary": "temporary", "intern": "internship", "internship": "internship", - "freelance": "freelance"} - NAME = "bamboohr" LABEL = "BambooHR" PATTERNS = ( @@ -49,9 +42,6 @@ PATTERNS = ( r"https?://(?!(?:app|resources|documentation|help)\.)([a-z0-9-]+)\.bamboohr\.com", ) -#: `locationType` -> `store._remote`'s words. Anything else says nothing. -_REMOTE = {"0": "onsite", "1": "remote", "2": "hybrid"} - #: Sent on the detail read: without it the same URL answers the page's HTML. _JSON = {"Accept": "application/json"} @@ -76,10 +66,11 @@ def feed_url(slug: str, ask: Ask) -> str: return f"https://{host(slug)}/careers/list" -def _remote(j: dict[str, Any]) -> str | None: +def _remote(j: dict[str, Any]) -> bool | str | None: + """`isRemote` when true, else `locationType` as the host writes it ("0", "1", "2").""" if j.get("isRemote") is True: - return "remote" - return _REMOTE.get(str(j.get("locationType") if j.get("locationType") is not None else "")) + return True + return None if j.get("locationType") is None else str(j["locationType"]).strip() or None def _place(j: dict[str, Any]) -> str: @@ -99,10 +90,6 @@ def _places(j: dict[str, Any]) -> list[Place]: return [place(text)] if text else [] -def _employment(j: dict[str, Any]) -> str | None: - return table_word(_EMPLOYMENT, j.get("employmentStatusLabel"), log, "employmentStatusLabel") - - def parse(body: bytes, slug: str) -> list[Row]: """`/careers/list` -> `{"meta": {"totalCount": n}, "result": [...]}`. A role has `id`, `jobOpeningName`, `departmentLabel`, `location` (`city`, @@ -136,7 +123,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "remote": _remote(j), "salary_text": "", # the list states no pay; the detail's `compensation` was null } - if employment := _employment(j): + if employment := str(j.get("employmentStatusLabel") or "").strip(): row["employment"] = employment out.append(row) return out @@ -160,6 +147,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: if place := _place(opening): out["location"] = place out["places"] = _places(opening) - if remote := _remote(opening): + if (remote := _remote(opening)) is not None: out["remote"] = remote return out diff --git a/plugins/readers/deel.py b/plugins/readers/deel.py index 54b51b6..a761ff0 100644 --- a/plugins/readers/deel.py +++ b/plugins/readers/deel.py @@ -24,7 +24,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import num, only, parse_dt, pay_line, table_word +from ._shared import num, only, parse_dt, pay_line log = logging.getLogger(__name__) @@ -59,15 +59,6 @@ _PUSH = re.compile(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*")\]\)') _ROW = re.compile(rb"([0-9a-f]*):") _TEXT = re.compile(rb"T([0-9a-f]+),") -#: Deel's `workArrangementEnum` -> `store._remote`'s words. -_REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"} - -#: The name of a `jobEmploymentTypes` entry -> the app's employment words. -_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", - "part time": "part_time", "contract": "contract", "contractor": "contract", - "consultant": "contract", "temporary": "temporary", "intern": "internship", - "internship": "internship", "freelance": "freelance"} - def slug_ok(slug: str) -> bool: return bool(_SLUG.fullmatch(slug)) @@ -121,10 +112,10 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]: def _employment(job: dict[str, Any]) -> str | None: - """The first `jobEmploymentTypes` entry that names a known type.""" + """The name of the first `jobEmploymentTypes` entry that has one.""" for entry in job.get("jobEmploymentTypes") or []: name = ((entry or {}).get("employmentType") or {}).get("name") - if found := table_word(_EMPLOYMENT, name, log, "employment type"): + if found := str(name or "").strip(): return found return None @@ -155,7 +146,7 @@ def parse(body: bytes, slug: str) -> list[Row]: description = p.get("richtextDescription") or "" if description.startswith("$"): description = stream.get(description[1:], "") - arrangement = str(job.get("workArrangementEnum") or "") + arrangement = str(job.get("workArrangementEnum") or "").strip() remote = {"REMOTE": True, "HYBRID": False, "ON_SITE": False}.get(arrangement) row: Row = { "source_id": post_id, @@ -166,7 +157,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "posted": parse_dt(published.get("createdAt") or p.get("createdAt")), "department": ", ".join(dict.fromkeys(departments)), "description": html.unescape(strip_html(description)), - "remote": _REMOTE.get(str(job.get("workArrangementEnum") or "")), + "remote": arrangement or None, "salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")), pay.get("currencyIsoCode"), "") if p.get("isCompensationVisible") else ""), diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py index e015f82..0d990e6 100644 --- a/plugins/readers/eightfold.py +++ b/plugins/readers/eightfold.py @@ -70,10 +70,6 @@ _NAME = re.compile(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9] #: Names a private network uses, never a company's career site. _LOCAL = (".localhost", ".local", ".internal", ".lan", ".home.arpa") -#: `workLocationOption` -> `store._remote`'s words. Only "onsite" and "hybrid" -#: were seen; the other spellings are the obvious ones. -_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"} - def parts(slug: str) -> tuple[str, str]: """`(host, domain)` of a slug, lowercased.""" @@ -162,7 +158,7 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]: "posted": parse_dt(p.get("creationTs") or p.get("postedTs")), "department": str(p.get("department") or "").strip(), "description": "", - "remote": _REMOTE.get(str(p.get("workLocationOption") or "").strip().lower()), + "remote": str(p.get("workLocationOption") or "").strip() or None, "salary_text": "", }) return out diff --git a/plugins/readers/gem.py b/plugins/readers/gem.py index bb8c6d7..ee90a1e 100644 --- a/plugins/readers/gem.py +++ b/plugins/readers/gem.py @@ -27,7 +27,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, Page, add_place, only, parse_dt, table_word +from ._shared import Get, Page, add_place, only, parse_dt log = logging.getLogger(__name__) @@ -65,14 +65,6 @@ DETAIL_QUERY = ( "job { id locationType employmentType requisitionId teamDisplayName " "department { id extId name } } compensationHtml } }") -#: `job.locationType` -> `store._remote`'s words. -_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"} - -#: `job.employmentType` -> the app's employment words. -_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contract": "contract", - "contractor": "contract", "intern": "internship", "internship": "internship", - "temporary": "temporary", "freelance": "freelance"} - def slug_ok(slug: str) -> bool: return bool(_BOARD.fullmatch(slug)) @@ -134,9 +126,9 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]: add_place(places, {"text": text, "remote": loc["isRemote"] if isinstance(loc.get("isRemote"), bool) else None}) job = p.get("job") or {} - remote = _REMOTE.get(str(job.get("locationType") or "").upper()) + remote: bool | str | None = str(job.get("locationType") or "").strip() or None if remote is None and locations and all(loc.get("isRemote") for loc in locations): - remote = "remote" + remote = True row: Row = { "source_id": ext_id, "title": title, @@ -149,7 +141,7 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]: "remote": remote, "salary_text": "", } - if employment := table_word(_EMPLOYMENT, job.get("employmentType"), log, "employmentType"): + if employment := str(job.get("employmentType") or "").strip(): row["employment"] = employment out.append(row) return out diff --git a/plugins/readers/hibob.py b/plugins/readers/hibob.py index 31fa9df..a1a9204 100644 --- a/plugins/readers/hibob.py +++ b/plugins/readers/hibob.py @@ -23,7 +23,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount +from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount log = logging.getLogger(__name__) @@ -38,21 +38,10 @@ READS = Reads( "app's index page with no rules (checked 2026-09-29)", ) -#: `employmentType`, the company's own name for it -> the app's employment -#: words. "Permanent" says nothing about hours and stays unmapped. -_EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time", - "full-time": "full_time", "regular part time": "part_time", - "part time": "part_time", "part-time": "part_time", "contract": "contract", - "contractor": "contract", "temporary": "temporary", "fixed term": "temporary", - "intern": "internship", "internship": "internship", "freelance": "freelance"} - NAME = "hibob" LABEL = "HiBob" PATTERNS = (r"https?://([a-z0-9-]+)\.careers\.hibob\.com",) -#: HiBob's `workspaceTypeId` -> `store._remote`'s words. -_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite", "onsite": "onsite"} - #: The texts of an ad, in the order the ad shows them. _SECTIONS = ("description", "responsibilities", "requirements", "benefits") @@ -126,13 +115,13 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: "posted": parse_dt(ad.get("publishedAt")), "department": str(ad.get("department") or "").strip(), "description": " ".join(html.unescape(text).split()), - "remote": _REMOTE.get(str(ad.get("workspaceTypeId") or "").strip().lower()), + "remote": str(ad.get("workspaceTypeId") or "").strip() or None, "salary_text": pay_line(lo, hi, currency, period), "salary_min": salary_min, "salary_max": salary_max, "currency": code, } - if employment := table_word(_EMPLOYMENT, ad.get("employmentType"), log, "employmentType"): + if employment := str(ad.get("employmentType") or "").strip(): row["employment"] = employment out.append(row) return out diff --git a/plugins/readers/jibe.py b/plugins/readers/jibe.py index 0e12946..7e2e617 100644 --- a/plugins/readers/jibe.py +++ b/plugins/readers/jibe.py @@ -38,7 +38,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, Page, add_place, only, parse_dt, place, table_word +from ._shared import Get, Page, add_place, only, parse_dt, place log = logging.getLogger(__name__) @@ -49,11 +49,6 @@ READS = Reads( robots="read: robots.txt allows /jobs, with crawl-delay 5 not honoured (checked 2026-09-29)", ) -#: `employment_type` -> the app's employment words. -_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contractor": "contract", - "contract": "contract", "temporary": "temporary", "intern": "internship", - "internship": "internship", "freelance": "freelance"} - NAME = "jibe" LABEL = "Jibe" PATTERNS = ( @@ -157,10 +152,10 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: } # One code, for a role at one place: over several countries it would # gate the others by the site's own. - code = str(j.get("country_code") or "").strip().upper() - if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code): + code = str(j.get("country_code") or "").strip() + if len(places) == 1 and code: row["country"] = code - if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"): + if employment := str(j.get("employment_type") or "").strip(): row["employment"] = employment out.append(row) return out diff --git a/plugins/readers/jobylon.py b/plugins/readers/jobylon.py index 74fc7b3..dc519b7 100644 --- a/plugins/readers/jobylon.py +++ b/plugins/readers/jobylon.py @@ -26,7 +26,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import Get, parse_dt, place, table_word +from ._shared import Get, parse_dt, place log = logging.getLogger(__name__) @@ -58,28 +58,10 @@ _SLUG = re.compile(r"[0-9]{1,12}") #: from. _ROLE_PATH = re.compile(r"/jobs/[0-9]+(?:-[A-Za-z0-9_-]*)?/?") -#: The embed's `experience` -> `app.models.SENIORITIES`. Only the coarse levels -#: are mapped; "Experienced" sits between "Mid level" and the managerial -#: levels and says too little to place, so it stays unmapped and is logged. -_SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern", - "entry level": "junior", "entry-level": "junior", "junior": "junior", - "graduate": "junior", "mid level": "mid", "mid-level": "mid", "mid": "mid", - "senior": "senior", "lead": "lead", "manager": "manager", "director": "director", - "executive": "exec", "not applicable": ""} - -#: The embed's `employment_type` -> the app's employment words. -_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", - "part time": "part_time", "contract": "contract", "consultant": "contract", - "temporary": "temporary", "internship": "internship", "intern": "internship", - "freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"} - _ASSIGNMENT = "JBL.embed_v2['jobs'] = " _KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _TRAILING_COMMA = re.compile(r",(\s*[\]}])") -#: Jobylon's `workspace` -> `store._remote`'s words. -_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"} - #: The English month names, by their first three letters, in the spellings #: `published_date` uses ("Sept. 25, 2026"). _MONTHS = {m: i for i, m in enumerate( @@ -205,7 +187,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "posted": _published(j.get("published_date")), "department": "" if function == "All functions" else function, "description": "", - "remote": _REMOTE.get(str(j.get("workspace") or "").strip().lower()), + "remote": str(j.get("workspace") or "").strip() or None, "salary_text": "", # the embed states no pay } # `summary` is the text the company gave the listing, the string @@ -213,9 +195,9 @@ def parse(body: bytes, slug: str) -> list[Row]: summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split()) if summary and summary != "None": row["teaser"] = summary - if seniority := table_word(_SENIORITY, j.get("experience"), log, "experience"): + if seniority := str(j.get("experience") or "").strip(): row["seniority"] = seniority - if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"): + if employment := str(j.get("employment_type") or "").strip(): row["employment"] = employment out.append(row) return out diff --git a/plugins/readers/lever.py b/plugins/readers/lever.py index f953792..83bb2f5 100644 --- a/plugins/readers/lever.py +++ b/plugins/readers/lever.py @@ -17,14 +17,13 @@ from __future__ import annotations import json import logging -import re from collections.abc import Callable from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount +from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount log = logging.getLogger(__name__) @@ -36,13 +35,6 @@ READS = Reads( robots="api: no robots.txt check needed (public board API), 2026-09-30", ) -#: `categories.commitment` -> the app's employment words. A commitment is free -#: text on Lever ("Permanent" is seen too and says nothing about hours). -_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time", - "part-time": "part_time", "part time": "part_time", "parttime": "part_time", - "intern": "internship", "internship": "internship", "contract": "contract", - "contractor": "contract", "temporary": "temporary", "freelance": "freelance"} - NAME = "lever" LABEL = "Lever" RANK = 20 @@ -153,10 +145,10 @@ def parse(body: bytes, slug: str) -> list[Row]: } # `country` is one code, for a role at one place; over several it would # gate the others by the first's country. - country = str(j.get("country") or "").strip().upper() - if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country): + country = str(j.get("country") or "").strip() + if len(places) == 1 and country: row["country"] = country - if employment := table_word(_EMPLOYMENT, categories.get("commitment"), log, "commitment"): + if employment := str(categories.get("commitment") or "").strip(): row["employment"] = employment rows.append(row) return rows diff --git a/tests/test_ats_ashby.py b/tests/test_ats_ashby.py index 49cd788..b776798 100644 --- a/tests/test_ats_ashby.py +++ b/tests/test_ats_ashby.py @@ -25,14 +25,14 @@ def test_the_primary_and_every_secondary_location_are_places() -> None: assert second["places"] == [{"text": "Dublin", "remote": True}] -def test_a_hybrid_role_is_not_remote_although_the_host_flags_it() -> None: +def test_remote_is_the_workplace_type_as_the_host_writes_it() -> None: first, second = ashby.parse(BOARD, "acme") - assert first["remote"] == "hybrid" and second["remote"] == "remote" + assert first["remote"] == "Hybrid" and second["remote"] == "Remote" -def test_the_employment_type_is_mapped() -> None: +def test_the_employment_type_is_the_hosts_word() -> None: first, second = ashby.parse(BOARD, "acme") - assert (first["employment"], second["employment"]) == ("full_time", "contract") + assert (first["employment"], second["employment"]) == ("FullTime", "Contract") def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None: diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py index f97357e..c7cd28a 100644 --- a/tests/test_ats_bamboohr.py +++ b/tests/test_ats_bamboohr.py @@ -47,15 +47,15 @@ def test_a_listing_becomes_rows() -> None: assert second["url"] == "https://configit.bamboohr.com/careers/37" assert second["department"] == "Marketing" and rows[0]["department"] == "" assert second["location"] == "København Ø" - assert second["remote"] == "hybrid" + assert second["remote"] == "2" assert (second["posted"], second["description"], second["salary_text"]) == (None, "", "") @pytest.mark.parametrize(("flag", "kind", "remote"), [ - (True, "0", "remote"), (None, "0", "onsite"), (None, "1", "remote"), (None, "2", "hybrid"), - (None, None, None), (False, "7", None)]) + (True, "0", True), (None, "0", "0"), (None, "1", "1"), (None, "2", "2"), + (None, None, None), (False, "7", "7")]) def test_remote_is_the_flag_then_the_location_type( - flag: bool | None, kind: str | None, remote: str | None) -> None: + flag: bool | None, kind: str | None, remote: bool | str | None) -> None: body = json.loads(_read("bamboohr.json")) body["result"][0] |= {"isRemote": flag, "locationType": kind} assert bamboohr.parse(json.dumps(body).encode(), "configit")[0]["remote"] == remote @@ -102,7 +102,7 @@ def test_detail_reads_the_text_the_date_and_the_country() -> None: assert found["posted"] == datetime(2026, 9, 23, tzinfo=UTC) assert found["location"] == "København Ø, Denmark" assert found["places"] == [{"text": "København Ø, Denmark", "remote": None}] - assert found["remote"] == "hybrid" + assert found["remote"] == "2" text = found["description"] assert text.startswith("Our Global Marketing Team is looking for") assert "Skills & Experience" in text and "&" not in text and "<" not in text @@ -144,11 +144,11 @@ def test_the_board_and_its_details_are_read_with_the_json_accept_on_the_detail() def test_the_role_states_one_place_and_the_employment() -> None: rows = bamboohr.parse(_read("bamboohr.json"), "configit") assert rows[1]["places"] == [{"text": "København Ø", "remote": None}] - assert rows[1]["employment"] == "full_time" + assert rows[1]["employment"] == "Full-Time" body = json.loads(_read("bamboohr.json")) body["result"][0] |= {"employmentStatusLabel": "Seasonal", "location": {"city": None}} first = bamboohr.parse(json.dumps(body).encode(), "configit")[0] - assert first["places"] == [] and "employment" not in first + assert first["places"] == [] and first["employment"] == "Seasonal" def test_it_declares_what_it_reads() -> None: diff --git a/tests/test_ats_deel.py b/tests/test_ats_deel.py index aa27f07..1bb1127 100644 --- a/tests/test_ats_deel.py +++ b/tests/test_ats_deel.py @@ -32,7 +32,7 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None: assert first["url"] == ("https://jobs.deel.com/klarna/job-details/" "21584046-56d5-44c4-8b18-7e15dc659827/overview") assert first["location"] == "Stockholm" and first["department"] == "Engineering" - assert first["remote"] == "onsite" + assert first["remote"] == "ON_SITE" assert first["description"].startswith("Klarna, briefly At Klarna") assert "<" not in first["description"] and "$" not in first["description"][:3] assert all(r["description"] for r in rows) @@ -41,18 +41,18 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None: "2026-09-07T12:33:38.484000+00:00" -def test_several_locations_are_joined_and_remote_uses_the_stores_words() -> None: +def test_several_locations_are_joined_and_remote_is_the_hosts_word() -> None: rows = {r["source_id"][:8]: r for r in deel.parse(PAGE, "klarna")} assert rows["a8be45ac"]["location"] == "Stockholm; London; Milan; Warsaw" - assert rows["f96bb9d4"]["remote"] == "remote" + assert rows["f96bb9d4"]["remote"] == "REMOTE" assert rows["f96bb9d4"]["location"] == "Ohio" # Every place is kept, in the host's order; the work arrangement is the # places' remote flag. assert rows["a8be45ac"]["places"] == [ {"text": name, "remote": False} for name in ("Stockholm", "London", "Milan", "Warsaw")] assert rows["f96bb9d4"]["places"] == [{"text": "Ohio", "remote": True}] - assert rows["f96bb9d4"]["employment"] == "contract" # "Consultant" - assert rows["21584046"]["employment"] == "full_time" + assert rows["f96bb9d4"]["employment"] == "Consultant" + assert rows["21584046"]["employment"] == "Full-time" def test_pay_is_only_text_since_the_period_is_not_stated() -> None: diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py index ad70888..8efb46f 100644 --- a/tests/test_ats_eightfold.py +++ b/tests/test_ats_eightfold.py @@ -87,8 +87,8 @@ def test_a_page_becomes_rows() -> None: @pytest.mark.parametrize(("option", "remote"), [ - ("onsite", "onsite"), ("hybrid", "hybrid"), ("remote", "remote"), (None, None), ("odd", None)]) -def test_the_work_location_option_maps_to_the_remote_words( + ("onsite", "onsite"), ("Hybrid", "Hybrid"), (None, None), ("odd", "odd"), (" ", None)]) +def test_the_work_location_option_is_the_hosts_word( option: str | None, remote: str | None) -> None: assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py index eeed4e5..97395ca 100644 --- a/tests/test_ats_gem.py +++ b/tests/test_ats_gem.py @@ -55,7 +55,7 @@ def test_a_listing_becomes_rows() -> None: assert first["source_id"] == "8080046002" # numeric, from a synced Greenhouse assert first["title"] == "Staff AI Product Engineer (Prague/Brno/Remote EU)" assert first["url"] == "https://jobs.gem.com/productboard/8080046002" - assert first["department"] == "Product Engineering" and first["remote"] == "hybrid" + assert first["department"] == "Product Engineering" and first["remote"] == "HYBRID" assert first["location"] == "Prague" assert (first["posted"], first["description"], first["salary_text"]) == (None, "", "") assert backend["source_id"] == "am9icG9zdDqWC1V_RM7U6ORvTd1Iu64r" # opaque @@ -64,16 +64,16 @@ def test_a_listing_becomes_rows() -> None: assert backend["places"] == [{"text": "Prague", "remote": False}, {"text": "Remote Europe", "remote": True}, {"text": "Brno", "remote": False}] - assert first["employment"] == "full_time" + assert first["employment"] == "FULL_TIME" assert len({r["source_id"] for r in rows}) == 11 @pytest.mark.parametrize(("kind", "flags", "remote"), [ - ("IN_OFFICE", [False], "onsite"), ("HYBRID", [True], "hybrid"), ("REMOTE", [False], "remote"), - (None, [True, True], "remote"), (None, [True, False], None), (None, [], None), - ("SOMETHING_NEW", [False], None)]) + ("IN_OFFICE", [False], "IN_OFFICE"), ("HYBRID", [True], "HYBRID"), ("REMOTE", [False], "REMOTE"), + (None, [True, True], True), (None, [True, False], None), (None, [], None), + ("SOMETHING_NEW", [False], "SOMETHING_NEW")]) def test_remote_is_the_location_type_then_the_locations( - kind: str | None, flags: list[bool], remote: str | None) -> None: + kind: str | None, flags: list[bool], remote: bool | str | None) -> None: posting = {"extId": "1", "title": "Role", "job": {"locationType": kind}, "locations": [{"city": "X", "isRemote": f} for f in flags]} assert gem.rows([posting], "acme")[0]["remote"] == remote diff --git a/tests/test_ats_hibob.py b/tests/test_ats_hibob.py index 02d03c2..c73dded 100644 --- a/tests/test_ats_hibob.py +++ b/tests/test_ats_hibob.py @@ -51,7 +51,7 @@ def test_a_board_of_general_applications_reads_as_it_is() -> None: rows = hibob.parse(DIXA, "dixa") assert [(r["title"], r["location"], r["remote"]) for r in rows] == [ ("Expression of interest", "Copenhagen, Denmark", "hybrid"), - ("Expression of interest", "London, United Kingdom", "onsite")] + ("Expression of interest", "London, United Kingdom", "on_site")] assert rows[1]["description"].startswith("We're always excited") @@ -153,10 +153,10 @@ def test_each_ad_names_one_site_and_the_host_lists_a_copy_per_site() -> None: assert len({r["source_id"] for r in dixa}) == 2 -def test_the_employment_type_is_mapped_and_a_permanent_one_is_left_unmapped() -> None: +def test_the_employment_type_is_the_hosts_word() -> None: rows = hibob.parse(BOARD, "liquidia") - assert rows[0]["employment"] == "full_time" # "Regular Full Time" - assert "employment" not in hibob.parse(DIXA, "dixa")[0] # "Permanent" + assert rows[0]["employment"] == "Regular Full Time" + assert hibob.parse(DIXA, "dixa")[0]["employment"] == "Permanent" def test_it_declares_copies_with_a_page_and_a_robots_decision() -> None: diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py index 4bb7999..0b03125 100644 --- a/tests/test_ats_jibe.py +++ b/tests/test_ats_jibe.py @@ -57,7 +57,7 @@ def test_a_page_becomes_rows() -> None: "Manchester, United Kingdom"] assert "country" not in several assert first["places"] == [{"text": "Bucharest, Romania", "remote": None}] - assert (first["country"], first["employment"]) == ("RO", "full_time") + assert (first["country"], first["employment"]) == ("RO", "FULL_TIME") assert all(r["description"] and "<" not in r["description"] and " " not in r["description"] for r in rows) diff --git a/tests/test_ats_jobylon.py b/tests/test_ats_jobylon.py index 90780a3..b744334 100644 --- a/tests/test_ats_jobylon.py +++ b/tests/test_ats_jobylon.py @@ -14,7 +14,6 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from app import models as app_models # noqa: E402 from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 from plugins.readers import Ask, jobylon # noqa: E402 @@ -36,20 +35,20 @@ def test_rows_come_from_the_embeds_literal() -> None: assert first["location"] == "Stockholm" assert first["places"] == [{"text": "Stockholm", "remote": None}] assert first["department"] == "Accounting & Finance" # `&` read as `&` - assert first["remote"] == "hybrid" + assert first["remote"] == "Hybrid" assert first["description"] == "" and first["salary_text"] == "" # A day, with no time of day: the embed says "Sept. 25, 2026". assert first["posted"] is not None and first["posted"].isoformat() == \ "2026-09-25T00:00:00+00:00" -def test_places_department_and_remote_follow_the_embeds_words() -> None: +def test_places_department_and_remote_are_the_embeds_words() -> None: rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")} assert rows["384238"]["location"] == "Houston; Dallas; Atlanta" assert [p["text"] for p in rows["384238"]["places"]] == ["Houston", "Dallas", "Atlanta"] assert rows["384238"]["department"] == "" # "All functions" names none - assert rows["382124"]["remote"] == "remote" - assert rows["382860"]["remote"] == "onsite" + assert rows["382124"]["remote"] == "Remote" + assert rows["382860"]["remote"] == "On-site" assert rows["380347"]["remote"] is None # a role with no workspace @@ -130,18 +129,14 @@ def test_an_empty_board_is_no_roles() -> None: assert jobylon.parse(EMPTY, "1056") == [] -def test_the_experience_is_mapped_and_a_word_the_table_lacks_is_logged( - caplog: pytest.LogCaptureFixture) -> None: - caplog.set_level("DEBUG", logger="plugins.readers.jobylon") +def test_the_experience_and_the_employment_type_are_the_hosts_words() -> None: rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")} - assert rows["384238"]["seniority"] == "mid" # "Mid level" - assert "seniority" not in rows["385191"] # "Experienced" says too little - assert "unknown experience 'Experienced'" in caplog.text - assert rows["385191"]["employment"] == "full_time" - assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values()) + assert rows["384238"]["seniority"] == "Mid level" + assert rows["385191"]["seniority"] == "Experienced" + assert rows["385191"]["employment"] == "Full-time" rows = {r["source_id"]: r for r in jobylon.parse( (FIXTURES / "jobylon-places.html").read_bytes(), "3143")} - assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "internship" + assert rows["502"]["seniority"] == "Student" and rows["502"]["employment"] == "Internship" def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None: diff --git a/tests/test_ats_lever.py b/tests/test_ats_lever.py index d8cbe27..1b4a2af 100644 --- a/tests/test_ats_lever.py +++ b/tests/test_ats_lever.py @@ -28,9 +28,9 @@ def test_the_country_is_only_taken_for_a_role_at_one_place() -> None: assert "country" not in several and one["country"] == "DE" -def test_the_commitment_is_mapped_and_a_permanent_one_is_left_unmapped() -> None: +def test_the_commitment_is_the_hosts_word() -> None: several, one = lever.parse(BOARD, "acme") - assert several["employment"] == "full_time" and "employment" not in one + assert several["employment"] == "Full-time" and one["employment"] == "Permanent" def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None: diff --git a/tests/test_prep_hosts_ashby.py b/tests/test_prep_hosts_ashby.py new file mode 100644 index 0000000..218db69 --- /dev/null +++ b/tests/test_prep_hosts_ashby.py @@ -0,0 +1,37 @@ +"""The ashby words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.ashby") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['REMOTE', 'EMPLOYMENT']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('onsite', 'onsite')]) +def test_remote_words(word: str, value: str) -> None: + assert hosts.REMOTE[word] == value + + +@pytest.mark.parametrize(("word", "value"), [('fulltime', 'full_time'), ('parttime', 'part_time'), ('intern', 'internship'), ('contract', 'contract'), ('temporary', 'temporary')]) +def test_employment_words(word: str, value: str) -> None: + assert hosts.EMPLOYMENT[word] == value diff --git a/tests/test_prep_hosts_bamboohr.py b/tests/test_prep_hosts_bamboohr.py new file mode 100644 index 0000000..66cf013 --- /dev/null +++ b/tests/test_prep_hosts_bamboohr.py @@ -0,0 +1,37 @@ +"""The bamboohr words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.bamboohr") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('0', 'onsite'), ('1', 'remote'), ('2', 'hybrid')]) +def test_remote_words(word: str, value: str) -> None: + assert hosts.REMOTE[word] == value + + +@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('part time', 'part_time'), ('intern', 'internship'), ('contractor', 'contract')]) +def test_employment_words(word: str, value: str) -> None: + assert hosts.EMPLOYMENT[word] == value diff --git a/tests/test_prep_hosts_deel.py b/tests/test_prep_hosts_deel.py new file mode 100644 index 0000000..518ebae --- /dev/null +++ b/tests/test_prep_hosts_deel.py @@ -0,0 +1,37 @@ +"""The deel words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.deel") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('on_site', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')]) +def test_remote_words(word: str, value: str) -> None: + assert hosts.REMOTE[word] == value + + +@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('consultant', 'contract'), ('internship', 'internship')]) +def test_employment_words(word: str, value: str) -> None: + assert hosts.EMPLOYMENT[word] == value diff --git a/tests/test_prep_hosts_eightfold.py b/tests/test_prep_hosts_eightfold.py new file mode 100644 index 0000000..4d134c0 --- /dev/null +++ b/tests/test_prep_hosts_eightfold.py @@ -0,0 +1,32 @@ +"""The eightfold words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.eightfold") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['REMOTE']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('onsite', 'onsite'), ('on_site', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')]) +def test_remote_words(word: str, value: str) -> None: + assert hosts.REMOTE[word] == value diff --git a/tests/test_prep_hosts_gem.py b/tests/test_prep_hosts_gem.py new file mode 100644 index 0000000..c442a89 --- /dev/null +++ b/tests/test_prep_hosts_gem.py @@ -0,0 +1,37 @@ +"""The gem words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.gem") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('in_office', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')]) +def test_remote_words(word: str, value: str) -> None: + assert hosts.REMOTE[word] == value + + +@pytest.mark.parametrize(("word", "value"), [('full_time', 'full_time'), ('contractor', 'contract'), ('intern', 'internship')]) +def test_employment_words(word: str, value: str) -> None: + assert hosts.EMPLOYMENT[word] == value diff --git a/tests/test_prep_hosts_hibob.py b/tests/test_prep_hosts_hibob.py new file mode 100644 index 0000000..5a6a284 --- /dev/null +++ b/tests/test_prep_hosts_hibob.py @@ -0,0 +1,37 @@ +"""The hibob words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.hibob") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('on_site', 'onsite'), ('onsite', 'onsite')]) +def test_remote_words(word: str, value: str) -> None: + assert hosts.REMOTE[word] == value + + +@pytest.mark.parametrize(("word", "value"), [('regular full time', 'full_time'), ('fixed term', 'temporary'), ('intern', 'internship')]) +def test_employment_words(word: str, value: str) -> None: + assert hosts.EMPLOYMENT[word] == value diff --git a/tests/test_prep_hosts_jibe.py b/tests/test_prep_hosts_jibe.py new file mode 100644 index 0000000..e71a794 --- /dev/null +++ b/tests/test_prep_hosts_jibe.py @@ -0,0 +1,32 @@ +"""The jibe words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.jibe") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['EMPLOYMENT']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('full_time', 'full_time'), ('contractor', 'contract'), ('intern', 'internship')]) +def test_employment_words(word: str, value: str) -> None: + assert hosts.EMPLOYMENT[word] == value diff --git a/tests/test_prep_hosts_jobylon.py b/tests/test_prep_hosts_jobylon.py new file mode 100644 index 0000000..ff11c61 --- /dev/null +++ b/tests/test_prep_hosts_jobylon.py @@ -0,0 +1,42 @@ +"""The jobylon words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.jobylon") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['SENIORITY', 'EMPLOYMENT', 'REMOTE']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('mid level', 'mid'), ('student', 'intern'), ('entry level', 'junior'), ('executive', 'exec'), ('not applicable', '')]) +def test_seniority_words(word: str, value: str) -> None: + assert hosts.SENIORITY[word] == value + + +@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('consultant', 'contract'), ('summer job', 'temporary')]) +def test_employment_words(word: str, value: str) -> None: + assert hosts.EMPLOYMENT[word] == value + + +@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('on-site', 'onsite')]) +def test_remote_words(word: str, value: str) -> None: + assert hosts.REMOTE[word] == value diff --git a/tests/test_prep_hosts_lever.py b/tests/test_prep_hosts_lever.py new file mode 100644 index 0000000..36105cb --- /dev/null +++ b/tests/test_prep_hosts_lever.py @@ -0,0 +1,32 @@ +"""The lever words, mapped to the app's vocabulary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import models # noqa: E402 +from plugins.readers._shared import EMPLOYMENTS # noqa: E402 + +hosts = importlib.import_module("app.prep.hosts.lever") + +VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE} + + +@pytest.mark.parametrize("table", ['EMPLOYMENT']) +def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None: + words = getattr(hosts, table) + assert all(key == key.strip().lower() for key in words) + assert set(words.values()) <= {*VOCABULARY[table], ""} + + +@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('fulltime', 'full_time'), ('intern', 'internship'), ('freelance', 'freelance')]) +def test_employment_words(word: str, value: str) -> None: + assert hosts.EMPLOYMENT[word] == value diff --git a/tests/test_readers_shared.py b/tests/test_readers_shared.py index 91b3474..9417710 100644 --- a/tests/test_readers_shared.py +++ b/tests/test_readers_shared.py @@ -24,10 +24,16 @@ from plugins.readers._shared import ( # noqa: E402 #: The module-level names a reader keeps its employment words in. _TABLES = ("_EMPLOYMENT", "_CONTRACT", "_SCHEDULE") +#: The readers of `ashby` to `lever` hand on the host's words; their tables +#: moved to `app/prep/hosts`. +_RAW = {"ashby", "bamboohr", "deel", "eightfold", "gem", "greenhouse", "hibob", "hrmanager", + "jibe", "jobylon", "lever"} + def test_every_employment_table_maps_into_the_shared_vocabulary() -> None: found = 0 for name, module in readers.READERS.items(): + assert not _RAW & {name} or not any(hasattr(module, t) for t in _TABLES), name for table_name in _TABLES: table = getattr(module, table_name, None) if table is None: @@ -36,7 +42,7 @@ def test_every_employment_table_maps_into_the_shared_vocabulary() -> None: values = table.values() if isinstance(table, dict) else [v for _k, v in table] # "" is a known word that names no employment ("permanent"). assert set(values) <= {*EMPLOYMENTS, ""}, (name, table_name) - assert found >= 15 + assert found >= 5 def test_the_vocabulary_is_the_documented_one() -> None: From 06eb929eaf246c24fb334d7ca569491ee85c61ed Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 11:07:07 +0200 Subject: [PATCH 2/4] Readers oracle to workday and the providers hand on the host's words; their word tables move to app/prep/hosts Co-Authored-By: Claude Opus 5.5 --- app/prep/hosts/oracle.py | 5 +++ app/prep/hosts/personio.py | 11 +++++ app/prep/hosts/phenom.py | 5 +++ app/prep/hosts/pinpoint.py | 12 ++++++ app/prep/hosts/recruitee.py | 15 +++++++ app/prep/hosts/rippling.py | 8 ++++ app/prep/hosts/smartrecruiters.py | 11 +++++ app/prep/hosts/teamtailor.py | 7 ++++ app/prep/hosts/workable.py | 8 ++++ app/prep/hosts/workday.py | 10 +++++ plugins/readers/adzuna.py | 3 +- plugins/readers/jobindex.py | 7 +--- plugins/readers/oracle.py | 26 ++++-------- plugins/readers/personio.py | 39 +++++------------- plugins/readers/phenom.py | 7 ++-- plugins/readers/pinpoint.py | 30 ++------------ plugins/readers/platsbanken.py | 10 ++--- plugins/readers/recruitee.py | 51 ++++-------------------- plugins/readers/rippling.py | 22 ++-------- plugins/readers/smartrecruiters.py | 30 ++++---------- plugins/readers/successfactors.py | 22 ++-------- plugins/readers/teamtailor.py | 8 +--- plugins/readers/workable.py | 31 +++++--------- plugins/readers/workday.py | 48 +++++++++------------- tests/test_ats.py | 13 +++--- tests/test_ats_oracle.py | 10 ++--- tests/test_ats_personio.py | 24 +++-------- tests/test_ats_phenom.py | 2 +- tests/test_ats_pinpoint.py | 7 +--- tests/test_ats_recruitee.py | 34 +++++----------- tests/test_ats_rippling.py | 24 +++++------ tests/test_ats_smartrecruiters.py | 28 +++---------- tests/test_ats_successfactors.py | 16 ++------ tests/test_ats_workable.py | 14 ++----- tests/test_ats_workday.py | 21 +++++----- tests/test_prep_hosts_oracle.py | 12 ++++++ tests/test_prep_hosts_personio.py | 22 ++++++++++ tests/test_prep_hosts_phenom.py | 12 ++++++ tests/test_prep_hosts_pinpoint.py | 24 +++++++++++ tests/test_prep_hosts_recruitee.py | 27 +++++++++++++ tests/test_prep_hosts_rippling.py | 19 +++++++++ tests/test_prep_hosts_smartrecruiters.py | 22 ++++++++++ tests/test_prep_hosts_teamtailor.py | 14 +++++++ tests/test_prep_hosts_workable.py | 18 +++++++++ tests/test_prep_hosts_workday.py | 21 ++++++++++ tests/test_readers_adzuna.py | 2 +- tests/test_readers_platsbanken.py | 4 +- tests/test_readers_shared.py | 32 +++++++++++++++ 48 files changed, 460 insertions(+), 388 deletions(-) create mode 100644 app/prep/hosts/oracle.py create mode 100644 app/prep/hosts/personio.py create mode 100644 app/prep/hosts/phenom.py create mode 100644 app/prep/hosts/pinpoint.py create mode 100644 app/prep/hosts/recruitee.py create mode 100644 app/prep/hosts/rippling.py create mode 100644 app/prep/hosts/smartrecruiters.py create mode 100644 app/prep/hosts/teamtailor.py create mode 100644 app/prep/hosts/workable.py create mode 100644 app/prep/hosts/workday.py create mode 100644 tests/test_prep_hosts_oracle.py create mode 100644 tests/test_prep_hosts_personio.py create mode 100644 tests/test_prep_hosts_phenom.py create mode 100644 tests/test_prep_hosts_pinpoint.py create mode 100644 tests/test_prep_hosts_recruitee.py create mode 100644 tests/test_prep_hosts_rippling.py create mode 100644 tests/test_prep_hosts_smartrecruiters.py create mode 100644 tests/test_prep_hosts_teamtailor.py create mode 100644 tests/test_prep_hosts_workable.py create mode 100644 tests/test_prep_hosts_workday.py diff --git a/app/prep/hosts/oracle.py b/app/prep/hosts/oracle.py new file mode 100644 index 0000000..05ac9f4 --- /dev/null +++ b/app/prep/hosts/oracle.py @@ -0,0 +1,5 @@ +"""Oracle Recruiting Cloud's words, for prep.""" + +#: `WorkplaceTypeCode` -> `REMOTE`'s words. A role that states none has an +#: empty code, which stays unknown. +REMOTE = {"ora_remote": "remote", "ora_hybrid": "hybrid", "ora_on_site": "onsite"} diff --git a/app/prep/hosts/personio.py b/app/prep/hosts/personio.py new file mode 100644 index 0000000..62bb864 --- /dev/null +++ b/app/prep/hosts/personio.py @@ -0,0 +1,11 @@ +"""Personio's words, for prep.""" + +#: `seniority`. `experienced` is what most roles say and names no level. +SENIORITY = {"entry-level": "junior", "student": "intern", "executive": "exec", + "experienced": ""} + +#: `employment` is the feed's `employmentType`, or, for a `permanent` one +#: (which says nothing about the hours), its `schedule`. +EMPLOYMENT = {"temporary": "temporary", "freelance": "contract", "intern": "internship", + "working_student": "internship", "trainee": "internship", + "permanent": "", "full-time": "full_time", "part-time": "part_time"} diff --git a/app/prep/hosts/phenom.py b/app/prep/hosts/phenom.py new file mode 100644 index 0000000..01873c3 --- /dev/null +++ b/app/prep/hosts/phenom.py @@ -0,0 +1,5 @@ +"""Phenom's words, for prep.""" + +#: The words a page's `checkRemote` says (King's pages say "On-site"; no +#: other was seen). +REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"} diff --git a/app/prep/hosts/pinpoint.py b/app/prep/hosts/pinpoint.py new file mode 100644 index 0000000..44889a7 --- /dev/null +++ b/app/prep/hosts/pinpoint.py @@ -0,0 +1,12 @@ +"""Pinpoint's words, for prep.""" + +#: `workplace_type`. Anything else says nothing. +REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"} + +#: What `employment_type` (`permanent_full_time`, `fixed_term_full_time`) +#: says, matched by fragment: the first fragment found in the lowercased +#: word decides, so a contract type beats the hours. The fixtures hold two +#: whole words only, so this is not turned into a whole-word table. +MATCH = [("intern", "internship"), ("freelance", "contract"), ("contract", "contract"), + ("fixed_term", "temporary"), ("temporary", "temporary"), + ("part_time", "part_time"), ("full_time", "full_time")] diff --git a/app/prep/hosts/recruitee.py b/app/prep/hosts/recruitee.py new file mode 100644 index 0000000..d302831 --- /dev/null +++ b/app/prep/hosts/recruitee.py @@ -0,0 +1,15 @@ +"""Recruitee's words, for prep.""" + +#: `experience_code`. `experienced` names no level and is known, so it maps +#: to none. +SENIORITY = {"student_school": "intern", "entry_level": "junior", "mid_level": "mid", + "experienced": ""} + +#: The words of `employment_type_code` (`fulltime_permanent`), which is +#: lowercased and split at "_" and "-"; the first entry found among its words +#: decides, so a contract type beats the hours. The fixtures hold three whole +#: codes only, so this is a table of words, not whole codes. +MATCH = [("internship", "internship"), ("intern", "internship"), + ("freelance", "contract"), ("contract", "contract"), + ("temporary", "temporary"), ("fixed", "temporary"), + ("parttime", "part_time"), ("fulltime", "full_time")] diff --git a/app/prep/hosts/rippling.py b/app/prep/hosts/rippling.py new file mode 100644 index 0000000..6c42ff6 --- /dev/null +++ b/app/prep/hosts/rippling.py @@ -0,0 +1,8 @@ +"""Rippling's words, for prep.""" + +#: The words of `employmentType.label` (`SALARIED_FT`), which is split at +#: "_" and the first token found here decides. The fixtures hold one whole +#: label only, so this is a table of tokens (lowercased), not whole words. +MATCH = [("ft", "full_time"), ("pt", "part_time"), ("intern", "internship"), + ("internship", "internship"), ("contract", "contract"), ("contractor", "contract"), + ("temp", "temporary"), ("temporary", "temporary"), ("seasonal", "temporary")] diff --git a/app/prep/hosts/smartrecruiters.py b/app/prep/hosts/smartrecruiters.py new file mode 100644 index 0000000..0306623 --- /dev/null +++ b/app/prep/hosts/smartrecruiters.py @@ -0,0 +1,11 @@ +"""SmartRecruiters' words, for prep.""" + +#: `experienceLevel.id`. The three in the middle name no level ("associate" is +#: half of Bosch's roles) and are known, so they map to none. +SENIORITY = {"internship": "intern", "entry_level": "junior", "director": "director", + "executive": "exec", "associate": "", "mid_senior_level": "", + "not_applicable": ""} + +#: `typeOfEmployment.id`. +EMPLOYMENT = {"permanent": "", "intern": "internship", "temporary": "temporary", + "contract": "contract", "part_time": "part_time", "full_time": "full_time"} diff --git a/app/prep/hosts/teamtailor.py b/app/prep/hosts/teamtailor.py new file mode 100644 index 0000000..d28997e --- /dev/null +++ b/app/prep/hosts/teamtailor.py @@ -0,0 +1,7 @@ +"""Teamtailor's words, for prep.""" + +#: `remoteStatus`. `none` says nothing about where the role is worked and +#: `temporary` is remote for a spell only, so both are known and stay +#: unknown rather than become "onsite" or "remote". +REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite", + "none": "", "temporary": ""} diff --git a/app/prep/hosts/workable.py b/app/prep/hosts/workable.py new file mode 100644 index 0000000..d18eccb --- /dev/null +++ b/app/prep/hosts/workable.py @@ -0,0 +1,8 @@ +"""Workable's words, for prep.""" + +#: `workplace`. +REMOTE = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite"} + +#: `type`. +EMPLOYMENT = {"full": "full_time", "part": "part_time", "contract": "contract", + "temporary": "temporary", "internship": "internship"} diff --git a/app/prep/hosts/workday.py b/app/prep/hosts/workday.py new file mode 100644 index 0000000..75e2adb --- /dev/null +++ b/app/prep/hosts/workday.py @@ -0,0 +1,10 @@ +"""Workday's words, for prep.""" + +#: `remoteType`. Only some tenants send it, and each words it its own way +#: (Zendesk's "Fully Flexible", Axis' "On-Site"); a word not listed, a bare +#: "No" among them, says nothing and stays unknown. +REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid", + "in office": "onsite", "on-site": "onsite", "onsite": "onsite"} + +#: `timeType`. +EMPLOYMENT = {"full time": "full_time", "part time": "part_time"} diff --git a/plugins/readers/adzuna.py b/plugins/readers/adzuna.py index afb0ba2..19780a5 100644 --- a/plugins/readers/adzuna.py +++ b/plugins/readers/adzuna.py @@ -24,7 +24,7 @@ from urllib.parse import urlencode from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Key, Place, Reads -from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word +from ._shared import Get, Page, Row, is_web_url, only, parse_dt NAME = "adzuna" LABEL = "Adzuna" @@ -124,7 +124,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: "places": places, "location": town, "country": country.upper(), - "remote": remote_word(title), "posted": parse_dt(job.get("created")), "teaser": _text(job.get("description") or ""), "salary_min": None if predicted else job.get("salary_min"), diff --git a/plugins/readers/jobindex.py b/plugins/readers/jobindex.py index 0dd6da9..c7bb7ce 100644 --- a/plugins/readers/jobindex.py +++ b/plugins/readers/jobindex.py @@ -28,7 +28,7 @@ from urllib.parse import urlencode from desk.sources import strip_html -from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word +from ._shared import Get, Page, Row, is_web_url, only, parse_dt, place from ._contract import Ask, Fields, Filters, Place, Reads NAME = "jobindex" @@ -131,9 +131,7 @@ def rows(decoded: Any, slug: str) -> list[Row]: company = full_title[len(title) + 2:].strip() area = _AREA.search(description) town = _text(area.group(1)) if area else decoded["area"] - remote = remote_word(f"{town} {title}") - places: list[Place] = [{"text": town, "remote": True if remote == "remote" else None}] \ - if town else [] + places: list[Place] = [place(town)] if town else [] out.append({ "source_id": item["id"], "title": title or full_title, @@ -142,7 +140,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: "places": places, "location": town, "country": "DK", - "remote": remote, "posted": parse_dt(item["published"]), "teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)), }) diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py index 9ac97eb..739a6c3 100644 --- a/plugins/readers/oracle.py +++ b/plugins/readers/oracle.py @@ -35,7 +35,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import NAMES, code_of -from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt +from ._shared import Get, Page, Row, add_place, only, parse_dt NAME = "oracle" LABEL = "Oracle Recruiting Cloud" @@ -55,7 +55,7 @@ PATTERNS = ( READS = Reads( scope="board", listing="paged", page_size=200, text="detail", teaser=True, complete="stated", places="many", filters=Filters(country="ids", remote="one"), - fields=Fields(posted="trusted", remote="stated", country="stated"), + fields=Fields(posted="trusted", remote="stated"), robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites//jobs " "(checked 2026-09-29)"), ) @@ -71,11 +71,6 @@ PAGE_SIZE = 200 _HOST = r"[a-z0-9-]+(?:\.[a-z0-9-]+)*\.oraclecloud\.(?:com|eu)" _SLUG = re.compile(rf"({_HOST})/([A-Za-z0-9_-]+)", re.IGNORECASE) -#: `WorkplaceTypeCode` -> `store._remote`'s words. A role that states none -#: has an empty code, which stays unknown. -_REMOTE = {"ora_remote": "remote", "ora_hybrid": "hybrid", "ora_on_site": "onsite"} - - def _parts(slug: str) -> tuple[str, str]: """`(host, site number)` of a slug that passed `slug_ok`.""" host, _, site = slug.partition("/") @@ -189,18 +184,14 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: # The id goes into a URL, and a listing is external content. if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id): continue - remote = _REMOTE.get(_text(role.get("WorkplaceTypeCode")).lower()) + remote = _text(role.get("WorkplaceTypeCode")) or None + at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site") places: list[Place] = [] - codes: set[str] = set() secondary = [s for s in role.get("secondaryLocations") or [] if isinstance(s, dict)] - for name, code in ((_text(role.get("PrimaryLocation")), - _text(role.get("PrimaryLocationCountry"))), - *((_text(s.get("Name")), _text(s.get("CountryCode"))) - for s in secondary)): - add_place(places, {"text": name, "remote": {"remote": True, "hybrid": False, - "onsite": False}.get(remote or "")}) - if code: - codes.add(code.upper()) + for name in (_text(role.get("PrimaryLocation")), + *(_text(s.get("Name")) for s in secondary)): + add_place(places, {"text": name, "remote": ( + True if (remote or "").lower() == "ora_remote" else False if at_office else None)}) out.append({ "source_id": role_id, "title": title, @@ -212,7 +203,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: or _text(role.get("Department")) or _text(role.get("Organization"))), "description": "", "teaser": _plain(role.get("ShortDescriptionStr")), - "country": country_if_single(codes), "remote": remote, "salary_text": "", # the resource states no pay }) diff --git a/plugins/readers/personio.py b/plugins/readers/personio.py index 6476768..8768f13 100644 --- a/plugins/readers/personio.py +++ b/plugins/readers/personio.py @@ -7,16 +7,13 @@ Personio. from __future__ import annotations -import logging import xml.etree.ElementTree as ET from collections.abc import Callable from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, table_word, yearly_amount - -log = logging.getLogger(__name__) +from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount NAME = "personio" LABEL = "Personio" @@ -30,19 +27,6 @@ READS = Reads( robots="api: no robots.txt check needed (public board API), 2026-09-30", ) -#: `seniority` -> the app's levels. `experienced` is what most roles say and -#: names no level, so it is known and maps to none; a word not listed is -#: logged at debug level so the table can grow. -_SENIORITY = {"entry-level": "junior", "student": "intern", "executive": "exec", - "experienced": ""} - -#: `employmentType` when it is not the usual permanent contract, to the app's -#: employment words; `permanent` says nothing about the hours, so `schedule` -#: decides then. -_CONTRACT = {"temporary": "temporary", "freelance": "contract", "intern": "internship", - "working_student": "internship", "trainee": "internship"} -_SCHEDULE = {"full-time": "full_time", "part-time": "part_time"} - #: Personio's `salaryInformation/type` spellings, to the plain period word #: `pay_line` wants ("yearly" -> "year"). Anything else is passed through. _PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour", @@ -85,18 +69,14 @@ def _salary_text(salary_info: ET.Element) -> str: salary_info.findtext("currencyCode"), period) -def _seniority(word: str) -> str | None: - return table_word(_SENIORITY, word, log, "seniority") - - def _employment(position: ET.Element) -> str | None: - contract = (position.findtext("employmentType") or "").strip().lower() - schedule = (position.findtext("schedule") or "").strip().lower() - if contract in _CONTRACT: - return _CONTRACT[contract] - if contract and contract != "permanent": - log.debug("personio: unknown employmentType %r", contract) - return _SCHEDULE.get(schedule) + """The host's word for the contract, or, for a `permanent` one (which says + nothing about the hours), the `schedule`.""" + contract = (position.findtext("employmentType") or "").strip() + schedule = (position.findtext("schedule") or "").strip() + if contract.lower() in ("", "permanent") and schedule: + return schedule + return contract or None def _places(position: ET.Element) -> list[Place]: @@ -148,8 +128,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "posted": parse_dt(position.findtext("createdAt")), "department": (position.findtext("department") or "").strip(), "description": strip_html(description), - "remote": any(p["remote"] for p in places) or None, - "seniority": _seniority(position.findtext("seniority") or ""), + "seniority": (position.findtext("seniority") or "").strip() or None, "employment": _employment(position), "salary_text": salary_text, "salary_min": salary_min, diff --git a/plugins/readers/phenom.py b/plugins/readers/phenom.py index 1c62876..a08f717 100644 --- a/plugins/readers/phenom.py +++ b/plugins/readers/phenom.py @@ -188,10 +188,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: _JSON_LD = re.compile(r"]+application/ld\+json[^>]*>(.*?)", re.S | re.I) -#: Where a page assigns its state, and the words its `checkRemote` says that -#: `store._remote` reads (King's pages say "On-site"; no other was seen). +#: Where a page assigns its state. Its `checkRemote` says "On-site" on King's +#: pages; no other word was seen. _STATE = "phApp.ddo = " -_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"} def _job_posting(body: str) -> dict[str, Any]: @@ -243,6 +242,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: raise ValueError("no description on the role's page") if posted := parse_dt(posting.get("datePosted")): out["posted"] = posted - if remote := _REMOTE.get(_text(job.get("checkRemote")).lower()): + if remote := _text(job.get("checkRemote")): out["remote"] = remote return out diff --git a/plugins/readers/pinpoint.py b/plugins/readers/pinpoint.py index 15cf6e3..f59fef9 100644 --- a/plugins/readers/pinpoint.py +++ b/plugins/readers/pinpoint.py @@ -18,7 +18,6 @@ from __future__ import annotations import html import json -import logging from collections.abc import Callable from typing import Any @@ -27,8 +26,6 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Reads from ._shared import Row, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount -log = logging.getLogger(__name__) - NAME = "pinpoint" LABEL = "Pinpoint" PATTERNS = (r"https?://([a-z0-9-]+)\.pinpointhq\.com",) @@ -39,9 +36,6 @@ READS = Reads( robots="read: robots.txt allows / (checked 2026-09-29)", ) -#: `workplace_type` -> `store._remote`'s words. Anything else says nothing. -_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"} - #: `compensation_frequency` -> the multiplier that makes a figure yearly. The #: spellings are a guess, since no board read so far states pay; a frequency #: not listed leaves the yearly columns empty, and `salary_text` still says it. @@ -51,12 +45,6 @@ _YEARLY = {"annually": 1, "annual": 1, "yearly": 1, "year": 1, "monthly": 12, "m _PERIOD = {"annually": "year", "annual": "year", "yearly": "year", "monthly": "month", "weekly": "week", "daily": "day", "hourly": "hour"} -#: What `employment_type` (`permanent_full_time`, `fixed_term_full_time`) says, -#: the first fragment found deciding, so a contract type beats the hours. -_EMPLOYMENT = (("intern", "internship"), ("freelance", "contract"), ("contract", "contract"), - ("fixed_term", "temporary"), ("temporary", "temporary"), - ("part_time", "part_time"), ("full_time", "full_time")) - #: The sections of a posting's text, in the order the page shows them: the #: description, then these fields, each with its own heading in `_header`. _SECTIONS = ("key_responsibilities", "skills_knowledge_expertise", "benefits") @@ -112,16 +100,6 @@ def _place(location: dict[str, Any]) -> str: return ", ".join(parts) -def _employment(code: str) -> str | None: - code = code.strip().lower() - for fragment, value in _EMPLOYMENT: - if fragment in code: - return value - if code: - log.debug("pinpoint: unknown employment_type %r", code) - return None - - def _pay(j: dict[str, Any]) -> tuple[int | None, int | None, str, str]: """`(salary_min, salary_max, currency, salary_text)` of a posting whose pay is shown (`compensation_visible`); the free-text `compensation` is the @@ -154,21 +132,21 @@ def parse(body: bytes, slug: str) -> list[Row]: continue job = j.get("job") or {} low, high, currency, salary_text = _pay(j) - workplace = str(j.get("workplace_type") or "").strip().lower() + workplace = str(j.get("workplace_type") or "").strip() place = _place(j.get("location") or {}) out.append({ "source_id": source_id, "title": title, "places": [{"text": place, "remote": {"remote": True, "onsite": False, - "on_site": False, "hybrid": False}.get(workplace)}] + "on_site": False, "hybrid": False}.get(workplace.lower())}] if place else [], "location": place, "url": url, "posted": None, # only the posting's own page has a date "department": str((job.get("department") or {}).get("name") or "").strip(), "description": _text(j), - "remote": _REMOTE.get(workplace), - "employment": _employment(str(j.get("employment_type") or "")), + "remote": workplace or None, + "employment": str(j.get("employment_type") or "").strip() or None, "salary_text": salary_text, "salary_min": low, "salary_max": high, diff --git a/plugins/readers/platsbanken.py b/plugins/readers/platsbanken.py index 48fd6ff..c876266 100644 --- a/plugins/readers/platsbanken.py +++ b/plugins/readers/platsbanken.py @@ -7,8 +7,8 @@ Checked live on 2026-09-29 and 2026-09-30, moved here from `plugins/search.py`. above 2000 is refused, so a search reads 2,100 results at most. It states `total`. A hit has the whole ad's text, so the list has the full text. An ad may name several workplaces: `workplace_addresses` holds one entry each, and -every one is a place of the role. `workplace_model` is mostly empty, so -`remote` comes from the headline and the model together. +every one is a place of the role. `workplace_model` is mostly empty; its +label is passed on as `remote`, in the host's words. """ from __future__ import annotations @@ -22,7 +22,7 @@ from urllib.parse import urlencode from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word +from ._shared import Get, Page, Row, is_web_url, only, parse_dt NAME = "platsbanken" LABEL = "Platsbanken" @@ -113,7 +113,7 @@ def rows(decoded: Any, slug: str) -> list[Row]: addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}] places = [place for address in addresses if isinstance(address, dict) if (place := _place(address))] - model = (hit.get("workplace_model") or {}).get("label") or "" + model = str((hit.get("workplace_model") or {}).get("label") or "").strip() out.append({ "source_id": str(hit.get("id") or ""), "title": title, @@ -122,7 +122,7 @@ def rows(decoded: Any, slug: str) -> list[Row]: "places": places, "location": places[0]["text"] if places else "", "country": "SE", - "remote": remote_word(f"{title} {model}"), + "remote": model or None, "posted": parse_dt(hit.get("publication_date")), "description": _text((hit.get("description") or {}).get("text") or ""), }) diff --git a/plugins/readers/recruitee.py b/plugins/readers/recruitee.py index e494d4c..36f1aab 100644 --- a/plugins/readers/recruitee.py +++ b/plugins/readers/recruitee.py @@ -8,16 +8,13 @@ Recruitee. from __future__ import annotations import json -import logging from collections.abc import Callable from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Row, add_place, country_if_single, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount - -log = logging.getLogger(__name__) +from ._shared import Row, add_place, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount NAME = "recruitee" LABEL = "Recruitee" @@ -27,23 +24,11 @@ PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",) READS = Reads( scope="board", listing="whole", text="listing", complete="whole", places="many", - fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated", + fields=Fields(pay="figures", posted="trusted", remote="stated", seniority="stated", employment="stated"), robots="api: no robots.txt check needed (public board API), 2026-09-30", ) -#: `experience_code` -> the app's levels. `experienced` names no level and is -#: known, so it maps to none; a word not listed is logged at debug level. -_SENIORITY = {"student_school": "intern", "entry_level": "junior", "mid_level": "mid", - "experienced": ""} - -#: The words of `employment_type_code` (`fulltime_permanent`), the first that -#: is found deciding, so a contract type beats the hours. -_EMPLOYMENT = (("internship", "internship"), ("intern", "internship"), - ("freelance", "contract"), ("contract", "contract"), - ("temporary", "temporary"), ("fixed", "temporary"), - ("parttime", "part_time"), ("fulltime", "full_time")) - #: What the board writes as a place for a role that is worked anywhere. _NO_PLACE = ("remote job",) @@ -82,28 +67,13 @@ def _salary_text(salary: dict[str, Any]) -> str: salary.get("currency"), period) -def _seniority(code: str) -> str | None: - return table_word(_SENIORITY, code, log, "experience_code") - - -def _employment(code: str) -> str | None: - words = set(code.strip().lower().replace("-", "_").split("_")) - for word, value in _EMPLOYMENT: - if word in words: - return value - if code.strip(): - log.debug("recruitee: unknown employment_type_code %r", code) - return None - - -def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]: - """Every place of an offer and the countries' codes among them. `locations` +def _places(offer: dict[str, Any]) -> list[Place]: + """Every place of an offer. `locations` lists them (city, state, country), `location` is the first as one line and is the fallback for an offer that lists none. The board writes "Remote job" where an offer has no place: that is not one, and a remote offer with no other place has the place "Remote".""" found: list[Place] = [] - codes: set[str] = set() def add(text: str) -> None: text = text.strip() @@ -119,15 +89,11 @@ def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]: if part and part not in parts: parts.append(part) add(", ".join(parts)) - if code := str(loc.get("country_code") or "").strip().upper(): - codes.add(code) if not found: add(str(offer.get("location") or offer.get("city") or "")) - if code := str(offer.get("country_code") or "").strip().upper(): - codes.add(code) if not found and offer.get("remote"): found.append({"text": "Remote", "remote": True}) - return found, codes + return found def parse(body: bytes, slug: str) -> list[Row]: @@ -145,7 +111,7 @@ def parse(body: bytes, slug: str) -> list[Row]: salary_min, salary_max, currency = yearly_amount( salary.get("min"), salary.get("max"), salary.get("currency"), salary.get("period"), {"year": 1, "month": 12}) - places, codes = _places(j) + places = _places(j) rows.append({ "source_id": str(j.get("id") or ""), "title": title, @@ -160,8 +126,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "salary_min": salary_min, "salary_max": salary_max, "currency": currency, - "country": country_if_single(codes), - "seniority": _seniority(str(j.get("experience_code") or "")), - "employment": _employment(str(j.get("employment_type_code") or "")), + "seniority": str(j.get("experience_code") or "").strip() or None, + "employment": str(j.get("employment_type_code") or "").strip() or None, }) return rows diff --git a/plugins/readers/rippling.py b/plugins/readers/rippling.py index 866e603..1e98e4d 100644 --- a/plugins/readers/rippling.py +++ b/plugins/readers/rippling.py @@ -12,7 +12,6 @@ from __future__ import annotations import html import json -import logging import re from collections.abc import Callable from typing import Any @@ -22,8 +21,6 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads from ._shared import Get, Row, add_place, parse_dt, yearly_amount -log = logging.getLogger(__name__) - NAME = "rippling" LABEL = "Rippling" KEEP_CASE = True @@ -46,11 +43,6 @@ _SITE = "ats.rippling.com" #: whole; it is also what discovery's patterns take. _SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*") -#: The words of `employmentType.label` (`SALARIED_FT`), split at "_". -_EMPLOYMENT = {"FT": "full_time", "PT": "part_time", "INTERN": "internship", - "INTERNSHIP": "internship", "CONTRACT": "contract", "CONTRACTOR": "contract", - "TEMP": "temporary", "TEMPORARY": "temporary", "SEASONAL": "temporary"} - #: A role's id in the board's own links. _UUID = re.compile(r"[0-9A-Za-z-]{8,64}") @@ -86,8 +78,7 @@ def parse(body: bytes, slug: str) -> list[Row]: role and place, so a role in three cities is three entries with one `uuid` (Rippling's own board: 638 entries, 326 roles); the rows merge them, the places kept as a list and joined with "; " for `location`. The entry has no description, date - or pay: `detail` reads them. `remote` is True only when every place is a - "Remote (...)" one, since a label says nothing else about the work.""" + or pay: `detail` reads them.""" data = json.loads(body) if not isinstance(data, list): raise ValueError("not a Rippling board: the answer is not a list") @@ -114,19 +105,12 @@ def parse(body: bytes, slug: str) -> list[Row]: for job_id, row in roles.items(): row["places"] = places[job_id] row["location"] = "; ".join(p["text"] for p in places[job_id]) - row["remote"] = True if places[job_id] and all( - p["remote"] for p in places[job_id]) else None return list(roles.values()) def _employment(kind: Any) -> str | None: - label = str((kind or {}).get("label") or "").strip().upper() if isinstance(kind, dict) else "" - for word in label.split("_"): - if word in _EMPLOYMENT: - return _EMPLOYMENT[word] - if label: - log.debug("rippling: unknown employmentType %r", label) - return None + """The host's `employmentType.label` (`SALARIED_FT`), as given.""" + return (str(kind.get("label") or "").strip() or None) if isinstance(kind, dict) else None def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: diff --git a/plugins/readers/smartrecruiters.py b/plugins/readers/smartrecruiters.py index 152bb7f..efc56a3 100644 --- a/plugins/readers/smartrecruiters.py +++ b/plugins/readers/smartrecruiters.py @@ -13,15 +13,12 @@ ignores a remote parameter, so remote is not filtered. from __future__ import annotations import json -import logging import re from collections.abc import Callable from typing import Any from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok, table_word - -log = logging.getLogger(__name__) +from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok NAME = "smartrecruiters" LABEL = "SmartRecruiters" @@ -41,17 +38,6 @@ READS = Reads( _API = "api.smartrecruiters.com" -#: `experienceLevel.id` -> the app's levels. The three in the middle name no -#: level ("associate" is half of Bosch's roles) and are known, so they map to -#: none; an id not listed is logged at debug level. -_SENIORITY = {"internship": "intern", "entry_level": "junior", "director": "director", - "executive": "exec", "associate": "", "mid_senior_level": "", - "not_applicable": ""} - -#: `typeOfEmployment.id` -> the app's employment words. -_EMPLOYMENT = {"permanent": "", "intern": "internship", "temporary": "temporary", - "contract": "contract", "part_time": "part_time", "full_time": "full_time"} - #: The page size asked for, which is also the most the host serves per page. PAGE_SIZE = 100 @@ -94,10 +80,9 @@ def parse(body: bytes, slug: str) -> list[Row]: return rows(json.loads(body), slug) -def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None: - """The `id` of the object `job[key]` looked up in `table`; none for a word - the table maps to none, and a word it does not know is logged.""" - return table_word(table, (job.get(key) or {}).get("id"), log, key) +def _word(job: dict[str, Any], key: str) -> str | None: + """The `id` of the object `job[key]`, as given.""" + return str((job.get(key) or {}).get("id") or "").strip() or None def rows(data: dict[str, Any], slug: str) -> list[Row]: @@ -122,7 +107,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: loc_text = ", ".join(part.strip() for part in (loc_text or "").split(",") if part.strip()) place: Place = {"text": loc_text, "remote": location.get("remote")} - country = str(location.get("country") or "").strip().upper() out.append({ "source_id": job_id, "title": title, @@ -134,8 +118,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: "description": "", "remote": location.get("remote"), "salary_text": "", # SmartRecruiters' list endpoint states no pay at all - "country": country if len(country) == 2 else None, - "seniority": _word(j, "experienceLevel", _SENIORITY), - "employment": _word(j, "typeOfEmployment", _EMPLOYMENT), + "country": str(location.get("country") or "").strip() or None, + "seniority": _word(j, "experienceLevel"), + "employment": _word(j, "typeOfEmployment"), }) return out diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py index 96d5858..9747575 100644 --- a/plugins/readers/successfactors.py +++ b/plugins/readers/successfactors.py @@ -59,7 +59,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import NAMES -from ._shared import Get, Page, Row, add_place, country_if_single, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt NAME = "successfactors" LABEL = "SuccessFactors" @@ -86,7 +86,7 @@ KEEP_CASE = True READS = Reads( scope="board", listing="paged", text="detail", complete="stated", places="many", filters=Filters(country="one"), - fields=Fields(posted="trusted", country="stated"), + fields=Fields(posted="trusted"), robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)", ) @@ -342,19 +342,6 @@ def _posted(value: str) -> datetime | None: return parse_dt(value) -def _country_code(meta: dict[str, str]) -> str: - """The address's country as an ISO code: `addressCountry`, or the code that - `_place` reads in a `streetAddress`; '' for neither.""" - code = meta.get("addressCountry", "").strip().upper() - if len(code) == 2: - return code - parts = [part.strip() for part in meta.get("streetAddress", "").split(",")] - for i in range(len(parts) - 1, 0, -1): - if len(parts) - i <= 2 and parts[i] in _COUNTRIES: - return parts[i] - return "" - - def _location(meta: dict[str, str]) -> str: """The address as "City, Country". Vestas and Demant state `addressLocality` and `addressCountry` (a code), the other sites one @@ -373,7 +360,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: """A role's own page: `description` from the block marked `itemprop="description"`, which Novo Nordisk's pages have without the microdata around it, and, from a page that has the `JobPosting` microdata, - `posted` (`datePosted`) and every place with the country's code. A page that + `posted` (`datePosted`) and every place. A page that is not a role's (the site answers 200 for a closed one with its search) has no description block and gives an empty answer.""" text = get(row["url"]).decode("utf-8", "replace") @@ -396,9 +383,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: if places: out["places"] = places out["location"] = "; ".join(p["text"] for p in places) - codes = {_country_code(address) for address in addresses} - if code := country_if_single(codes): - out["country"] = code return out diff --git a/plugins/readers/teamtailor.py b/plugins/readers/teamtailor.py index 075e7e5..937d1f8 100644 --- a/plugins/readers/teamtailor.py +++ b/plugins/readers/teamtailor.py @@ -51,12 +51,6 @@ _HOST = re.compile(r"[a-z0-9-]+(\.[a-z0-9-]+)*") #: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`. _TT = "{https://teamtailor.com/locations}" -#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing -#: about where the role is worked and `temporary` is remote for a spell only, -#: so both stay unknown rather than become "onsite" or "remote". -_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"} - - def site(slug: str) -> str: """The host a career site is served from: `acme` is acme.teamtailor.com, and a slug with a dot is a site on the company's own domain, @@ -141,7 +135,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]: "department": ((item.findtext(f"{_TT}department") or "").strip() or (item.findtext(f"{_TT}role") or "").strip()), "description": html.unescape(strip_html(item.findtext("description") or "")), - "remote": _REMOTE.get(status), + "remote": status or None, "salary_text": "", # Teamtailor's feed states no pay at all }) return out diff --git a/plugins/readers/workable.py b/plugins/readers/workable.py index 99597ed..4ed795c 100644 --- a/plugins/readers/workable.py +++ b/plugins/readers/workable.py @@ -8,14 +8,11 @@ board with roles was seen; jobhunt-hq did not read Workable. from __future__ import annotations import json -import logging from collections.abc import Callable from typing import Any from ._contract import Ask, Fields, Place, Reads -from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt, path_slug_ok, table_word - -log = logging.getLogger(__name__) +from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok NAME = "workable" LABEL = "Workable" @@ -28,17 +25,12 @@ PATTERNS = ( READS = Reads( scope="board", listing="paged", page_size=10, text="none", complete="stated", places="many", - fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"), + fields=Fields(posted="trusted", remote="stated", employment="stated"), robots="api: no robots.txt check needed (public board API), 2026-09-30", ) _API = "apply.workable.com" -#: `type` -> the app's employment words; a type not listed is logged at debug -#: level so the table can grow. -_EMPLOYMENT = {"full": "full_time", "part": "part_time", "contract": "contract", - "temporary": "temporary", "internship": "internship"} - # No probe: Workable's API sits behind a rate limit that a run of guesses # trips, and a Workable site links its board plainly anyway. @@ -67,13 +59,12 @@ def parse(body: bytes, slug: str) -> list[Row]: return rows(json.loads(body), slug) -def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]: - """Every place of a role and the countries' codes among them. `locations` +def _places(job: dict[str, Any], remote: bool) -> list[Place]: + """Every place of a role. `locations` lists them (city, region, country; `hidden` ones are stated too), and `location` is the first alone, the fallback for a role that lists none. A place is remote when the role is.""" found: list[Place] = [] - codes: set[str] = set() listed = [loc for loc in job.get("locations") or [] if isinstance(loc, dict)] for loc in listed or [job.get("location") or {}]: parts: list[str] = [] @@ -83,9 +74,7 @@ def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]: parts.append(part) text = ", ".join(parts) add_place(found, {"text": text, "remote": remote}) - if code := str(loc.get("countryCode") or "").strip().upper(): - codes.add(code) - return found, codes + return found def rows(data: dict[str, Any], slug: str) -> list[Row]: @@ -104,8 +93,9 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: shortcode = (j.get("shortcode") or "").strip() if not title or not shortcode: continue - remote = bool(j.get("remote")) or j.get("workplace") == "remote" - places, codes = _places(j, remote) + workplace = str(j.get("workplace") or "").strip() + remote = bool(j.get("remote")) or workplace == "remote" + places = _places(j, remote) dept = j.get("department") or "" if isinstance(dept, list): dept = ", ".join(d for d in dept if d) @@ -118,9 +108,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: "posted": parse_dt(j.get("published")), "department": dept.strip(), "description": "", - "remote": remote, + "remote": workplace or j.get("remote"), "salary_text": "", # Workable's list endpoint states no pay at all - "country": country_if_single(codes), - "employment": table_word(_EMPLOYMENT, j.get("type"), log, "type"), + "employment": str(j.get("type") or "").strip() or None, }) return out diff --git a/plugins/readers/workday.py b/plugins/readers/workday.py index a2e6ec6..73b08f9 100644 --- a/plugins/readers/workday.py +++ b/plugins/readers/workday.py @@ -29,7 +29,6 @@ from __future__ import annotations import html import json -import logging import re from collections.abc import Callable from datetime import UTC, datetime, timedelta @@ -39,9 +38,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import code_of -from ._shared import Get, Page, Row, add_place, only, parse_dt, table_word - -log = logging.getLogger(__name__) +from ._shared import Get, Page, Row, add_place, only, parse_dt NAME = "workday" LABEL = "Workday" @@ -50,7 +47,7 @@ KEEP_CASE = True READS = Reads( scope="board", listing="paged", page_size=20, ceiling=2000, text="detail", complete="stated", places="many", filters=Filters(country="ids", remote="ids"), - fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"), + fields=Fields(posted="trusted", remote="stated", employment="stated"), robots="read: robots.txt allows // (checked 2026-09-29)", ) @@ -167,7 +164,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]: """The tenant's host ids, read off one page of one role: `country` maps an ISO code to `=` in the facet whose values are countries (most of them, `_COUNTRY_SHARE`; the one with most countries - when two are), and `remote` maps each facet value that `_REMOTE` reads as + when two are), and `remote` maps each facet value that `_REMOTE_FACET` names as remote ("Remote", "Fully Flexible"), in whatever facet, to its id. A tenant with neither states no such dimension, and the runner then does not filter on it.""" @@ -184,7 +181,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]: country = found for v in values: word = " ".join(str(v.get("descriptor") or "").split()) - if v.get("id") and _remote(word) == "remote": + if v.get("id") and word.lower() in _REMOTE_FACET: remote[word.lower()] = f"{name}={v['id']}" out: dict[str, dict[str, str]] = {} if country: @@ -216,11 +213,14 @@ def _today() -> datetime: return datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) -#: `remoteType` -> `store._remote`'s words. Only some tenants send it, and -#: each words it its own way (Zendesk's "Fully Flexible", Axis' "On-Site"); a -#: word not listed, a bare "No" among them, says nothing and stays unknown. -_REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid", - "in office": "onsite", "on-site": "onsite", "onsite": "onsite"} +#: The `remoteType` words a facet names that mean remote, which the `remote` +#: filter looks for; the row keeps the host's word whatever it is (only some +#: tenants send it, each in its own way: Zendesk's "Fully Flexible", Axis' +#: "On-Site"). +_REMOTE_FACET = ("remote", "fully flexible") + +#: The words that put a role's places in an office, for their `remote` flag. +_OFFICE = ("hybrid", "in office", "on-site", "onsite") def _requisition(posting: dict[str, Any], path: str) -> str: @@ -234,12 +234,10 @@ def _requisition(posting: dict[str, Any], path: str) -> str: def _remote(value: Any) -> str | None: - return _REMOTE.get(str(value or "").strip().lower()) + """The host's `remoteType`, as given.""" + return str(value or "").strip() or None -#: `timeType` -> the app's employment words. -_EMPLOYMENT = {"full time": "full_time", "part time": "part_time"} - #: What `locationsText` says of a role at several places: "2 Locations". The #: places arrive with the role's own read. _COUNT = re.compile(r"\d+ locations?", re.I) @@ -259,8 +257,9 @@ def _place(text: Any, remote: str | None) -> Place | None: if len(parts) > 1 and "," not in text and all(parts) \ and not parts[0].lower().startswith(("remote", "hybrid", "on-site", "onsite")): text = ", ".join(reversed(parts)) - return {"text": text, "remote": {"remote": True, "hybrid": False, "onsite": False}.get( - remote or "")} + word = (remote or "").lower() + return {"text": text, "remote": True if word in _REMOTE_FACET + else False if word in _OFFICE else None} def rows(decoded: dict[str, Any], slug: str) -> list[Row]: @@ -318,9 +317,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: if places := _places(posting, remote): out["places"] = places out["location"] = "; ".join(p["text"] for p in places) - if len(places) == 1 and (code := _country_code(posting)): - out["country"] = code - if employment := table_word(_EMPLOYMENT, posting.get("timeType"), log, "timeType"): + if employment := str(posting.get("timeType") or "").strip(): out["employment"] = employment if posted := parse_dt(posting.get("startDate")): out["posted"] = posted @@ -329,15 +326,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: return out -def _country_code(posting: dict[str, Any]) -> str | None: - """The role's country as an ISO code, from the requisition's location. The - code belongs to the main location only, so it is used for a role with one - place.""" - country = (posting.get("jobRequisitionLocation") or {}).get("country") or {} - code = str(country.get("alpha2Code") or "").strip().upper() - return code if len(code) == 2 else None - - def _places(posting: dict[str, Any], remote: str | None) -> list[Place]: """The main location and the additional ones, each once, which replace the listing's "2 Locations". A single location that does not name its country diff --git a/tests/test_ats.py b/tests/test_ats.py index 3f855fe..74d5b1a 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -132,10 +132,10 @@ def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None: assert [r["source_id"] for r in rows] == ["6125026", "6125041"] assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/" assert rows[0]["location"] == "France" - assert rows[0]["remote"] is True + assert rows[0]["remote"] == "remote" assert rows[0]["department"] == "Product" assert rows[1]["location"] == "Berlin, Germany" - assert rows[1]["remote"] is False + assert rows[1]["remote"] == "hybrid" assert rows[0]["description"] == "" # the list endpoint carries no text assert rows[0]["salary_text"] == "" # Workable's list endpoint states no pay at all @@ -173,8 +173,9 @@ def test_personio_builds_the_job_url_and_reads_cdata_description() -> None: assert [r["source_id"] for r in rows] == ["6001", "6002", "6003"] assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001" assert rows[0]["description"] == "Grow our pipeline." - assert rows[0]["remote"] is True # "Remote - Germany" - assert rows[1]["remote"] is None # "Munich" + assert rows[0]["places"][0]["remote"] is True # "Remote - Germany" + assert rows[1]["places"][0]["remote"] is None # "Munich" + assert "remote" not in rows[0] # the feed states none def test_personio_reads_salary_information_when_present() -> None: @@ -291,9 +292,9 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None: assert country_of(rows[0]["location"]) == "SE" -def test_teamtailor_maps_remote_status_and_falls_back_to_role_for_department() -> None: +def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None: rows = ats._parse_teamtailor(TEAMTAILOR, "kry") - assert [r["remote"] for r in rows] == ["hybrid", "remote", None] # none says nothing + assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them assert rows[1]["title"] == "Läkare" # trimmed assert rows[1]["department"] == "Software Development" assert ats.classify(rows[1]["department"], "Sales Lead") == "sales" diff --git a/tests/test_ats_oracle.py b/tests/test_ats_oracle.py index e5b4123..4442319 100644 --- a/tests/test_ats_oracle.py +++ b/tests/test_ats_oracle.py @@ -59,19 +59,19 @@ def test_the_locations_are_the_primary_and_the_secondary_ones() -> None: assert rows[2]["location"] == "India" -def test_every_place_is_kept_with_its_work_mode_and_the_countrys_code() -> None: +def test_every_place_is_kept_with_its_work_mode() -> None: rows = _rows() assert [p["text"] for p in rows[1]["places"]] == [ "Malaysia", "Philippines", "Indonesia", "Thailand"] assert [p["remote"] for p in rows[0]["places"]] == [False, False] # hybrid assert [p["remote"] for p in rows[4]["places"]] == [True] # remote assert [p["remote"] for p in rows[3]["places"]] == [None] # states none - # One country when the places share one, none over several. - assert [r["country"] for r in rows] == [None, None, "IN", "DE", "US"] + assert all("country" not in r for r in rows) # prep derives it from the places -def test_remote_follows_the_workplace_type_code() -> None: - assert [r["remote"] for r in _rows()] == ["hybrid", "hybrid", "onsite", None, "remote"] +def test_remote_is_the_workplace_type_code_as_given() -> None: + assert [r["remote"] for r in _rows()] == [ + "ORA_HYBRID", "ORA_HYBRID", "ORA_ON_SITE", None, "ORA_REMOTE"] def test_a_department_is_taken_where_a_tenant_fills_one() -> None: diff --git a/tests/test_ats_personio.py b/tests/test_ats_personio.py index cb644b7..e84f35c 100644 --- a/tests/test_ats_personio.py +++ b/tests/test_ats_personio.py @@ -3,17 +3,14 @@ No network: the fixtures are saved feeds.""" from __future__ import annotations -import logging import sys from pathlib import Path -import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from app.models import SENIORITIES # noqa: E402 from plugins.readers import Ask, personio # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -37,23 +34,12 @@ def test_additional_offices_are_places_of_the_role_each_once() -> None: assert [p["text"] for p in first["places"]] == ["Remote - Germany", "Berlin", "Hamburg"] assert [p["remote"] for p in first["places"]] == [True, None, None] assert first["location"] == "Remote - Germany; Berlin; Hamburg" - assert first["remote"] is True # one of the places is remote + assert "remote" not in first # the feed states none assert _rows()[1]["places"] == [{"text": "Munich", "remote": None}] - assert _rows()[1]["remote"] is None -def test_the_stated_level_and_employment_are_mapped() -> None: +def test_the_stated_level_and_employment_are_the_hosts_words() -> None: rows = _rows() - assert [r["seniority"] for r in rows] == ["junior", "exec", "intern"] - # permanent + full-time; permanent + full-time; working student wins over part-time - assert [r["employment"] for r in rows] == ["full_time", "full_time", "internship"] - - -def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged( - caplog: pytest.LogCaptureFixture) -> None: - assert {v for v in personio._SENIORITY.values() if v} <= set(SENIORITIES) - with caplog.at_level(logging.DEBUG, logger=personio.__name__): - assert personio._seniority("wizard") is None - assert personio._seniority("experienced") is None # known, names no level - assert personio._seniority("") is None - assert [r.getMessage() for r in caplog.records] == ["personio: unknown seniority 'wizard'"] + assert [r["seniority"] for r in rows] == ["entry-level", "executive", "student"] + # permanent + full-time gives the schedule; a working student is the contract's word + assert [r["employment"] for r in rows] == ["full-time", "full-time", "working_student"] diff --git a/tests/test_ats_phenom.py b/tests/test_ats_phenom.py index 0decca2..0cd40c8 100644 --- a/tests/test_ats_phenom.py +++ b/tests/test_ats_phenom.py @@ -226,7 +226,7 @@ def test_detail_falls_back_to_the_page_state_where_there_is_no_job_posting() -> got = phenom.detail(_get_of("phenom-detail-3.html", []), SLUG, row) assert got["description"].startswith("Craft: Games Design & Production Job Description:") assert "<" not in got["description"] and "&" not in got["description"] - assert got["remote"] == "onsite" + assert got["remote"] == "On-site" assert "posted" not in got and set(got) <= set(ats.DETAIL_FIELDS) diff --git a/tests/test_ats_pinpoint.py b/tests/test_ats_pinpoint.py index bffc5cb..e091caf 100644 --- a/tests/test_ats_pinpoint.py +++ b/tests/test_ats_pinpoint.py @@ -161,9 +161,6 @@ def test_a_role_has_its_one_place_with_the_work_mode_of_the_role() -> None: assert rows[0]["location"] == "Bucharest" -def test_employment_is_read_from_the_type_and_an_unknown_one_is_left() -> None: +def test_employment_is_the_hosts_type_as_given() -> None: rows = pinpoint.parse((FIXTURES / "pinpoint.json").read_bytes(), "xeneta") - assert [r["employment"] for r in rows] == ["full_time", "temporary"] - assert pinpoint._employment("permanent_part_time") == "part_time" - assert pinpoint._employment("internship") == "internship" - assert pinpoint._employment("something_new") is None + assert [r["employment"] for r in rows] == ["permanent_full_time", "fixed_term_full_time"] diff --git a/tests/test_ats_recruitee.py b/tests/test_ats_recruitee.py index 33f031e..cb20ea8 100644 --- a/tests/test_ats_recruitee.py +++ b/tests/test_ats_recruitee.py @@ -4,17 +4,14 @@ bunq's board, read on 2026-09-30.""" from __future__ import annotations -import logging import sys from pathlib import Path -import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from app.models import SENIORITIES # noqa: E402 from plugins.readers import Ask, recruitee # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -27,19 +24,18 @@ def _rows() -> list[dict]: def test_the_declaration_says_what_the_feed_gives() -> None: reads = recruitee.READS assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing") - assert reads.fields.posted == "trusted" and reads.fields.country == "stated" + assert reads.fields.posted == "trusted" and reads.fields.country == "none" assert reads.robots.startswith("api: no robots.txt check needed") assert recruitee.feed_url("acme", Ask()) == "https://acme.recruitee.com/api/offers/" -def test_every_location_is_a_place_and_the_country_is_stated_when_they_share_one() -> None: +def test_every_location_is_a_place_and_no_country_is_stated() -> None: first, _remote, berlin = _rows() assert [p["text"] for p in first["places"]] == [ "Amsterdam, Noord-Holland, Netherlands", "Sofia, Bulgaria"] assert first["location"] == "Amsterdam, Noord-Holland, Netherlands; Sofia, Bulgaria" - assert first["country"] is None # NL and BG + assert "country" not in first # prep derives it from the places assert berlin["places"] == [{"text": "Berlin, Germany", "remote": None}] - assert berlin["country"] == "DE" def test_remote_job_is_not_a_place() -> None: @@ -48,28 +44,16 @@ def test_remote_job_is_not_a_place() -> None: assert remote["location"] == "Remote" offer = {"id": 1, "title": "T", "careers_url": "https://a.example/o/t", "location": "Remote job", "remote": False} - assert recruitee._places(offer) == ([], set()) + assert recruitee._places(offer) == [] def test_an_offer_without_a_locations_list_falls_back_to_its_location() -> None: offer = {"location": "Lisbon, Portugal", "country_code": "pt"} - assert recruitee._places(offer) == ([{"text": "Lisbon, Portugal", "remote": None}], {"PT"}) + assert recruitee._places(offer) == [{"text": "Lisbon, Portugal", "remote": None}] -def test_the_stated_level_and_employment_are_mapped() -> None: +def test_the_stated_level_and_employment_are_the_hosts_codes() -> None: rows = _rows() - assert [r["seniority"] for r in rows] == [None, "junior", "intern"] - assert [r["employment"] for r in rows] == ["full_time", "part_time", "internship"] - assert recruitee._employment("fulltime_fixed_term") == "temporary" - assert recruitee._employment("freelance") == "contract" - - -def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged( - caplog: pytest.LogCaptureFixture) -> None: - assert {v for v in recruitee._SENIORITY.values() if v} <= set(SENIORITIES) - with caplog.at_level(logging.DEBUG, logger=recruitee.__name__): - assert recruitee._seniority("senior_level") is None - assert recruitee._employment("odd_job") is None - assert [r.getMessage() for r in caplog.records] == [ - "recruitee: unknown experience_code 'senior_level'", - "recruitee: unknown employment_type_code 'odd_job'"] + assert [r["seniority"] for r in rows] == ["experienced", "entry_level", "student_school"] + assert [r["employment"] for r in rows] == [ + "fulltime_permanent", "parttime_permanent", "internship"] diff --git a/tests/test_ats_rippling.py b/tests/test_ats_rippling.py index 74da897..0f10598 100644 --- a/tests/test_ats_rippling.py +++ b/tests/test_ats_rippling.py @@ -41,21 +41,18 @@ def test_rows_merge_one_entry_per_place_into_one_role() -> None: assert first["description"] == "" and first["salary_text"] == "" and first["posted"] is None -def test_remote_is_true_only_when_every_place_is_a_remote_one() -> None: +def test_a_remote_place_is_flagged_and_the_role_states_no_remote() -> None: by_id = {r["source_id"][:8]: r for r in rippling.parse(BOARD, "rippling")} - assert by_id["46b4af52"]["remote"] is True and by_id["19ae5b34"]["remote"] is True - assert by_id["75ad50c6"]["remote"] is None # an office label says nothing either way - mixed = json.dumps([ - {"uuid": "abcdefgh-1", "name": "A", "workLocation": {"label": "Remote (Germany)"}}, - {"uuid": "abcdefgh-1", "name": "A", "workLocation": {"label": "Berlin"}}]).encode() - assert rippling.parse(mixed, "x")[0]["remote"] is None + assert [p["remote"] for p in by_id["46b4af52"]["places"]] == [True] + assert [p["remote"] for p in by_id["75ad50c6"]["places"]] == [None] # an office label + assert all("remote" not in r for r in by_id.values()) # prep derives it from the places def test_prisma_board_reads_two_roles() -> None: rows = rippling.parse(PRISMA, "prisma-careers") - assert [(r["title"], r["department"], r["location"], r["remote"]) for r in rows] == [ - ("General Applications: Engineering", "Engineering", "Remote (Germany)", True), - ("General Applications: Product", "Product", "Remote (Germany)", True)] + assert [(r["title"], r["department"], r["location"]) for r in rows] == [ + ("General Applications: Engineering", "Engineering", "Remote (Germany)"), + ("General Applications: Product", "Product", "Remote (Germany)")] def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None: @@ -92,7 +89,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non assert "<" not in found["description"] assert found["location"] == "San Francisco, CA; Seattle, WA; New York, NY" assert [p["text"] for p in found["places"]] == ["San Francisco, CA", "Seattle, WA", "New York, NY"] - assert found["employment"] == "full_time" # employmentType.label SALARIED_FT + assert found["employment"] == "SALARIED_FT" # employmentType.label assert found["posted"] is not None and found["posted"].isoformat() == \ "2025-04-07T12:29:19.348000-07:00" assert (found["salary_min"], found["salary_max"], found["currency"]) == (216000, 378000, "USD") @@ -203,13 +200,12 @@ def test_discovery_finds_the_board_slug() -> None: ("rippling", "prisma-careers"), ("rippling", "Acme"), ("rippling", "beta")] -def test_a_role_at_several_places_keeps_each_once_and_is_remote_only_if_all_are() -> None: +def test_a_role_at_several_places_keeps_each_once() -> None: rows = {r["source_id"]: r for r in rippling.parse(BOARD, "rippling")} director = rows[DIRECTOR] assert [p["text"] for p in director["places"]] == [ "San Francisco, CA", "Seattle, WA", "New York, NY"] assert director["location"] == "San Francisco, CA; Seattle, WA; New York, NY" - assert director["remote"] is None remote = next(r for r in rows.values() if r["places"] == [ {"text": "Remote (United States)", "remote": True}]) - assert remote["remote"] is True + assert remote["places"][0]["remote"] is True diff --git a/tests/test_ats_smartrecruiters.py b/tests/test_ats_smartrecruiters.py index aa616c7..e2db36e 100644 --- a/tests/test_ats_smartrecruiters.py +++ b/tests/test_ats_smartrecruiters.py @@ -4,17 +4,14 @@ No network: the shapes are those of Bosch's board, read on 2026-09-30.""" from __future__ import annotations import json -import logging import sys from pathlib import Path -import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from app.models import SENIORITIES # noqa: E402 from plugins.readers import Ask, smartrecruiters # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -54,38 +51,23 @@ def test_page_reads_by_offset_and_takes_the_ask() -> None: def test_the_place_is_one_clean_line_with_its_remote_flag_and_country() -> None: [row] = _rows(_posting()) assert row["places"] == [{"text": "Hatvan, Hungary", "remote": False}] # ", ," gone - assert row["location"] == "Hatvan, Hungary" and row["country"] == "HU" + assert row["location"] == "Hatvan, Hungary" and row["country"] == "hu" [remote] = _rows(_posting(location={"remote": True, "fullLocation": "Remote", "country": ""})) assert remote["places"] == [{"text": "Remote", "remote": True}] and remote["country"] is None [none] = _rows(_posting(location={})) assert none["places"] == [] and none["location"] == "" -def test_the_stated_level_and_employment_are_mapped() -> None: +def test_the_stated_level_and_employment_are_the_hosts_words() -> None: [row] = _rows(_posting(experienceLevel={"id": "internship", "label": "Internship"}, typeOfEmployment={"id": "intern", "label": "Intern"})) - assert (row["seniority"], row["employment"]) == ("intern", "internship") - levels = {"entry_level": "junior", "director": "director", "executive": "exec", - "associate": None, "mid_senior_level": None, "not_applicable": None} - for word, want in levels.items(): - [row] = _rows(_posting(experienceLevel={"id": word})) - assert row["seniority"] == want, word - [row] = _rows(_posting(typeOfEmployment={"id": "permanent"})) - assert row["employment"] is None + assert (row["seniority"], row["employment"]) == ("internship", "intern") + [row] = _rows(_posting(experienceLevel={"id": "mid_senior_level"})) + assert row["seniority"] == "mid_senior_level" [row] = _rows(_posting()) assert row["seniority"] is None and row["employment"] is None -def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged( - caplog: pytest.LogCaptureFixture) -> None: - assert {v for v in smartrecruiters._SENIORITY.values() if v} <= set(SENIORITIES) - with caplog.at_level(logging.DEBUG, logger=smartrecruiters.__name__): - [row] = _rows(_posting(experienceLevel={"id": "guru"})) - assert row["seniority"] is None - assert [r.getMessage() for r in caplog.records] == [ - "smartrecruiters: unknown experienceLevel 'guru'"] - - def test_the_saved_page_still_reads() -> None: data = json.loads((FIXTURES / "smartrecruiters.json").read_bytes()) assert [r["location"] for r in smartrecruiters.rows(data, "Acme")] == ["Berlin, Germany", diff --git a/tests/test_ats_successfactors.py b/tests/test_ats_successfactors.py index 044b62d..ea237b7 100644 --- a/tests/test_ats_successfactors.py +++ b/tests/test_ats_successfactors.py @@ -315,7 +315,7 @@ def _detail(name: str, url: str = "https://careers.vestas.com/job/x/1431772533/" return successfactors.detail(get, "careers.vestas.com", {"url": url}) # type: ignore[arg-type] -def test_detail_reads_the_description_the_date_and_the_country_code_of_an_address() -> None: +def test_detail_reads_the_description_the_date_and_the_places_of_an_address() -> None: got = _detail("successfactors-job.html") assert got["description"].startswith("Senior Management | Full-Time | Commercial Are you") assert "<" not in got["description"] @@ -324,7 +324,7 @@ def test_detail_reads_the_description_the_date_and_the_country_code_of_an_addres # The locality and the country's name; the region is cut to "Seou" by the site. assert got["location"] == "Seoul, South Korea" assert got["places"] == [{"text": "Seoul, South Korea", "remote": None}] - assert got["country"] == "KR" + assert "country" not in got # prep derives it from the places def test_detail_keeps_every_address_of_a_role_at_several_places() -> None: @@ -333,15 +333,6 @@ def test_detail_keeps_every_address_of_a_role_at_several_places() -> None: assert [p["text"] for p in got["places"]] == [ "Seoul, South Korea", "Busan, South Korea", "Aarhus, Denmark"] assert got["location"] == "Seoul, South Korea; Busan, South Korea; Aarhus, Denmark" - assert "country" not in got # KR and DK: no single country for the role - - -def test_detail_states_the_country_when_every_address_is_in_it() -> None: - text = _read("successfactors-places-detail.html").replace( - 'content="Aarhus, DK, 8200"', 'content="Daegu, KR, 41200"') - got = successfactors.detail(lambda url, **kw: text.encode(), "careers.vestas.com", - {"url": "https://careers.vestas.com/job/x/1/"}) - assert got["country"] == "KR" def test_the_listing_place_is_the_roles_only_place_until_its_page_is_read() -> None: @@ -351,11 +342,10 @@ def test_the_listing_place_is_the_roles_only_place_until_its_page_is_read() -> N for r in rows) -def test_detail_reads_the_country_from_a_street_address() -> None: +def test_detail_reads_the_place_from_a_street_address() -> None: """Systematic and most other sites give one `streetAddress`, "City, CC, zip".""" got = _detail("successfactors-job-street.html") assert got["location"] == "Bern, Switzerland" - assert got["country"] == "CH" assert got["posted"] == datetime(2026, 9, 28, 2, tzinfo=UTC) assert got["description"].startswith("Senior Solution Engineer Defence") diff --git a/tests/test_ats_workable.py b/tests/test_ats_workable.py index db5291f..674e8a5 100644 --- a/tests/test_ats_workable.py +++ b/tests/test_ats_workable.py @@ -5,11 +5,9 @@ No network: the fixture has the shape of Hugging Face's board, read on from __future__ import annotations import json -import logging import sys from pathlib import Path -import pytest ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: @@ -49,21 +47,15 @@ def test_every_location_of_a_role_is_a_place_once() -> None: assert [p["text"] for p in first["places"]] == ["Berlin, Germany", "Munich, Bavaria, Germany", "Paris, France"] assert first["location"] == "Berlin, Germany; Munich, Bavaria, Germany; Paris, France" - assert first["country"] is None # DE and FR + assert "country" not in first # prep derives it from the places assert [p["remote"] for p in first["places"]] == [False] * 3 assert remote["places"] == [{"text": "France", "remote": True}] - assert remote["country"] == "FR" # A list-less answer (the older shape) still gives its one place. assert oslo["places"] == [{"text": "Oslo, Norway", "remote": False}] - assert oslo["country"] == "NO" -def test_the_stated_hours_are_mapped_and_an_unknown_type_is_logged( - caplog: pytest.LogCaptureFixture) -> None: - with caplog.at_level(logging.DEBUG, logger=workable.__name__): - rows = _rows() - assert [r["employment"] for r in rows] == ["full_time", "part_time", None] - assert [r.getMessage() for r in caplog.records] == ["workable: unknown type 'volunteer'"] +def test_the_stated_hours_are_the_hosts_word() -> None: + assert [r["employment"] for r in _rows()] == ["full", "part", "volunteer"] def test_the_shape_of_a_real_list_answer_is_read() -> None: diff --git a/tests/test_ats_workday.py b/tests/test_ats_workday.py index dadb2b3..164d175 100644 --- a/tests/test_ats_workday.py +++ b/tests/test_ats_workday.py @@ -54,8 +54,8 @@ def test_rows_read_the_listing_of_a_tenant_that_sends_remote_type() -> None: "Associate-Solutions-Architect_R35358") assert first["posted"] == TODAY assert first["description"] == "" and first["department"] == "" and first["salary_text"] == "" - # "Fully Flexible" is Zendesk's word for remote. - assert [r["remote"] for r in rows] == ["remote", "onsite", "hybrid", "remote", "hybrid"] + # The host's own words, "Fully Flexible" among them; prep maps them. + assert [r["remote"] for r in rows] == ["Fully Flexible", "In Office", "Hybrid", "Fully Flexible", "Hybrid"] def test_rows_of_a_tenant_without_remote_type_leave_remote_unknown() -> None: @@ -80,7 +80,7 @@ def test_rows_of_a_site_called_jobs_state_the_requisition_id_and_no_date_past_30 assert rows[1]["places"] == [{"text": "Remote US TX", "remote": True}] assert rows[2]["url"].startswith("https://redhat.wd5.myworkdayjobs.com/jobs/job/") assert rows[3]["posted"] is None # "Posted 30+ Days Ago" says only that it is older - assert [r["remote"] for r in rows] == ["onsite", "remote", "remote", "hybrid"] + assert [r["remote"] for r in rows] == ["Onsite", "Remote", "Remote", "Hybrid"] @pytest.mark.parametrize(("text", "expected"), [ @@ -275,13 +275,12 @@ def test_detail_reads_the_description_date_remote_and_place_of_the_role() -> Non SLUG, row) assert asked == [f"https://{HOST}/wday/cxs/zendesk/zendesk" "/job/Remote-Mexico-City-Mexico/Associate-Solutions-Architect_R35358"] - assert set(got) == {"description", "location", "places", "posted", "remote", "country", - "employment"} + assert set(got) == {"description", "location", "places", "posted", "remote", "employment"} assert got["posted"] == datetime(2026, 9, 29, tzinfo=UTC) - assert got["remote"] == "remote" + assert got["remote"] == "Fully Flexible" assert got["location"] == "Remote, Mexico City, Mexico" assert got["places"] == [{"text": "Remote, Mexico City, Mexico", "remote": True}] - assert (got["country"], got["employment"]) == ("MX", "full_time") + assert got["employment"] == "Full time" and "country" not in got assert got["description"].startswith("Job Description") or len(got["description"]) > 200 assert "<" not in got["description"] and " " not in got["description"] assert " " not in got["description"] @@ -296,7 +295,7 @@ def test_detail_names_every_location_of_a_role_listed_as_several() -> None: "/job/Remote-Italy/Telco-Cloud-Consultant_R-059566-2"] assert got["location"] == "Remote Italy; Remote Spain" assert [p["text"] for p in got["places"]] == ["Remote Italy", "Remote Spain"] - assert "country" not in got # the code belongs to the main location alone + assert "country" not in got def _detail(**info: Any) -> bytes: @@ -337,9 +336,9 @@ def test_a_country_dash_city_place_is_turned_round() -> None: assert workday._place("12 Locations", None) is None -def test_detail_takes_no_word_for_remote_it_does_not_know() -> None: +def test_detail_hands_on_a_word_for_remote_it_does_not_know() -> None: row = workday.rows(_load("workday.json"), SLUG)[0] - assert "remote" not in workday.detail(_get_of(_detail(remoteType="No"), []), SLUG, row) + assert workday.detail(_get_of(_detail(remoteType="No"), []), SLUG, row)["remote"] == "No" def test_detail_refuses_a_row_that_is_not_a_role_of_the_board() -> None: @@ -383,7 +382,7 @@ def test_detail_is_read_through_the_getter_ats_binds( monkeypatch.setattr(ats, "_get", fake_get) row = workday.rows(_load("workday.json"), SLUG)[0] got = workday.detail(ats._getter(object(), "workday", SLUG), SLUG, row) # type: ignore[arg-type] - assert got["remote"] == "remote" + assert got["remote"] == "Fully Flexible" assert seen[0][:2] == ("workday", SLUG) diff --git a/tests/test_prep_hosts_oracle.py b/tests/test_prep_hosts_oracle.py new file mode 100644 index 0000000..e49b28c --- /dev/null +++ b/tests/test_prep_hosts_oracle.py @@ -0,0 +1,12 @@ +"""Oracle's word tables for prep: the host's codes, and what they mean.""" + +from __future__ import annotations + +import importlib + +oracle = importlib.import_module("app.prep.hosts.oracle") + + +def test_workplace_type_codes_read() -> None: + assert oracle.REMOTE == {"ora_remote": "remote", "ora_hybrid": "hybrid", + "ora_on_site": "onsite"} diff --git a/tests/test_prep_hosts_personio.py b/tests/test_prep_hosts_personio.py new file mode 100644 index 0000000..1ae6157 --- /dev/null +++ b/tests/test_prep_hosts_personio.py @@ -0,0 +1,22 @@ +"""Personio's word tables for prep: the feed's own words, and what they mean.""" + +from __future__ import annotations + +import importlib + +personio = importlib.import_module("app.prep.hosts.personio") + + +def test_seniority_words_read() -> None: + seniority = personio.SENIORITY + assert [seniority[w] for w in ("entry-level", "executive", "student")] == [ + "junior", "exec", "intern"] + assert seniority["experienced"] == "" # known, names no level + assert "wizard" not in seniority + + +def test_employment_words_read_from_the_contract_or_the_schedule() -> None: + employment = personio.EMPLOYMENT + assert employment["working_student"] == "internship" and employment["freelance"] == "contract" + assert employment["permanent"] == "" + assert employment["full-time"] == "full_time" and employment["part-time"] == "part_time" diff --git a/tests/test_prep_hosts_phenom.py b/tests/test_prep_hosts_phenom.py new file mode 100644 index 0000000..38e84f2 --- /dev/null +++ b/tests/test_prep_hosts_phenom.py @@ -0,0 +1,12 @@ +"""Phenom's word table for prep: the words a page's `checkRemote` says.""" + +from __future__ import annotations + +import importlib + +phenom = importlib.import_module("app.prep.hosts.phenom") + + +def test_check_remote_words_read() -> None: + assert phenom.REMOTE["on-site"] == "onsite" # King's pages + assert phenom.REMOTE["remote"] == "remote" and phenom.REMOTE["hybrid"] == "hybrid" diff --git a/tests/test_prep_hosts_pinpoint.py b/tests/test_prep_hosts_pinpoint.py new file mode 100644 index 0000000..26d0727 --- /dev/null +++ b/tests/test_prep_hosts_pinpoint.py @@ -0,0 +1,24 @@ +"""Pinpoint's word tables for prep: `workplace_type` and `employment_type`.""" + +from __future__ import annotations + +import importlib + +pinpoint = importlib.import_module("app.prep.hosts.pinpoint") + + +def _match(code: str) -> str | None: + """The first entry of `MATCH` found in the lowercased code.""" + code = code.strip().lower() + return next((value for fragment, value in pinpoint.MATCH if fragment in code), None) + + +def test_workplace_types_read() -> None: + assert pinpoint.REMOTE == {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", + "remote": "remote"} + + +def test_employment_types_read_by_their_fragments() -> None: + assert [_match(w) for w in ("permanent_full_time", "fixed_term_full_time", + "permanent_part_time", "internship", "something_new")] == [ + "full_time", "temporary", "part_time", "internship", None] diff --git a/tests/test_prep_hosts_recruitee.py b/tests/test_prep_hosts_recruitee.py new file mode 100644 index 0000000..3d4072e --- /dev/null +++ b/tests/test_prep_hosts_recruitee.py @@ -0,0 +1,27 @@ +"""Recruitee's word tables for prep: the host's codes, and what they mean.""" + +from __future__ import annotations + +import importlib + +recruitee = importlib.import_module("app.prep.hosts.recruitee") + + +def _match(word: str) -> str | None: + """The first entry of `MATCH` found among the words of a code.""" + words = set(word.strip().lower().replace("-", "_").split("_")) + return next((value for fragment, value in recruitee.MATCH if fragment in words), None) + + +def test_experience_codes_read() -> None: + seniority = recruitee.SENIORITY + assert [seniority[w] for w in ("student_school", "entry_level", "mid_level")] == [ + "intern", "junior", "mid"] + assert seniority["experienced"] == "" + assert "senior_level" not in seniority + + +def test_employment_type_codes_read_by_their_words() -> None: + assert [_match(w) for w in ("fulltime_permanent", "parttime_permanent", "internship", + "fulltime_fixed_term", "freelance", "odd_job")] == [ + "full_time", "part_time", "internship", "temporary", "contract", None] diff --git a/tests/test_prep_hosts_rippling.py b/tests/test_prep_hosts_rippling.py new file mode 100644 index 0000000..756dd67 --- /dev/null +++ b/tests/test_prep_hosts_rippling.py @@ -0,0 +1,19 @@ +"""Rippling's word table for prep: the words of `employmentType.label`.""" + +from __future__ import annotations + +import importlib + +rippling = importlib.import_module("app.prep.hosts.rippling") + + +def _match(label: str) -> str | None: + """The first entry of `MATCH` found among the "_"-separated words of a label.""" + words = set(label.strip().lower().split("_")) + return next((value for token, value in rippling.MATCH if token in words), None) + + +def test_employment_labels_read_by_their_words() -> None: + assert [_match(w) for w in ("SALARIED_FT", "HOURLY_PT", "INTERN", "CONTRACTOR", + "SEASONAL", "ODD")] == [ + "full_time", "part_time", "internship", "contract", "temporary", None] diff --git a/tests/test_prep_hosts_smartrecruiters.py b/tests/test_prep_hosts_smartrecruiters.py new file mode 100644 index 0000000..8390c0d --- /dev/null +++ b/tests/test_prep_hosts_smartrecruiters.py @@ -0,0 +1,22 @@ +"""SmartRecruiters' word tables for prep: the host's ids, and what they mean.""" + +from __future__ import annotations + +import importlib + +smartrecruiters = importlib.import_module("app.prep.hosts.smartrecruiters") + + +def test_experience_levels_read() -> None: + seniority = smartrecruiters.SENIORITY + assert [seniority[w] for w in ("internship", "entry_level", "director", "executive")] == [ + "intern", "junior", "director", "exec"] + # Known, and naming no level. + assert [seniority[w] for w in ("associate", "mid_senior_level", "not_applicable")] == [""] * 3 + assert "guru" not in seniority + + +def test_employment_types_read() -> None: + employment = smartrecruiters.EMPLOYMENT + assert employment["intern"] == "internship" and employment["permanent"] == "" + assert employment["full_time"] == "full_time" and employment["part_time"] == "part_time" diff --git a/tests/test_prep_hosts_teamtailor.py b/tests/test_prep_hosts_teamtailor.py new file mode 100644 index 0000000..2856866 --- /dev/null +++ b/tests/test_prep_hosts_teamtailor.py @@ -0,0 +1,14 @@ +"""Teamtailor's word table for prep: the feed's `remoteStatus`.""" + +from __future__ import annotations + +import importlib + +teamtailor = importlib.import_module("app.prep.hosts.teamtailor") + + +def test_remote_status_words_read() -> None: + remote = teamtailor.REMOTE + assert [remote[w] for w in ("fully", "hybrid", "onsite")] == ["remote", "hybrid", "onsite"] + # Known, and saying nothing about where the role is worked. + assert remote["none"] == "" and remote["temporary"] == "" diff --git a/tests/test_prep_hosts_workable.py b/tests/test_prep_hosts_workable.py new file mode 100644 index 0000000..268bdf0 --- /dev/null +++ b/tests/test_prep_hosts_workable.py @@ -0,0 +1,18 @@ +"""Workable's word tables for prep: the host's own words, lowercased, and what they mean.""" + +from __future__ import annotations + +import importlib + +workable = importlib.import_module("app.prep.hosts.workable") + + +def test_workplace_words_read() -> None: + assert workable.REMOTE == {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite"} + + +def test_type_words_read() -> None: + assert workable.EMPLOYMENT == {"full": "full_time", "part": "part_time", + "contract": "contract", "temporary": "temporary", + "internship": "internship"} + assert "volunteer" not in workable.EMPLOYMENT # an unknown word is prep's to report diff --git a/tests/test_prep_hosts_workday.py b/tests/test_prep_hosts_workday.py new file mode 100644 index 0000000..a23d698 --- /dev/null +++ b/tests/test_prep_hosts_workday.py @@ -0,0 +1,21 @@ +"""Workday's word tables for prep: the host's own words, lowercased, and what they mean.""" + +from __future__ import annotations + +import importlib + + +workday = importlib.import_module("app.prep.hosts.workday") + + +def test_remote_type_words_of_the_tenants_read() -> None: + remote = workday.REMOTE + # Zendesk's "Fully Flexible", Axis' "On-Site", Red Hat's "Onsite". + assert [remote[w] for w in ("remote", "fully flexible", "hybrid", "in office", + "on-site", "onsite")] == [ + "remote", "remote", "hybrid", "onsite", "onsite", "onsite"] + assert "no" not in remote # a bare "No" says nothing + + +def test_time_type_words_read() -> None: + assert workday.EMPLOYMENT == {"full time": "full_time", "part time": "part_time"} diff --git a/tests/test_readers_adzuna.py b/tests/test_readers_adzuna.py index ef378c5..a50a036 100644 --- a/tests/test_readers_adzuna.py +++ b/tests/test_readers_adzuna.py @@ -46,7 +46,7 @@ def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pa assert "description" not in first assert first["company"] == "Acme GmbH" assert first["places"] == [{"text": "Berlin, Deutschland", "remote": None}] - assert first["country"] == "DE" and first["remote"] == "remote" + assert first["country"] == "DE" and "remote" not in first assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR") assert (predicted["salary_min"], predicted["currency"]) == (None, None) diff --git a/tests/test_readers_platsbanken.py b/tests/test_readers_platsbanken.py index 4adc47b..88d5a37 100644 --- a/tests/test_readers_platsbanken.py +++ b/tests/test_readers_platsbanken.py @@ -61,12 +61,12 @@ def test_it_reads_title_company_date_and_the_full_text() -> None: first, _no_town, remote = platsbanken.rows(decoded, "") assert first["title"] == "Postdoctoral Researcher" assert first["company"] == "European Spallation Source Eric" - assert first["country"] == "SE" and first["remote"] is None + assert first["country"] == "SE" and first["remote"] == "Arbete på plats" assert first["url"] == "https://arbetsformedlingen.se/platsbanken/annonser/31457942" assert first["posted"].date().isoformat() == "2026-09-09" assert first["description"].startswith("The European Spallation Source") assert "teaser" not in first - assert (remote["remote"], remote["company"], remote["posted"]) == ("remote", "", None) + assert (remote["remote"], remote["company"], remote["posted"]) == (None, "", None) def test_it_sends_the_region_the_offset_and_no_region_for_all_of_sweden() -> None: diff --git a/tests/test_readers_shared.py b/tests/test_readers_shared.py index 91b3474..e03529f 100644 --- a/tests/test_readers_shared.py +++ b/tests/test_readers_shared.py @@ -2,6 +2,7 @@ from __future__ import annotations +import importlib import logging import sys from pathlib import Path @@ -24,6 +25,16 @@ from plugins.readers._shared import ( # noqa: E402 #: The module-level names a reader keeps its employment words in. _TABLES = ("_EMPLOYMENT", "_CONTRACT", "_SCHEDULE") +#: The hosts whose word tables are prep's, in `app/prep/hosts/.py`. +_PREP_HOSTS = ("oracle", "personio", "phenom", "pinpoint", "recruitee", "rippling", + "smartrecruiters", "teamtailor", "workable", "workday") + + +def _prep_table(name: str, table_name: str): + """A word table of `app.prep.hosts.`, or None where it has none.""" + module = importlib.import_module(f"app.prep.hosts.{name}") + return getattr(module, table_name, None) + def test_every_employment_table_maps_into_the_shared_vocabulary() -> None: found = 0 @@ -36,9 +47,30 @@ def test_every_employment_table_maps_into_the_shared_vocabulary() -> None: values = table.values() if isinstance(table, dict) else [v for _k, v in table] # "" is a known word that names no employment ("permanent"). assert set(values) <= {*EMPLOYMENTS, ""}, (name, table_name) + for name in _PREP_HOSTS: + for table_name in ("EMPLOYMENT", "MATCH"): + table = _prep_table(name, table_name) + if table is None: + continue + values = table.values() if isinstance(table, dict) else [v for _k, v in table] + found += 1 + assert set(values) <= {*EMPLOYMENTS, ""}, (name, table_name) assert found >= 15 +def test_the_prep_tables_of_the_hosts_speak_the_apps_vocabulary() -> None: + from app.models import SENIORITIES + for name in _PREP_HOSTS: + seniority = _prep_table(name, "SENIORITY") + if seniority is not None: + assert set(seniority.values()) <= {*SENIORITIES, ""}, name + remote = _prep_table(name, "REMOTE") + if remote is not None: + assert set(remote.values()) <= {"remote", "hybrid", "onsite", ""}, name + for table in (seniority, remote, _prep_table(name, "EMPLOYMENT")): + assert table is None or all(k == k.lower() for k in table), name + + def test_the_vocabulary_is_the_documented_one() -> None: assert EMPLOYMENTS == ("full_time", "part_time", "contract", "temporary", "internship", "freelance") From d04ceae7e05f5eb98305c5a57fd6cfc7ff65f923 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 11:19:22 +0200 Subject: [PATCH 3/4] Prep stage: host words stored raw, mapped centrally in app/prep after every pull upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 --- app/cli.py | 32 ++- app/jobs/__init__.py | 3 +- app/jobs/cascade.py | 32 +-- app/jobs/prep.py | 31 +++ app/jobs/pull.py | 10 +- app/models.py | 19 ++ app/prep/__init__.py | 22 +++ app/prep/country.py | 55 ++++++ app/prep/hosts/__init__.py | 61 ++++++ app/prep/hosts/platsbanken.py | 12 ++ app/prep/stage.py | 139 +++++++++++++ app/prep/words.py | 110 +++++++++++ app/static/sources.js | 41 +++- app/storage/gazetteer.py | 33 ++++ app/storage/schema.py | 84 +++++++- app/storage/store.py | 184 ++++++++++++----- app/web/api.py | 17 ++ plugins/readers/_shared.py | 6 +- tests/prepped.py | 16 ++ tests/test_architecture.py | 2 +- tests/test_companies.py | 23 +-- tests/test_dupes_storage.py | 2 +- tests/test_enrich.py | 16 +- tests/test_extract.py | 29 +-- tests/test_gazetteer.py | 37 ++-- tests/test_prep.py | 358 ++++++++++++++++++++++++++++++++++ tests/test_pull.py | 18 +- tests/test_score.py | 35 ++-- tests/test_search.py | 13 +- tests/test_storage.py | 249 +++++++++++------------ tests/test_web.py | 109 ++++++----- 31 files changed, 1468 insertions(+), 330 deletions(-) create mode 100644 app/jobs/prep.py create mode 100644 app/prep/__init__.py create mode 100644 app/prep/country.py create mode 100644 app/prep/hosts/__init__.py create mode 100644 app/prep/hosts/platsbanken.py create mode 100644 app/prep/stage.py create mode 100644 app/prep/words.py create mode 100644 tests/prepped.py create mode 100644 tests/test_prep.py diff --git a/app/cli.py b/app/cli.py index 95878a0..a5deb45 100644 --- a/app/cli.py +++ b/app/cli.py @@ -8,6 +8,7 @@ app agents list|prompt app stages [extract|summarise on|off] app dupes + app prep [unmapped] [--remap] app db init `app job` queues the work on a queue of its own and drains it here, so a @@ -160,6 +161,13 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("dupes", help="recompute which open roles are copies of one posting") + prep = sub.add_parser( + "prep", help="map the roles' host words and fold the copies, or list the unmapped words") + prep.add_argument("action", nargs="?", choices=["unmapped"], + help="list the host words no table maps, instead of running") + prep.add_argument("--remap", action="store_true", + help="map every role again, closed ones included") + db = sub.add_parser("db", help="database housekeeping") db.add_argument("action", choices=["init"]) return parser @@ -188,6 +196,8 @@ def main(argv: list[str] | None = None) -> int: return run_seniority(args) if args.command == "dupes": return run_dupes(args) + if args.command == "prep": + return run_prep(args) return init_db(args) except (FileNotFoundError, config_mod.ConfigError) as exc: print(exc, file=sys.stderr) @@ -655,7 +665,7 @@ def run_seniority(args: argparse.Namespace) -> int: def run_dupes(args: argparse.Namespace) -> int: - """`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`, + """`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`, now. Prints the groups (kept roles with a copy), the copies the feed hides and the rows whose `dup_of` changed, as JSON.""" from app.storage import Store @@ -666,6 +676,26 @@ def run_dupes(args: argparse.Namespace) -> int: return 0 +def run_prep(args: argparse.Namespace) -> int: + """`app prep`: the prep stage now (`app.prep.run`), printing what it did + as JSON; `--remap` maps every role again. `app prep unmapped` lists the + host words no table maps, per field and host, most roles first.""" + from app import prep + from app.storage import Store + + config = config_mod.load(args.config) + store = Store(config.database) + if args.action == "unmapped": + found = store.unmapped() + if not found: + print("every host word is mapped") + for w in found: + print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}") + return 0 + print(json.dumps(prep.run(store, remap=args.remap)._asdict())) + return 0 + + def init_db(args: argparse.Namespace) -> int: """Create the file, or bring it to the current schema.""" from app.storage import LATEST, Store diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index 35ed510..ec9bd9e 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -2,11 +2,12 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. pull.py read every board and search and store the roles they yield + prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold extract.py title and description into the gate columns, on Haiku embed.py one vector per open role, for fit score.py every open role scored for every enabled search agent summarise.py about 60 words on each of the best roles, on Haiku - cascade.py pull → embed → score → extract → score → summarise, what `TASKS` + cascade.py pull → prep → embed → score → ... → summarise, what `TASKS` offers, the `pull` job a schedule fires, and that schedule stages.py whether extract and summarise run: the switch, else config.yaml sources.py what a pull reads, changed by hand: boards and searches diff --git a/app/jobs/cascade.py b/app/jobs/cascade.py index f99863b..c430545 100644 --- a/app/jobs/cascade.py +++ b/app/jobs/cascade.py @@ -1,10 +1,13 @@ """The tasks roledesk offers, and the order a pull runs them in. - pull → embed → score → enrich → embed → score → extract → score → summarise + pull → prep → embed → score → enrich → prep → embed → score → extract → score + → summarise newsdesk's cascade convention: a cascade is a tuple of task names, and the final job of one task starts the next (`step` is the callback it carries). -Nothing is alive between two tasks. A stage is addressed by its position in +Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what +the pull stored and folds the copies before anything reads the roles; after +`enrich` it maps the roles their own pages filled. A stage is addressed by its position in the tuple, not its name, since `score` runs twice. Every stage is always in the chain, and a stage that is switched off finds @@ -12,8 +15,9 @@ nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull d satisfy, is an extract stage with no batches whose final job moves straight on. The free stages go first: embed and score cost no tokens, and the score they produce is what bounds the rest. Enrich reads the best roles without -text on their own pages (`app/jobs/enrich.py`), and embed and score follow for -the roles it filled; when it filled none, `step` passes over both. Extract +text on their own pages (`app/jobs/enrich.py`), and prep, embed and score +follow for the roles it filled; when it filled none, `step` passes over all +three. Extract reads only the best roles of each enabled agent, as deep and as low as that agent's own `limits` say; the second score then applies the gates to the columns it filled. @@ -27,7 +31,7 @@ having either way. and a cascade is started by one: the `pull` job's only work is to ask for the chain. Its default schedule, `every 6h`, is declared here beside it. -`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued +`enrich` is a job and a cascade the same way, `enrich → prep → embed → score`, queued behind the `score` that follows a change to the ranking other than a pull's (`enrich_after`): an agent saved, a company's affinity changed. A vote moves one agent's taste a little and queues neither. @@ -40,20 +44,20 @@ from dataclasses import replace from functools import partial from typing import Any -from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs +from app.jobs import embed, enrich, extract, prep, pull, score, summarise # noqa: F401 -- registers the jobs from desk.tools import agent from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job STEP = "app.jobs.cascade:step" CASCADES: dict[str, tuple[str, ...]] = { - "pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score", - "summarise"), - "enrich": ("enrich", "embed", "score"), + "pull": ("pull", "prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", + "score", "summarise"), + "enrich": ("enrich", "prep", "embed", "score"), } #: What follows `enrich` for the roles it filled, passed over when it filled none. -REFILL = ("embed", "score") +REFILL = ("prep", "embed", "score") def _once(config: Any, store: Any) -> list[dict[str, Any]]: @@ -103,6 +107,8 @@ STAGES: dict[str, Callable[[str], Task]] = { "embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"), "score": lambda cascade: Task(kind="score", finder=_once, label="Scoring roles for every agent"), + "prep": lambda cascade: Task(kind="prep", finder=_once, + label="Mapping the roles' host words"), "summarise": _summarise, "enrich": lambda cascade: enrich.enrich_task(), } @@ -189,7 +195,7 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str @job("enrich", "Reading the best roles on their own pages", solo=True, resource="http") def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: - """Ask for the enrich cascade, `enrich → embed → score`. Its own work is + """Ask for the enrich cascade, `enrich → prep → embed → score`. Its own work is nil, as the `pull` job's. Under `NULL` there is no queue to take it.""" if not context.task(chain("enrich")): raise RuntimeError("no queue to start the enrich cascade on") @@ -210,7 +216,7 @@ def enrich_after(queue: Any, score_job: Any) -> Any: #: What `app job ` and `/api/jobs` start besides `pull`: one stage #: each, on its own. `pull` is not a task here, since it is a job: a name that #: is both would be listed twice by `/api/jobs/kinds`. -for _name in ("extract", "embed", "score", "summarise"): +for _name in ("extract", "embed", "score", "summarise", "prep"): TASKS[_name] = partial(task, _name) @@ -220,5 +226,5 @@ for _name in ("extract", "embed", "score", "summarise"): # setting: a cadence edited or switched off under Runs survives # every restart (see pipeline/schedule.py). declare("pull", "pull", "every 6h", - label="Pull every board, then embed, score, read the best roles' pages, " + label="Pull every board, then map, embed, score, read the best roles' pages, " "extract, rescore and summarise") diff --git a/app/jobs/prep.py b/app/jobs/prep.py new file mode 100644 index 0000000..013ba4b --- /dev/null +++ b/app/jobs/prep.py @@ -0,0 +1,31 @@ +"""`prep`: the stage that maps what a pull stored, `app.prep.run`. + +In the pull cascade it follows the pull, whose settle has closed what is +gone, and maps every role the pull changed, then folds the copies across +the whole database (`Store.rebuild_dupes`, which the pull's settle used to +run). After `enrich` it maps the roles their own pages filled, since only +those are due. `solo`, so two cascades never write the same rows at once. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from app import prep +from app.config import Config +from app.storage import Store +from pipeline import NULL, RunContext, job + +log = logging.getLogger(__name__) + + +@job("prep", "Mapping the roles' host words", solo=True, resource="cpu") +def run_prep(config: Config, store: Store, *, remap: bool = False, + context: RunContext = NULL) -> dict[str, Any]: + """Map the due roles, every open one when the tables changed, and fold + the copies. `remap` maps every role, closed ones included.""" + done = prep.run(store, remap=remap) + log.info("prep: %d read, %d written, %d words unmapped, %s copies%s", done.read, + done.written, done.unmapped, done.copies, ", tables changed" if done.remapped else "") + return done._asdict() diff --git a/app/jobs/pull.py b/app/jobs/pull.py index c0a1cab..505c58d 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -244,10 +244,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *, roles reopened by that batch, after the switch closed them, so every switched-off one is closed again here. - After the closes, `Store.rebuild_dupes` recomputes which open roles are - copies of one posting, so a closed kept role hands its group on and the - roles this run added join theirs; `copies` is how many open roles it - leaves hidden as copies. + The `prep` stage that follows maps what this run stored and folds the + copies (`Store.rebuild_dupes`), so a closed kept role hands its group on + and the roles this run added join theirs. """ answered = searched = failed = incomplete = closed = 0 stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds") @@ -284,10 +283,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *, closed += store.close_unseen("search", stale) if run_marker: release_run(store, run_marker) - copies = store.rebuild_dupes().copies roles = sum(int(result.get("roles", 0)) for result in tally.results) return {"boards": answered, "searches": searched, "failed": failed, - "incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies} + "incomplete": incomplete, "roles": roles, "closed": closed} def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int: diff --git a/app/models.py b/app/models.py index 02c90ab..cd2292c 100644 --- a/app/models.py +++ b/app/models.py @@ -649,6 +649,13 @@ SENIORITIES: tuple[str, ...] = get_args(Seniority) #: The levels a seniority filter may name: `SENIORITIES` without the unknown. LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level) +#: The kind of contract a posting states, as `app.prep` maps a host's word. +#: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the +#: readers' own tests, since a plugin imports nothing from `app`. +Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship", + "freelance"] +EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e) + #: Whether a posting says it sponsors a work permit. '' is "it does not say". Permit = Literal["", "sponsors", "no_sponsorship"] PERMITS: tuple[str, ...] = get_args(Permit) @@ -701,6 +708,16 @@ def trusted_posted(host: str) -> bool: _POSTED_FLOOR = "2015-01-01T00:00:00+00:00" +def host_word(value: Any) -> str: + """A host's word as `roles.*_host` stores it: stripped text, a bool as + 'true' or 'false', '' for nothing. What `app.prep` maps.""" + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + return str(value).strip() + + def posted_host(raw: dict[str, Any]) -> str: """The host a role's dates came from, read the way `ats` and `search` both put it in `Item.raw`: `board` for an ATS board (`greenhouse`, @@ -797,6 +814,8 @@ class Role(Model): seniority_model: str = "" seniority_stated: str = "" seniority_title: str = "" + #: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`. + employment: str = "" function: str = "" permit: Permit = "" #: ISO 639-1 codes the posting says are required, lowercase. diff --git a/app/prep/__init__.py b/app/prep/__init__.py new file mode 100644 index 0000000..7d64ac8 --- /dev/null +++ b/app/prep/__init__.py @@ -0,0 +1,22 @@ +"""Prep: every decision about a role that needs a rule or data across roles. + +Readers scrape and hand on the host's own words (`Store.upsert_role` keeps +them in `roles.*_host`); prep maps them, once, centrally, between the pull +and embedding. One door: `from app.prep import ...`. + + words.py seniority, employment and remote: the host's table, the + vocabulary itself, the shared remote words + country.py the country from the resolved places, else the host's + hosts/ one table file per reader (`hosts.TABLES`) + stage.py `run`: the stage over the due roles, `unmapped` and the fold +""" + +from __future__ import annotations + +from app.prep.country import country +from app.prep.hosts import TABLES +from app.prep.stage import STATE, Prepped, company, fingerprint, run +from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority + +__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country", + "employment", "fingerprint", "lookup", "remote", "run", "seniority"] diff --git a/app/prep/country.py b/app/prep/country.py new file mode 100644 index 0000000..3caf32b --- /dev/null +++ b/app/prep/country.py @@ -0,0 +1,55 @@ +"""A role's country, from its places, else from the one its host states.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from app import places + + +def of_host(gaz: places.Gazetteer, stated: str) -> str | None: + """The ISO code a host's country names, in the host's spelling: a code + ("DK", "dk"), or a name the gazetteer has as an alias of a country + ("Denmark", "Danmark"), `EU` or Europe by name, or a text that resolves to places of one + country. '' for nothing stated, None for a text naming no country.""" + text = (stated or "").strip() + if not text: + return "" + if ("country", text.upper()) in gaz.by_key or text.upper() == "EU": + return text.upper() + named = {"EU" if gaz.places[p].key == "europe" else gaz.places[p].key + for p in gaz.aliases.get(places.fold(text), ()) + if gaz.places[p].kind == "country" + or (gaz.places[p].kind, gaz.places[p].key) == ("continent", "europe")} + if len(named) == 1: + return named.pop() + found = {gaz.country(p) for p in gaz.resolve(text)} - {""} + return found.pop() if len(found) == 1 else None + + +def _in_europe(gaz: places.Gazetteer, code: str) -> bool: + if code == "EU": + return True + europe = gaz.by_key.get(("continent", "europe")) + place_id = gaz.by_key.get(("country", code)) + return europe is not None and place_id is not None and europe in gaz.ancestors(place_id) + + +def country(gaz: places.Gazetteer, place_ids: Iterable[int], host_country: str = "") -> str: + """The one country the role's resolved places lie in, continents aside; + Europe alone is `EU`; else, when they + lie in several all in Europe, `EU` (what `country_of` wrote for "Dublin, + London" and the country gate reads as Europe); else the host's stated + country (`of_host`); else ''.""" + ids = list(place_ids) + # A continent beside a country is the wider unit a line adds ("Home + # based - EMEA; London" is GB), as `country_of` had it; alone, Europe + # is `EU`. + found = {gaz.country(p) for p in ids} - {""} + if not found and any(gaz.places[p].key == "europe" for p in ids if p in gaz.places): + found = {"EU"} + if len(found) == 1: + return found.pop() + if len(found) > 1 and all(_in_europe(gaz, code) for code in found): + return "EU" + return of_host(gaz, host_country) or "" diff --git a/app/prep/hosts/__init__.py b/app/prep/hosts/__init__.py new file mode 100644 index 0000000..518890a --- /dev/null +++ b/app/prep/hosts/__init__.py @@ -0,0 +1,61 @@ +"""Each host's own words, one module per reader, named as the reader is. + +A module defines any of three dicts, keyed by the host's word lowercased: + +- `SENIORITY`, values in `app.models.SENIORITIES` +- `EMPLOYMENT`, values in `app.models.EMPLOYMENTS` +- `REMOTE`, values in `app.models.REMOTE` (remote, hybrid, onsite) + +A value of "" is a word the host uses that names no value ("Experienced", +"Permanent"): known, so it is not listed as unmapped. + +A host whose employment word is a code of several parts (`permanent_full_time`, +`SALARIED_FT`) may add `MATCH`, `[(fragment, value), ...]`, tried in order +after `EMPLOYMENT` with the first hit winning, and must then say how a +fragment matches in `MATCH_MODE`: "substring" (the fragment is in the +lowercased word) or "token" (it equals one of the word's parts split at `_` +and `-`, so `ft` does not match inside `software`). `TABLES` loads every +module of this package by name, so a table is added by adding its file. +""" + +from __future__ import annotations + +import importlib +import pkgutil +from typing import Any + +#: The dict a module names, by the field it maps. +FIELDS = {"seniority": "SENIORITY", "employment": "EMPLOYMENT", "remote": "REMOTE"} + +#: How a `MATCH` fragment may match. +MATCH_MODES = ("substring", "token") + + +def load() -> dict[str, dict[str, Any]]: + """`host -> field -> {word: value}` for every module of the package, the + words lowercased and stripped. A host without a table for a field has + no entry for it. A `MATCH` is `host -> "match" -> {"mode": ..., "rules": + [[fragment, value], ...]}`; a module with `MATCH` and no valid + `MATCH_MODE` fails the import by name.""" + tables: dict[str, dict[str, Any]] = {} + for info in sorted(pkgutil.iter_modules(__path__), key=lambda i: i.name): + if info.name.startswith("_"): + continue + module = importlib.import_module(f"{__name__}.{info.name}") + for field, name in FIELDS.items(): + table = getattr(module, name, None) + if isinstance(table, dict): + tables.setdefault(info.name, {})[field] = { + str(k).strip().lower(): str(v) for k, v in table.items()} + rules = getattr(module, "MATCH", None) + if rules is not None: + mode = getattr(module, "MATCH_MODE", None) + if mode not in MATCH_MODES: + raise ValueError(f"app/prep/hosts/{info.name}.py has MATCH and its MATCH_MODE " + f"is {mode!r}, not one of {', '.join(MATCH_MODES)}") + tables.setdefault(info.name, {})["match"] = { + "mode": mode, "rules": [[str(f).lower(), str(v)] for f, v in rules]} + return tables + + +TABLES = load() diff --git a/app/prep/hosts/platsbanken.py b/app/prep/hosts/platsbanken.py new file mode 100644 index 0000000..ac510fc --- /dev/null +++ b/app/prep/hosts/platsbanken.py @@ -0,0 +1,12 @@ +"""Platsbanken's words, for prep.""" + +#: `workplace_model.label`, in Swedish; mostly empty. Only "Arbete på plats" +#: is in the fixtures; the others are JobTech's other labels as the site +#: writes them, and a label not here is listed as unmapped. +REMOTE: dict[str, str] = { + "arbete på plats": "onsite", "på plats": "onsite", + "hybridarbete": "hybrid", "hybrid": "hybrid", "delvis på distans": "hybrid", + "delvis distans": "hybrid", + "distansarbete": "remote", "arbete på distans": "remote", "helt på distans": "remote", + "distans": "remote", +} diff --git a/app/prep/stage.py b/app/prep/stage.py new file mode 100644 index 0000000..02aa499 --- /dev/null +++ b/app/prep/stage.py @@ -0,0 +1,139 @@ +"""The prep stage: map what the readers handed on, then fold the copies. + +A pull stores the host's own words and place texts (`Store.upsert_role`) +and marks the role due. `run` maps every due role: its company through +`company_aliases`, its places against the gazetteer, its level, contract +and remote through the host's tables (`app.prep.words`), and its country +from the places (`app.prep.country`). When the tables changed since the +last run (`fingerprint`, kept in state as `prep:tables`), every open role +is mapped again from its stored words, with no new pull. Then the words no +table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the +copies across the whole database. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from typing import Any, NamedTuple + +from app import places +from app.models import company_key +from app.models import company_fold +from app.prep import words +from app.prep.country import country, of_host +from app.prep.hosts import TABLES +from app.prep.words import Tables +from app.storage import Store + +#: Raised when the rules in this package change the way a stored word maps, +#: so every open role is mapped again as it is after a table changes. +VERSION = 1 + +#: The state key the last run's `fingerprint` is kept under. +STATE = "prep:tables" + + + +class Prepped(NamedTuple): + """What one run did: roles read, rows written, whether every role was + mapped again, words left unmapped, and the open copies after the fold + (None when it was not run).""" + + read: int + written: int + remapped: bool + unmapped: int + copies: int | None + + +def fingerprint(tables: Tables = TABLES) -> str: + """A hash of every host table, the shared remote words and `VERSION`.""" + blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS}, + sort_keys=True) + return hashlib.sha1(blob.encode()).hexdigest() + + +def company(folds: Mapping[str, Sequence[str]], name: str) -> str: + """The key a role from `name` belongs to, as `Store.resolve_company` + has it: the company an alias names, the name's own key first when + several share the fold, else `company_key(name)`.""" + own = company_key(name) + keys = folds.get(company_fold(name), ()) + return own if own in keys or not keys else keys[0] + + +def _places(raw: Any) -> list[Any]: + try: + found = json.loads(raw) if isinstance(raw, str) else raw + except ValueError: + return [] + return found if isinstance(found, list) else [] + + +def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]], + place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]: + """One role as `Store.write_prep` takes it, from a `Store.prep_rows` row.""" + host = str(row["host"] or "") + stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables) + return { + "uid": row["uid"], + "due": row["prep_due"], + "company_key": company(folds, row["company"]) if row["company"] else row["company_key"], + "seniority_stated": stated, + "seniority_title": titled, + "employment": words.employment(host, row["employment_host"], tables), + "remote": words.remote(host, row["remote_host"] or None, (), "", tables), + "remote_places": words.remote(host, None, _places(row["raw_places"]), row["place"], + tables), + "country": country(gaz, place_ids, row["country_host"]), + } + + +def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool: + level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"] + return bool(row["prep_due"]) or level != row["seniority"] \ + or (found["remote"] and found["remote"] != row["remote"]) \ + or (found["remote_places"] and not row["remote"]) \ + or (found["country"] and found["country"] != row["country"]) \ + or any(found[c] != row[c] for c in ("company_key", "seniority_stated", + "seniority_title", "employment")) + + +def unknown(field: str, host: str, word: str, gaz: places.Gazetteer, + tables: Tables = TABLES) -> bool: + """Whether no table, vocabulary or gazetteer knows this host word.""" + if field == "country": + return of_host(gaz, word) is None + if field == "remote" and word in ("true", "false"): + return False + return words.lookup(field, host, word, tables) is None + + +def run(store: Store, *, remap: bool = False, tables: Tables = TABLES, + dupes: bool = True) -> Prepped: + """Map the due roles, or every open one when the tables changed, or + every role with `remap`; list the unmapped words; fold the copies.""" + mark = fingerprint(tables) + whole = remap or store.state.get(STATE) != mark + rows = store.prep_rows("due") + if whole: + due = {r["uid"] for r in rows} + rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due] + ids = store.resolve_role_places([r["uid"] for r in rows]) + gaz = store.gazetteer() + folds = store.company_folds() + written = [] + for row in rows: + found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables) + if _changed(row, found): + written.append(found) + store.write_prep(written) + left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)] + store.replace_unmapped(left) + if whole: + store.state[STATE] = mark + copies = store.rebuild_dupes().copies if dupes else None + return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left), + copies=copies) diff --git a/app/prep/words.py b/app/prep/words.py new file mode 100644 index 0000000..4d71dc8 --- /dev/null +++ b/app/prep/words.py @@ -0,0 +1,110 @@ +"""A host's word to the app's vocabulary: seniority, employment and remote. + +Each field is looked up the same way: the host's table in `hosts`, keyed by +the word lowercased, then a word that already is a value of the vocabulary +(what readers stored before they handed on the host's words, and what a +host with the app's own words sends), then, for remote, the shared word map. +An employment word may also match a host's `MATCH` fragments. A word none +of these knows is unmapped: it maps to '' and `lookup` says so, +so the prep stage can list it. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from typing import Any + +from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of +from app.prep.hosts import TABLES +from app.seniority import from_title + +#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`). +Tables = Mapping[str, Mapping[str, Any]] + +VOCABULARIES: dict[str, tuple[str, ...]] = { + "seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE} + +#: How hosts without a table of their own write remote, after `_squash`: +#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office"). +REMOTE_WORDS: dict[str, str] = { + "remote": "remote", "fullyremote": "remote", "remotefirst": "remote", + "hybrid": "hybrid", + "onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite", + "officebased": "onsite", "onpremise": "onsite", +} + + +def _match(rules: Mapping[str, Any], key: str) -> str | None: + """The value of the first `MATCH` fragment `key` holds, by its mode.""" + parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set() + for fragment, value in rules["rules"]: + if fragment in parts if rules["mode"] == "token" else fragment in key: + return str(value) + return None + + +def _squash(word: str) -> str: + return "".join(c for c in word.lower() if c not in " -_") + + +def lookup(field: str, host: str, word: str, + tables: Tables = TABLES) -> str | None: + """The value `word` maps to for `host`, '' for a known word that names + none, None for a word nothing knows. An empty word is ''.""" + key = (word or "").strip().lower() + if not key: + return "" + table = tables.get(host, {}) + found = table.get(field, {}).get(key) + if found is None and field == "employment" and "match" in table: + found = _match(table["match"], key) + if found is not None: + return found + if key in VOCABULARIES[field]: + return key + if field == "remote": + return REMOTE_WORDS.get(_squash(key)) + return None + + +def seniority(host: str, word: str, title: str, + tables: Tables = TABLES) -> tuple[str, str]: + """`(stated, from the title)`: the host's word through its table, and + `seniority.from_title`. `roles.seniority` is the model's level, else the + stated one, else the title's (`Store.write_prep`).""" + return lookup("seniority", host, word, tables) or "", from_title(title) + + +def employment(host: str, word: str, + tables: Tables = TABLES) -> str: + return lookup("employment", host, word, tables) or "" + + +def _place_text(place: Any) -> str: + return str((place.get("text") if isinstance(place, Mapping) else place) or "") + + +def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "", + tables: Tables = TABLES) -> str: + """How the role is worked, one of `REMOTE` or ''. + + `value` is the host's own (a bool, or its word; `host_word`'s 'true' and + 'false' are bools). True is remote. False says nothing more than the + places do: SmartRecruiters, Ashby, Recruitee and Workable send it for + every role that is not remote, hybrid ones included. A word goes + through the host's table, else the shared map. When the value says + nothing, the place line or the place texts are read for a word + (`remote_of`: "Hybrid, Berlin"), and last a place the reader marked + remote is remote.""" + if value is True or value == "true": + return "remote" + said = value not in (None, False, "false") + if said and (found := lookup("remote", host, str(value), tables)): + return found + places = list(places) + if found := remote_of(line or "; ".join(map(_place_text, places))): + return found + if any(isinstance(p, Mapping) and p.get("remote") is True for p in places): + return "remote" + return "" diff --git a/app/static/sources.js b/app/static/sources.js index c986b8b..e306ae5 100644 --- a/app/static/sources.js +++ b/app/static/sources.js @@ -19,7 +19,9 @@ * Under the Follow form of Company boards, a fold lists what each reader * declares (`/api/readers`): how its list arrives, where the text is, what * it filters at the source, how it states places, and the robots.txt - * decision taken when it was built. + * decision taken when it was built. A second fold lists the host words no + * table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and + * host, most roles first; it is left out when there are none. */ (function () { "use strict"; @@ -320,6 +322,38 @@ $("#board-form").after(fold); } + /** The host words prep could not map, one row per field and host. */ + function renderUnmapped(words) { + if (!words.length) return; + const groups = new Map(); + for (const w of words) { + const key = `${w.field} · ${w.host || "no host"}`; + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(w); + } + const fold = document.createElement("details"); + fold.className = "prune"; + fold.id = "unmapped"; + const summary = document.createElement("summary"); + summary.textContent = `Host words no table maps (${words.length})`; + const list = document.createElement("div"); + list.className = "srows"; + for (const [key, found] of groups) { + const row = document.createElement("div"); + row.className = "srow"; + const name = document.createElement("span"); + name.className = "name"; + name.textContent = key; + const sub = document.createElement("span"); + sub.className = "sub"; + sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · "); + row.append(name, sub); + list.appendChild(row); + } + fold.append(summary, list); + ($("#readers") || $("#board-form")).after(fold); + } + // ------------------------------------------------------ one section shown // Sources shows one section: the one the fragment names or contains, or the @@ -360,7 +394,10 @@ document.addEventListener("DOMContentLoaded", async () => { if (!$("#searches-list")) return; - if ($("#board-form")) api("/api/readers").then(renderReaders, () => {}); + if ($("#board-form")) { + api("/api/readers").then(renderReaders, () => {}) + .then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {}); + } try { await load(); } catch (err) { diff --git a/app/storage/gazetteer.py b/app/storage/gazetteer.py index ae3386f..dd987a2 100644 --- a/app/storage/gazetteer.py +++ b/app/storage/gazetteer.py @@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here. from __future__ import annotations +import json import sqlite3 from collections.abc import Iterable, Mapping, Sequence from typing import Any @@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str, conn.executemany(_INSERT, rows(gaz, uid, place_texts)) +def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None: + """Replace one role's own places with these texts, unresolved: what a + pull stores, for `app.prep` to resolve (`relink_uids`).""" + conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,)) + conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None) + for position, text in enumerate(place_texts)]) + + +def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer, + uids: Sequence[str]) -> dict[str, list[int]]: + """Resolve the own places of these roles, writing only those that now + resolve differently. Returns `uid -> place ids` in the host's order for + every role of `uids` with places, what prep reads the country from.""" + held: dict[str, dict[tuple[int, str], set[int | None]]] = {} + for start in range(0, len(uids), 5000): + chunk = json.dumps(list(uids[start:start + 5000])) + for uid, position, text, place_id in conn.execute( + "SELECT uid, position, text, place_id FROM role_places WHERE url = '' " + "AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid", + (chunk,)): + held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id) + found: dict[str, list[int]] = {} + changed = [] + for uid, rows_of in held.items(): + now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of} + if now != rows_of: + changed.append((uid, [text for _, text in sorted(rows_of)])) + found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])] + write_many(conn, gaz, changed) + return found + + def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer, by_uid: Iterable[tuple[str, Sequence[str]]]) -> int: """`write` for many roles in one statement each way. Returns rows written.""" diff --git a/app/storage/schema.py b/app/storage/schema.py index 60b0cb6..4b08f93 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -18,7 +18,9 @@ import sqlite3 from collections.abc import Callable from app import places -from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host +from app.models import ( + SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host, +) from app.seniority import from_title from app.storage import gazetteer @@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles ( teaser TEXT NOT NULL DEFAULT '', seniority_model TEXT NOT NULL DEFAULT '', seniority_stated TEXT NOT NULL DEFAULT '', - seniority_title TEXT NOT NULL DEFAULT '' + seniority_title TEXT NOT NULL DEFAULT '', + seniority_host TEXT NOT NULL DEFAULT '', + employment_host TEXT NOT NULL DEFAULT '', + remote_host TEXT NOT NULL DEFAULT '', + country_host TEXT NOT NULL DEFAULT '', + employment TEXT NOT NULL DEFAULT '', + prep_due INTEGER NOT NULL DEFAULT 0 ); +-- The *_host columns hold the host's own words as a reader handed them on +-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps +-- them into seniority_stated, employment, remote and country. prep_due is above 0 +-- on a role whose words, company, title or places changed since prep last +-- mapped it (Store.upsert_role), which is what the next prep reads. -- roles.teaser is the short text a list carries, kept until the full text -- arrives and embedded while there is none; a later empty teaser does not -- wipe it. seniority_model, seniority_stated and seniority_title are the @@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors ( embedded_at TEXT NOT NULL ); +-- A host word no table of app/prep/hosts maps, per field (seniority, +-- employment, remote, country) and host: how many open roles carry it and +-- when prep first met it. Rewritten by every prep; a word added to a table +-- leaves at the next one. +CREATE TABLE IF NOT EXISTS unmapped ( + field TEXT NOT NULL, + host TEXT NOT NULL, + word TEXT NOT NULL, + roles INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + PRIMARY KEY (field, host, word) +); + -- The string map a desk plugin is handed as self.state. One namespace shared -- by every plugin, so a plugin prefixes its keys. CREATE TABLE IF NOT EXISTS state ( @@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid); CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid); CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id); +-- The roles the next prep maps. Partial: after a prep there are none. +CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0; """ #: The table whose absence marks a file that has never been a roledesk @@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; (21, "companies.all_roles: a company tag that reads its boards with the agents' wide " "reach, given to every favourite and taking over a board's {\"reach\": \"all\"}", lambda conn: _add_all_roles(conn)), + # A Python step: the host words are read out of raw's JSON. + (22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own " + "words are stored and app.prep maps them", lambda conn: _add_host_words(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None: "to tag; it reads the agents' reach now", target, key) +_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment") + +_V22 = """ +CREATE TABLE IF NOT EXISTS unmapped ( + field TEXT NOT NULL, + host TEXT NOT NULL, + word TEXT NOT NULL, + roles INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + PRIMARY KEY (field, host, word) +); +DELETE FROM state WHERE key = 'prep:tables'; +""" + + +def _add_host_words(conn: sqlite3.Connection) -> None: + """Migration 22's step. The columns and `unmapped`, and the host words + of every role filled from its `raw` where it has them. Before this, + readers mapped `seniority` and `employment` in their own tables, so what + `raw` holds there is mostly a value of the vocabulary already, which + `app.prep` takes as it is. `remote` and `country` are the host's as + given. The `prep:tables` fingerprint goes, so the first prep maps every + open role from these words. Runs twice without harm.""" + columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")} + for column in _V22_COLUMNS: + if column not in columns: + conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''") + if "prep_due" not in columns: + conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0") + for statement in _statements(_V22): + conn.execute(statement) + updates = [] + for uid, raw in conn.execute( + "SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL " + "OR json_extract(raw, '$.employment') IS NOT NULL " + "OR json_extract(raw, '$.remote') IS NOT NULL " + "OR json_extract(raw, '$.country') IS NOT NULL").fetchall(): + try: + given = json.loads(raw or "{}") + except ValueError: + continue + if isinstance(given, dict): + updates.append((*(host_word(given.get(k)) for k in ( + "seniority", "employment", "remote", "country")), uid)) + conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, " + "remote_host = ?, country_host = ? WHERE uid = ?", updates) + log.info("migration 22: host words from raw for %d roles", len(updates)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/storage/store.py b/app/storage/store.py index dd0360b..b5b9932 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -22,13 +22,12 @@ import numpy as np from desk.models import Item from app import dupes, places from app.models import ( - AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application, + AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace, RoleRow, ScoredRole, Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key, - posted_at, posted_host, remote_of, title_key, + host_word, posted_at, posted_host, remote_of, title_key, ) -from app.seniority import from_title from app.storage import gazetteer, schema #: Text columns a plugin may fill from `Item.raw` under the same name. An @@ -36,7 +35,17 @@ from app.storage import gazetteer, schema #: says less than `extract` found keeps what `extract` found. `function` is #: not here: it is not a board field (see the note by `keep_filled` below). #: Nor is `seniority`, which is the title's level unless a board states one. -_RAW_TEXT = ("place", "country", "currency") +_RAW_TEXT = ("place", "currency") + +#: The host's own words a reader hands on in `Item.raw`, stored as +#: `_host` (`host_word`) for `app.prep` to map. `seniority_host` is +#: rewritten every pull, as the host's level always was; the others are kept +#: filled, since a role's own page (`enrich`) may state what its listing +#: does not. +_HOST_WORDS = ("seniority", "employment", "remote", "country") +_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host") +#: What prep maps from that a pull rewrites whatever it brings. +_PREP_OVERWRITTEN = ("title", "seniority_host") _RAW_INT = ("salary_min", "salary_max") #: What `extract` may fill. Only an empty column is written: a board's own @@ -154,14 +163,11 @@ def _iso(moment: datetime) -> str: return moment.astimezone(UTC).isoformat(timespec="seconds") -def _remote(value: Any) -> str: - """A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything - unrecognised is '' (unknown), which `extract` may fill later.""" - if value is True: - return "remote" - text = str(value or "").strip().lower().replace("-", "").replace(" ", "") - text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text) - return text if text in REMOTE else "" +def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]: + """One role's own place texts as stored, in the host's order.""" + return list(dict.fromkeys(r[0] for r in conn.execute( + "SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid", + (uid,)))) def _like(text: str) -> str: @@ -607,10 +613,19 @@ class Store: """Store one posting. Returns True when it was new. `first_seen` and `created_at` are written on insert only; `last_seen` on - every call, and a role seen again is open again. The gate columns come + every call, and a role seen again is open again. The raw columns come from `item.raw` under their own names, with `company` falling back to - the subscription's and `country` to what `place` names; an empty value - never overwrites a filled column. + the subscription's; an empty value never overwrites a filled column. + + Nothing is mapped here: that is `app.prep`, which runs after every + pull. The host's words go to `seniority_host`, `employment_host`, + `remote_host` and `country_host` (`host_word`), `company_key` is + written unresolved on a new role only, and the mapped columns + (`seniority_stated`, `seniority_title`, `seniority`, `employment`, + `remote`, `country`) are left as prep last wrote them. `prep_due` + is 1 on a new role and grows by one when the title, the company, a + host word or the place texts changed or the role reopens, which + is what the next prep reads. `posted_at` (see `app.models.posted_at`) is set from this pull's own `published_at` and `stamp`; on an update the `ON CONFLICT` clause below @@ -627,20 +642,14 @@ class Store: alone for good, whatever the plugin's raw guess now says — see `set_extracted` for the other half of the rule. - `seniority` is the first filled of three levels. `seniority_model` - is extract's (`set_extracted`) and never written here. - `seniority_stated` is the host's word as `raw["seniority"]` carries - it, already in `SENIORITIES` (anything else is ''), and - `seniority_title` is `seniority.from_title`; both are rewritten every - pull. The classifier is called here rather than in a plugin, so a - role from any source gets it, the way `country` comes from `place`. - The role's places (`role_places`) are the texts of `raw["places"]`, - else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved - against the gazetteer; `place` is the line shown, the texts joined - when `raw` has none. Like `place`, a pull with no place leaves them - as they are. `place_fold` is written with `place`, under the same - keep-filled rule. `teaser` is kept-filled too. + else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved + when they differ from the stored ones; prep resolves them. Texts that + did not change keep their resolved rows, so the feed's Place filter + stays right between a pull and its prep. `place` is the line shown, + the texts joined when `raw` has none. Like `place`, a pull with no + place leaves them as they are. `place_fold` is written with `place`, + under the same keep-filled rule. `teaser` is kept-filled too. A role is embedded from its description, else its teaser, else its title (`app.jobs.embed.document`). When a pull brings a better one of @@ -660,12 +669,11 @@ class Store: "source": item.source, "source_id": item.source_id, "subscription_id": subscription.id if subscription else None, - "company_key": self.resolve_company(company), + "company_key": company_key(company), "company": company, "title": item.title, "title_key": title_key(item.title), "place_key": place_key(place), - "remote": _remote(raw.get("remote")), "description": item.body, "feed_text": item.excerpt, "original_url": item.location, @@ -677,30 +685,26 @@ class Store: "created_at": stamp, } row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT} + row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS} row["place"] = place - row["country"] = row["country"] or country_of(place) - row["remote"] = row["remote"] or remote_of(place) row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} row["function"] = str(raw.get("function") or "") - stated = str(raw.get("seniority") or "") - row["seniority_stated"] = stated if stated in SENIORITIES else "" - row["seniority_title"] = from_title(item.title) - row["seniority"] = row["seniority_stated"] or row["seniority_title"] row["teaser"] = str(raw.get("teaser") or "") row["place_fold"] = places.fold(row["place"]) # A board's own pay line, one per pull like title or published_at — # not kept-filled like _RAW_TEXT: a board that stops stating pay # must be able to clear it, not leave a stale line behind. row["salary_text"] = str(raw.get("salary_text") or "") + row["prep_due"] = 1 - keep_filled = ("company", "company_key", "place_key", "remote", "description", - "feed_text", "original_url", "teaser", *_RAW_TEXT) + keep_filled = ("company", "place_key", "description", "feed_text", "original_url", + "teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS) updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled] updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})" for c in (*_RAW_INT, "subscription_id")] updates += [f"{c}=excluded.{c}" for c in ("title", "title_key", "published_at", "raw", "last_seen", - "salary_text", "seniority_stated", "seniority_title")] + "salary_text", "seniority_host")] updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)") # Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh # pull's classifier guess overwrite what extract decided, which is @@ -711,11 +715,17 @@ class Store: updates.append("function=CASE WHEN roles.extracted_at IS NULL " "THEN COALESCE(NULLIF(excluded.function, ''), roles.function) " "ELSE roles.function END") - # The model's level, else the host's, else the title's. - updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), " - "NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)") updates.append("place_fold=CASE WHEN excluded.place != '' " "THEN excluded.place_fold ELSE roles.place_fold END") + # One more than before when what prep maps from changed, or the role + # reopens, so a prep that read the old values leaves it due + # (`Store.prepped`). `roles.` is the stored row here, `excluded.` + # what this pull brought. + changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN] + changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})" + for c in ("company", *_KEPT_HOST_WORDS)] + updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} " + "THEN roles.prep_due + 1 ELSE roles.prep_due END") updates.append("closed_at=NULL") columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) @@ -725,8 +735,11 @@ class Store: conn.execute( f"INSERT INTO roles ({columns}) VALUES ({marks}) " f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row) - if place_texts: - gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts) + if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts): + gazetteer.write_texts(conn, row["uid"], place_texts) + if before is not None: + conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?", + (row["uid"],)) # `embed` encodes a role once, from the best text it has. When a # later pull brings a better one, as a host that states the text # on the role's own page does, the vector goes, so the next @@ -1253,6 +1266,89 @@ class Store: remotes) return len(countries) + len(remotes) + # ----------------------------------------------------------------- prep + + def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]: + """What `app.prep` maps a role from, and what it holds now: the roles + due (`prep_due` above 0), every open role, or every role.""" + where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope] + return self._rows( + f"""SELECT uid, prep_due, {_HOST} AS host, title, company, company_key, place, + seniority_host, employment_host, remote_host, country_host, + seniority_model, seniority_stated, seniority_title, seniority, + employment, remote, country, + json_extract(raw, '$.places') AS raw_places + FROM roles WHERE {where}""") + + def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]: + """Resolve these roles' own places against the gazetteer, writing the + rows that resolve differently; `uid -> place ids` for each that has + places (`gazetteer.relink_uids`).""" + with self.transaction() as conn: + return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids)) + + def company_folds(self) -> dict[str, list[str]]: + """`fold -> company keys` of `company_aliases`, sorted: what + `resolve_company` asks one name at a time, for prep's whole batch.""" + out: dict[str, list[str]] = {} + for r in self._rows("SELECT fold, company_key FROM company_aliases " + "ORDER BY fold, company_key"): + out.setdefault(str(r["fold"]), []).append(str(r["company_key"])) + return out + + def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int: + """Write what prep mapped, one dict per role with `uid`, `due` (the + `prep_due` it read), `company_key`, `seniority_stated`, + `seniority_title`, `employment`, `remote`, `remote_places` and + `country`. `seniority` is the model's level, else the stated, else + the title's. An empty `remote` or `country` leaves the column as it + is, since `extract` fills an empty one. `remote_places`, what the + place texts say, only fills an empty column: it is a guess from a + line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull + changed the role since it was read. Returns the rows written.""" + with self.transaction() as conn: + return conn.executemany( + """UPDATE roles SET company_key = :company_key, + seniority_stated = :seniority_stated, + seniority_title = :seniority_title, + seniority = COALESCE(NULLIF(seniority_model, ''), + NULLIF(:seniority_stated, ''), :seniority_title), + employment = :employment, + remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''), + :remote_places), + country = COALESCE(NULLIF(:country, ''), country), + prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END + WHERE uid = :uid""", mapped).rowcount + + def host_words(self) -> list[tuple[str, str, str, int]]: + """`(field, host, word, open roles)` for every host word an open role + carries, the word lowercased: what prep checks against its tables.""" + found: list[tuple[str, str, str, int]] = [] + for field in ("seniority", "employment", "remote", "country"): + found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows( + f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles + WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")] + return found + + def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None: + """Make `unmapped` these `(field, host, word, roles)`, keeping the + `first_seen` of a word already listed.""" + stamp = self.now() + with self.transaction() as conn: + seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute( + "SELECT field, host, word, first_seen FROM unmapped")} + conn.execute("DELETE FROM unmapped") + conn.executemany( + "INSERT INTO unmapped (field, host, word, roles, first_seen) " + "VALUES (?, ?, ?, ?, ?)", + [(*r, seen.get(r[:3], stamp)) for r in rows]) + + def unmapped(self) -> list[dict[str, Any]]: + """The host words no table maps, by field and host, most roles first.""" + return [dict(r) for r in self._rows( + "SELECT field, host, word, roles, first_seen FROM unmapped " + "ORDER BY field, host, roles DESC, word")] + # -------------------------------------------------------------- extract def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]: diff --git a/app/web/api.py b/app/web/api.py index 27e1589..8cf95c1 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -897,6 +897,23 @@ def list_readers() -> list[ReaderOut]: for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))] +class UnmappedOut(BaseModel): + """A host word no table of `app/prep/hosts` maps (`Store.unmapped`).""" + + field: str + host: str + word: str + roles: int + first_seen: str + + +@router.get("/prep/unmapped") +def list_unmapped(store: StoreDep) -> list[UnmappedOut]: + """The host words the last prep could not map, per field and host, most + roles first: what a table in `app/prep/hosts` should learn.""" + return [UnmappedOut(**w) for w in store.unmapped()] + + class SearchBody(BaseModel): provider: str name: str = "" diff --git a/plugins/readers/_shared.py b/plugins/readers/_shared.py index 6894556..c7cf67a 100644 --- a/plugins/readers/_shared.py +++ b/plugins/readers/_shared.py @@ -197,9 +197,9 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)" -#: The employment words a row's `employment` may hold, for every host. A -#: host's table maps its own words onto these; a word that names none of them -#: (a permanent contract says nothing about hours) maps to "". +#: A copy of `app.models.EMPLOYMENTS`, the vocabulary `app.prep` maps a +#: host's employment word onto, kept for the readers' tests since a plugin +#: imports nothing from `app`. `tests/test_prep.py` holds the two equal. EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance") _REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE) diff --git a/tests/prepped.py b/tests/prepped.py new file mode 100644 index 0000000..03557b6 --- /dev/null +++ b/tests/prepped.py @@ -0,0 +1,16 @@ +"""`upsert`: a role stored as a pull stores it and then mapped, as the prep +stage after the pull maps it. For tests of what the feed, the gates and the +pages read, which are prep's values, not the raw ones `upsert_role` keeps.""" + +from __future__ import annotations + +from typing import Any + +from app import prep +from app.storage import Store + + +def upsert(store: Store, item: Any, subscription: Any = None) -> bool: + new = store.upsert_role(item, subscription) + prep.run(store, dupes=False) + return new diff --git a/tests/test_architecture.py b/tests/test_architecture.py index c3dc326..58d46d7 100644 --- a/tests/test_architecture.py +++ b/tests/test_architecture.py @@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1] #: no database, so storage may use them. `main` assembles the app from #: everything below it; `cli` and `__main__` are the entry points and may #: reach anything. -LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli", +LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "prep", "jobs", "web", "main", "cli", "__main__"] APP = "app" EXTERNAL = {"desk", "pipeline", "auth"} diff --git a/tests/test_companies.py b/tests/test_companies.py index e17a0bb..7ba9a86 100644 --- a/tests/test_companies.py +++ b/tests/test_companies.py @@ -13,6 +13,7 @@ from app.models import company_fold from app.storage import Store from app.storage.schema import _add_company_aliases from desk.models import Item +from prepped import upsert def undo_v20(conn: sqlite3.Connection) -> None: @@ -56,10 +57,10 @@ def test_a_company_is_its_own_alias(store: Store) -> None: def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None: store.add_company("Ørsted") - store.upsert_role(posting("1", "Orsted")) - store.upsert_role(posting("2", "ORSTED A/S")) - store.upsert_role(posting("3", "Ørsted")) - store.upsert_role(posting("4", "Nobody")) + upsert(store, posting("1", "Orsted")) + upsert(store, posting("2", "ORSTED A/S")) + upsert(store, posting("3", "Ørsted")) + upsert(store, posting("4", "Nobody")) assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"] assert [c.key for c in store.companies()] == ["ørsted"] @@ -67,7 +68,7 @@ def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None: def test_an_added_alias_finds_the_company(store: Store) -> None: store.add_company("Ørsted") store.add_alias("ørsted", "Dong Energy") - store.upsert_role(posting("1", "Dong Energy A/S")) + upsert(store, posting("1", "Dong Energy A/S")) assert company_of(store, "1") == "ørsted" with pytest.raises(KeyError): store.add_alias("nope", "X") @@ -90,15 +91,15 @@ def test_duplicates_are_listed_by_fold_and_not_merged(store: Store) -> None: assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])] assert len(store.companies()) == 3 # The company with the name's own key wins when a role names it. - store.upsert_role(posting("1", "Orsted")) + upsert(store, posting("1", "Orsted")) assert company_of(store, "1") == "orsted" def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None: store.add_company("Ørsted") store.add_company("Orsted") - store.upsert_role(posting("1", "Orsted")) - store.upsert_role(posting("2", "Ørsted")) + upsert(store, posting("1", "Orsted")) + upsert(store, posting("2", "Ørsted")) sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted", board="greenhouse") a, b = store.save_agent("A"), store.save_agent("B") @@ -116,7 +117,7 @@ def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None: assert store.duplicate_companies() == [] assert store.company_aliases() == {"ørsted": ["orsted"]} # A later role from the gone spelling lands on the kept company. - store.upsert_role(posting("3", "Orsted")) + upsert(store, posting("3", "Orsted")) assert company_of(store, "3") == "ørsted" with pytest.raises(ValueError): store.merge_companies("ørsted", "ørsted") @@ -147,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) -> "VALUES (?, ?, '', '')", (key, name)) conn.execute("PRAGMA user_version = 19") reopened = Store(path) - assert reopened.migrations_applied == [20, 21] + assert reopened.migrations_applied == [20, 21, 22] assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])] assert len(reopened.companies()) == 3 @@ -184,7 +185,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp conn.execute("ALTER TABLE companies DROP COLUMN all_roles") conn.execute("PRAGMA user_version = 20") reopened = Store(path) - assert reopened.migrations_applied == [21] + assert reopened.migrations_applied == [21, 22] assert {c.key: c.all_roles for c in reopened.companies()} == { "mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0} params = {s.target: s.params for s in reopened.subscriptions(source="ats")} diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py index 86ece58..66c823a 100644 --- a/tests/test_dupes_storage.py +++ b/tests/test_dupes_storage.py @@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None: migrated = Store(path) - assert migrated.migrations_applied == [18, 19, 20, 21] + assert migrated.migrations_applied == [18, 19, 20, 21, 22] assert dup_of(migrated) == {first.uid: None, second.uid: first.uid} with sqlite3.connect(path) as conn: assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'" diff --git a/tests/test_enrich.py b/tests/test_enrich.py index 541b95d..ed2190c 100644 --- a/tests/test_enrich.py +++ b/tests/test_enrich.py @@ -250,16 +250,16 @@ class _Context: return t -def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None: +def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_enrich() -> None: context = _Context() empty = Tally(results=({"filled": 0},)) - out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3}, + out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 4}, context=context) - assert out["next"] == "extract" and context.made[0].carries["at"] == 6 + assert out["next"] == "extract" and context.made[0].carries["at"] == 8 out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})), - {"cascade": "pull", "task": "enrich", "at": 3}, context=context) - assert out["next"] == "embed" and context.made[1].carries["at"] == 4 + {"cascade": "pull", "task": "enrich", "at": 4}, context=context) + assert out["next"] == "prep" and context.made[1].carries["at"] == 5 out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0}, context=context) @@ -290,8 +290,10 @@ def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles( assert started is not None jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr] key=lambda j: j.id) - assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"] - assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"] + assert [j.result.get("task") for j in jobs if j.kind == FINAL] == \ + ["enrich", "prep", "embed", "score"] + assert [j.kind for j in jobs if j.kind != FINAL] == \ + ["enrich", enrich.KIND, "prep", "embed", "score"] assert web["asked"] == [f"{BASE}/acme/1"] embedded = next(j for j in jobs if j.kind == "embed") assert embedded.result["embedded"] == 1 diff --git a/tests/test_extract.py b/tests/test_extract.py index df8c28c..22cefd5 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -16,6 +16,7 @@ from app.config import Config from app.jobs import extract from app.models import Limits from app.storage import Store +from prepped import upsert def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item: @@ -77,7 +78,7 @@ def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model: def test_a_batch_writes_the_columns_and_stamps_extracted_at( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: uid = posting("1").uid - store.upsert_role(posting("1")) + upsert(store, posting("1")) ranked(store, posting("1")) stub(monkeypatch, lambda roles: [answer( r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior", @@ -100,7 +101,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at( def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: item = posting("1", place="Berlin", remote="hybrid", salary_min=70000) - store.upsert_role(item) + upsert(store, item) ranked(store, item) stub(monkeypatch, lambda roles: [answer( r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000, @@ -116,13 +117,13 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced( def test_the_next_upsert_keeps_what_extract_filled( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) ranked(store, posting("1")) stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid") for r in roles]) modeltask.run(extract.TASK, config, store) - store.upsert_role(posting("1")) + upsert(store, posting("1")) role = store.role(posting("1").uid) assert role is not None and (role.country, role.seniority) == ("DE", "mid") @@ -131,7 +132,7 @@ def test_the_next_upsert_keeps_what_extract_filled( def test_closed_and_answered_roles_are_not_offered( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: for sid in ("open", "closed", "done"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) ranked(store, posting("open"), posting("closed"), posting("done")) with store.transaction() as conn: conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' " @@ -147,7 +148,7 @@ def test_closed_and_answered_roles_are_not_offered( def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) ranked(store, posting("1")) stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")]) @@ -170,7 +171,7 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read( the boards produced. A role nobody ranks is never sent; a batch dealt a uid finishes it.""" for sid in ("best", "second", "low", "unscored", "gated"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) one = store.save_agent("One", "python backend", limits=Limits(extract_top=1, extract_min=0.5)) two = store.save_agent("Two", "python backend", enabled=False) @@ -209,7 +210,7 @@ def test_a_value_outside_the_vocabulary_is_dropped() -> None: def test_disabled_finds_nothing_and_asks_nobody( store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) model = stub(monkeypatch, lambda roles: []) result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store) @@ -220,7 +221,7 @@ def test_disabled_finds_nothing_and_asks_nobody( def test_a_role_read_before_without_a_level_is_read_again_for_it( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: item = posting("1", seniority="senior") - store.upsert_role(item) + upsert(store, item) ranked(store, item) with store.transaction() as conn: conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', " @@ -243,8 +244,8 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text( withtext = posting("text", title="Senior Engineer", seniority="mid") noText = posting("teaser", title="Analyst") noText.body = "" - store.upsert_role(withtext) - store.upsert_role(noText) + upsert(store, withtext) + upsert(store, noText) with store.transaction() as conn: conn.execute("UPDATE roles SET teaser = 'Join our analytics team' " "WHERE source_id = 'teaser'") @@ -265,7 +266,7 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text( def test_an_answer_with_no_level_is_charged_until_given_up( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: item = posting("1") - store.upsert_role(item) + upsert(store, item) ranked(store, item) stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles]) @@ -279,7 +280,7 @@ def test_an_answer_with_no_level_is_charged_until_given_up( def test_each_agents_best_come_first_then_the_rest(store: Store) -> None: items = [posting(s) for s in ("a1", "a2", "a3", "b1")] for item in items: - store.upsert_role(item) + upsert(store, item) one = store.save_agent("One", "python backend") two = store.save_agent("Two", "python backend") store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""), @@ -296,7 +297,7 @@ def test_agreement_per_host(store: Store) -> None: ("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"), ("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"), ("5", "lever", "senior", "")]: - store.upsert_role(posting(sid, board=board, seniority=stated)) + upsert(store, posting(sid, board=board, seniority=stated)) with store.transaction() as conn: conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?", (model, sid)) diff --git a/tests/test_gazetteer.py b/tests/test_gazetteer.py index 4e5e9f7..5fab1c2 100644 --- a/tests/test_gazetteer.py +++ b/tests/test_gazetteer.py @@ -24,6 +24,7 @@ from app.places import Gazetteer, fold, seed from app.storage import Store from test_places import LINES, ORACLE_TOKENS +from prepped import upsert def gazetteer() -> Gazetteer: @@ -173,7 +174,7 @@ def test_a_fresh_file_is_seeded(store: Store) -> None: def test_a_joined_line_is_several_places(store: Store) -> None: - store.upsert_role(posting("1", place="Copenhagen; London; Warsaw")) + upsert(store, posting("1", place="Copenhagen; London; Warsaw")) uid = posting("1").uid assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"), (2, "Warsaw", "Warsaw")] @@ -183,7 +184,7 @@ def test_a_joined_line_is_several_places(store: Store) -> None: def test_the_places_of_the_reader_contract_come_first(store: Store) -> None: - store.upsert_role(posting("1", place="Berlin, Munich", places=[ + upsert(store, posting("1", place="Berlin, Munich", places=[ {"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False}, {"text": "Remote (DE)", "remote": True}])) assert texts(store, posting("1").uid) == [ @@ -191,7 +192,7 @@ def test_the_places_of_the_reader_contract_come_first(store: Store) -> None: def test_places_without_a_line_make_the_line(store: Store) -> None: - store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None}, + upsert(store, posting("1", places=[{"text": "Lund", "remote": None}, {"text": "Malmö", "remote": None}])) role = store.role(posting("1").uid) assert role is not None @@ -199,26 +200,26 @@ def test_places_without_a_line_make_the_line(store: Store) -> None: def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None: - store.upsert_role(posting("1", place="Aarhus")) - store.upsert_role(posting("1")) + upsert(store, posting("1", place="Aarhus")) + upsert(store, posting("1")) assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")] - store.upsert_role(posting("1", place="Odense")) + upsert(store, posting("1", place="Odense")) assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")] def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None: - store.upsert_role(posting("1", place="Aarhus")) + upsert(store, posting("1", place="Aarhus")) uid = posting("1").uid with sqlite3.connect(store.path) as conn: conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) " "VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,)) - store.upsert_role(posting("1", place="Aalborg")) + upsert(store, posting("1", place="Aalborg")) assert [(p.text, p.url) for p in store.role_places(uid)] == [ ("Aalborg", ""), ("Odense", "https://copy")] def test_an_extracted_place_is_resolved(store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid store.set_extracted(uid, {"place": "Lund, Sweden"}) assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")] @@ -236,7 +237,7 @@ def several(store: Store) -> dict[str, str]: "malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund", "zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"} for key, line in lines.items(): - store.upsert_role(posting(key, place=line)) + upsert(store, posting(key, place=line)) return {key: posting(key).uid for key in lines} @@ -274,7 +275,7 @@ def test_a_token_that_names_nothing_is_refused(store: Store) -> None: def test_a_country_given_without_a_place_still_counts(store: Store) -> None: - store.upsert_role(posting("1", country="DK")) + upsert(store, posting("1", country="DK")) assert uids(store, "country:DK") == {posting("1").uid} @@ -312,15 +313,15 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store, def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None: for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")): - store.upsert_role(posting(str(n), place=line)) + upsert(store, posting(str(n), place=line)) assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1), ("jena", "Jena", 1)] assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)] def test_an_alias_links_what_it_names(store: Store) -> None: - store.upsert_role(posting("1", place="Kbh. K")) - store.upsert_role(posting("2", place="Copenhagen")) + upsert(store, posting("1", place="Kbh. K")) + upsert(store, posting("2", place="Copenhagen")) uid = posting("1").uid assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)] assert store.add_place_alias("Kbh", "place:copenhagen") == 1 @@ -333,8 +334,8 @@ def test_an_alias_links_what_it_names(store: Store) -> None: def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None: - store.upsert_role(posting("1", place="Oberkochen; Jena")) - store.upsert_role(posting("2", place="Paderborn, Germany")) + upsert(store, posting("1", place="Oberkochen; Jena")) + upsert(store, posting("2", place="Paderborn, Germany")) oberkochen = store.add_place("Oberkochen", "city", "country:DE") store.add_place("Paderborn", "city", "country:DE") assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")] @@ -355,7 +356,7 @@ def test_an_alias_for_no_single_place_is_refused(store: Store) -> None: def test_relink_places_resolves_every_role_again(store: Store) -> None: - store.upsert_role(posting("1", place="Oberkochen")) + upsert(store, posting("1", place="Oberkochen")) with sqlite3.connect(store.path) as conn: conn.execute("UPDATE role_places SET place_id = " "(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')") @@ -381,7 +382,7 @@ def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tup lines = {} for n, (place, country) in enumerate(LINES): item = posting(f"line{n}", place=place, country=country) - store.upsert_role(item) + upsert(store, item) lines[item.uid] = (place, country) return store, lines diff --git a/tests/test_prep.py b/tests/test_prep.py new file mode 100644 index 0000000..cfa6439 --- /dev/null +++ b/tests/test_prep.py @@ -0,0 +1,358 @@ +"""`app.prep`: the host's words mapped centrally, the stage over the due +roles, the unmapped list, the table fingerprint and migration 22. The host +tables here are fakes; each real table is held to its host by R1's and R2's +tests, and to the vocabularies by the last tests of this file.""" + +from __future__ import annotations + +import importlib +import json +import sqlite3 +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import cli, models, prep # noqa: E402 +from app.prep import hosts, words # noqa: E402 +from app.prep.country import country, of_host # noqa: E402 +from app.storage import Store # noqa: E402 +from desk.models import Item # noqa: E402 +from plugins.readers import _shared # noqa: E402 + +TABLES: dict[str, Any] = { + "acme": { + "seniority": {"experienced": "", "sr": "senior"}, + "employment": {"regular full time": "full_time", "permanent": ""}, + "remote": {"2": "hybrid", "fully": "remote"}, + }, + "codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}}, + "parts": {"match": {"mode": "substring", + "rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}}, +} + + +# ------------------------------------------------------------------- words + +@pytest.mark.parametrize(("field", "word", "value"), [ + ("seniority", "SR", "senior"), + ("seniority", " Experienced ", ""), # known, names no level + ("seniority", "senior", "senior"), # already the vocabulary + ("seniority", "Grand Poobah", None), # unknown + ("employment", "Regular Full Time", "full_time"), + ("employment", "Permanent", ""), + ("employment", "contract", "contract"), + ("employment", "Zero hours", None), + ("remote", "2", "hybrid"), + ("remote", "On-site", "onsite"), # the shared words + ("remote", "on_site", "onsite"), + ("remote", "IN_OFFICE", "onsite"), + ("remote", "office", "onsite"), + ("remote", "Hybrid", "hybrid"), + ("remote", "unspecified", None), + ("seniority", "", ""), +]) +def test_a_word_goes_through_the_hosts_table_then_the_vocabulary( + field: str, word: str, value: str | None) -> None: + assert words.lookup(field, "acme", word, TABLES) == value + + +def test_another_hosts_table_is_not_used() -> None: + assert words.lookup("seniority", "other", "sr", TABLES) is None + + +def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None: + assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time" + assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time" + assert words.lookup("employment", "codes", "software", TABLES) is None + assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary" + assert words.employment("parts", "permanent_full_time", TABLES) == "full_time" + + +def test_seniority_is_the_stated_level_and_the_titles() -> None: + assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff") + assert words.seniority("acme", "", "Engineer", TABLES) == ("", "") + + +def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None: + assert words.remote("acme", True, (), "", TABLES) == "remote" + assert words.remote("acme", "true", (), "", TABLES) == "remote" + assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote" + # False says nothing the places do not: hosts send it for hybrid roles too. + assert words.remote("acme", False, (), "Berlin", TABLES) == "" + assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid" + assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \ + == "remote" + assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}], + "", TABLES) == "remote" + + +# ----------------------------------------------------------------- country + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +def ids(store: Store, *texts: str) -> list[int]: + gaz = store.gazetteer() + return [p for text in texts for p in gaz.resolve(text)] + + +def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None: + gaz = store.gazetteer() + assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK" + assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU" + assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == "" + assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB" + assert country(gaz, ids(store, "Europe")) == "EU" + assert country(gaz, [], "dk") == "DK" + assert country(gaz, ids(store, "Remote"), "Sverige") == "SE" + + +@pytest.mark.parametrize(("stated", "code"), [ + ("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"), + ("", ""), ("Narnia", None)]) +def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str, + code: str | None) -> None: + assert of_host(store.gazetteer(), stated) == code + + +# ------------------------------------------------------------------- stage + +def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme", + **raw: Any) -> Item: + found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}", + title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), + raw={"board": host, "company": "Acme"} | raw) + found.source = "ats" + return found + + +def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row: + [found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",)) + return found + + +def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None: + store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2", + places=[{"text": "Aarhus", "remote": None}])) + raw = row(store, "1") + assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \ + ("SR", "Regular Full Time", "2") + assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \ + ("", "", "", "") + assert raw["prep_due"] == 1 + assert store.role_places(raw["uid"])[0].place_id is None + + done = prep.run(store, tables=TABLES) + + assert (done.read, done.written, done.remapped) == (1, 1, True) + mapped = row(store, "1") + assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"], + mapped["remote"], mapped["country"], mapped["prep_due"]) == \ + ("senior", "senior", "full_time", "hybrid", "DK", 0) + assert store.role_places(mapped["uid"])[0].name == "Aarhus" + + +def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None: + store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}])) + store.upsert_role(item("2", seniority="sr")) + prep.run(store, tables=TABLES) + store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}])) + store.upsert_role(item("2", seniority="experienced")) + assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1) + # The place stays resolved when its text did not change. + assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus" + + store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}])) + assert row(store, "1")["prep_due"] == 1 + assert prep.run(store, tables=TABLES).read == 2 + assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == "" + + +def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None: + store.upsert_role(item("1", seniority="sr")) + [read] = store.prep_rows("due") + store.upsert_role(item("1", seniority="experienced")) + store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)]) + assert row(store, "1")["prep_due"] == 2 + + +def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract( + store: Store) -> None: + store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin")) + uid = row(store, "1")["uid"] + with store.transaction() as conn: + conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' " + "WHERE uid = ?", (uid,)) + prep.run(store, tables=TABLES) + mapped = row(store, "1") + assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \ + ("senior", "lead", "remote") + # The host's own statement does overwrite. + store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2")) + prep.run(store, tables=TABLES) + assert row(store, "1")["remote"] == "hybrid" + + +def test_the_company_resolves_through_its_aliases(store: Store) -> None: + store.add_company("Ørsted") + store.upsert_role(item("1", company="ORSTED A/S")) + assert row(store, "1")["company_key"] != "ørsted" + prep.run(store, tables=TABLES) + assert row(store, "1")["company_key"] == "ørsted" + + +def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None: + store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified", + country="Narnia")) + store.upsert_role(item("2", seniority="Grand Poobah", remote=False)) + store.upsert_role(item("3", host="other", seniority="Grand Poobah")) + done = prep.run(store, tables=TABLES) + assert done.unmapped == 4 + listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()} + assert listed == {("seniority", "acme", "grand poobah"): 2, + ("seniority", "other", "grand poobah"): 1, + ("remote", "acme", "unspecified"): 1, + ("country", "acme", "narnia"): 1} + first = {w["word"]: w["first_seen"] for w in store.unmapped()} + + # A word a table learns leaves the list at the next run; the rest keep + # the moment they were first met. + learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}} + prep.run(store, tables=learned) + assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \ + first + assert ("seniority", "acme", "grand poobah") not in \ + {(w["field"], w["host"], w["word"]) for w in store.unmapped()} + assert row(store, "1")["seniority"] == "exec" + + +def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None: + store.upsert_role(item("1", seniority="Distinguished")) + store.upsert_role(item("2", seniority="Distinguished")) + with store.transaction() as conn: + conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'") + assert prep.run(store, tables=TABLES).remapped + assert not prep.run(store, tables=TABLES).remapped + assert store.state[prep.STATE] == prep.fingerprint(TABLES) + + learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"distinguished": "staff"}}} + done = prep.run(store, tables=learned) + assert (done.remapped, done.read) == (True, 1) + assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "") + # `--remap` maps the closed roles too. + assert prep.run(store, tables=learned, remap=True).read == 2 + assert row(store, "2")["seniority"] == "staff" + + +def test_prep_folds_the_copies(store: Store) -> None: + for n in ("1", "2"): + store.upsert_role(item(n, place="Aarhus")) + assert prep.run(store, tables=TABLES).copies == 1 + assert prep.run(store, tables=TABLES, dupes=False).copies is None + + +# --------------------------------------------------------------- migration + +def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None: + path = tmp_path / "old.db" + store = Store(path) + store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True, + country="DK")) + store.upsert_role(item("2")) + store.state[prep.STATE] = "old" + with sqlite3.connect(path) as conn: + conn.execute("DROP INDEX idx_roles_prep_due") + for column in ("seniority_host", "employment_host", "remote_host", "country_host", + "employment", "prep_due"): + conn.execute(f"ALTER TABLE roles DROP COLUMN {column}") + conn.execute("DROP TABLE unmapped") + conn.execute("PRAGMA user_version = 21") + + migrated = Store(path) + + assert migrated.migrations_applied == [22] + one = row(migrated, "1") + assert (one["seniority_host"], one["employment_host"], one["remote_host"], + one["country_host"]) == ("senior", "full_time", "true", "DK") + assert row(migrated, "2")["remote_host"] == "" + assert prep.STATE not in migrated.state + # The first prep maps every open role from those words, the vocabulary + # values the old readers stored as themselves. + prep.run(migrated, tables={}) + one = row(migrated, "1") + assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \ + ("senior", "full_time", "remote", "DK") + + +# ------------------------------------------------------------ CLI and API + +def test_app_prep_runs_and_lists_the_unmapped_words( + tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + path = tmp_path / "config.yaml" + path.write_text(f"database: {tmp_path / 'roledesk.db'}\n") + Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah")) + + assert cli.main(["-c", str(path), "prep"]) == 0 + assert json.loads(capsys.readouterr().out)["read"] == 1 + assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0 + assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"] + assert cli.main(["-c", str(path), "prep", "--remap"]) == 0 + assert json.loads(capsys.readouterr().out)["remapped"] is True + + +def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None: + from fastapi.testclient import TestClient + + from app.config import Config, ServerConfig + from app.main import create_app + + app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), + database=tmp_path / "roledesk.db")) + app.state.store.upsert_role(item("1", employment="Zero hours")) + prep.run(app.state.store, tables=TABLES) + with TestClient(app) as client: + found = client.get("/api/prep/unmapped").json() + assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \ + [("employment", "acme", "zero hours", 1)] + + +# ------------------------------------------------------------- the tables + +def test_every_host_table_maps_into_its_vocabulary() -> None: + """Passes with no table at all; R1's and R2's files are held here.""" + for host, tables in hosts.load().items(): + for field, table in tables.items(): + if field == "match": + assert table["mode"] in hosts.MATCH_MODES, host + values = [value for _, value in table["rules"]] + field = "employment" + else: + assert all(key == key.strip().lower() for key in table), (host, field) + values = list(table.values()) + assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field) + + +def test_a_match_table_must_say_how_it_matches(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n') + monkeypatch.setattr(hosts, "__path__", [str(tmp_path)]) + monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False) + with pytest.raises(ValueError, match=r"loose\.py has MATCH"): + hosts.load() + (tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n') + monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False) + importlib.invalidate_caches() + assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]} + + +def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None: + assert _shared.EMPLOYMENTS == models.EMPLOYMENTS diff --git a/tests/test_pull.py b/tests/test_pull.py index 8afdd1a..2001dbb 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -24,6 +24,7 @@ from app.models import Application # noqa: E402 from plugins import readers # noqa: E402 from plugins.readers import _runner # noqa: E402 from plugins.ats import ATSSource # noqa: E402 +from app import prep # noqa: E402 from app.jobs import cascade, pull # noqa: E402 from app.storage import Store # noqa: E402 @@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0, - "roles": 1, "closed": 1, "copies": 0} + "roles": 1, "closed": 1} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} @@ -163,13 +164,14 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0, - "roles": 0, "closed": 0, "copies": 0} + "roles": 0, "closed": 0} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} -def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None: - """The settle's last step: a board's repeats of one job are one role, - and when the kept one closes the next copy is kept in its place.""" +def test_the_prep_after_the_settle_rebuilds_the_copies(store: Store) -> None: + """The prep that follows the settle folds: a board's repeats of one job + are one role, and when the kept one closes the next copy is kept in its + place.""" sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") for job_id in ("1", "2", "3"): @@ -182,8 +184,8 @@ def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None: class FakeTally: results = ({"roles": 3, "run": "run-1"},) - result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert result["copies"] == 2 + pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] + assert prep.run(store).copies == 2 [kept] = store.roles_page() assert kept.copies == 2 @@ -191,7 +193,7 @@ def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None: store.state["ats:ids:greenhouse:acme"] = json.dumps( [f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone]) result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert (result["closed"], result["copies"]) == (1, 1) + assert (result["closed"], prep.run(store).copies) == (1, 1) [after] = store.roles_page() assert after.uid != kept.uid and after.copies == 1 diff --git a/tests/test_score.py b/tests/test_score.py index 439c25e..54f4f79 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -24,6 +24,7 @@ from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402 from app.jobs import cascade, embed, score # noqa: E402 from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402 from app.storage import Store # noqa: E402 +from prepped import upsert # noqa: E402 NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC) @@ -97,7 +98,7 @@ def role(store: Store, source_id: str, title: str = "Backend Engineer", *, title=title, body=body, published_at=published_at, raw={"company": company, **raw}) item.source = "ats" - store.upsert_role(item) + upsert(store, item) return item.uid @@ -1023,7 +1024,7 @@ def test_the_shipped_starter_searches_validate() -> None: # ---------------------------------------------------------------- cascade -def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise( +def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise( store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", @@ -1053,14 +1054,15 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise( assert batch is not None jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id) stages = [j.result.get("task") for j in jobs if j.kind == FINAL] - # Enrich found no board with role pages, so the embed and score behind it - # are passed over. - assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"] + # Enrich found no board with role pages, so the prep, embed and score + # behind it are passed over. + assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score", + "summarise"] work = [j.kind for j in jobs if j.kind != FINAL] # The `pull` job and the chain it asked for are one cascade, so the Jobs # tab shows a scheduled pull as one run. Extract and summarise are off and # find nothing; score runs twice, before and after extract. - assert work == ["pull", "pull_batch", "embed", "score", "score"] + assert work == ["pull", "pull_batch", "prep", "embed", "score", "score"] assert all(j.state == "done" for j in jobs) assert len(store.scores(agent.id)) == 1 # The board was stamped with the run the `pull` job claimed, and the @@ -1109,11 +1111,12 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None: def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None: """By position, not by name: `score` is in the chain twice, once before extract and once after, and the second must lead on to summarise.""" - assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score", - "extract", "score", "summarise") - assert [cascade.follows("pull", at) for at in range(9)] == \ - ["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""] - assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == "" + assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep", + "embed", "score", "extract", "score", "summarise") + assert [cascade.follows("pull", at) for at in range(11)] == \ + ["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score", + "summarise", ""] + assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == "" assert cascade.follows("nope", 0) == "" assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0} @@ -1126,10 +1129,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None: made.append(t) return t - out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5}, - context=Context()) - assert out == {"task": "score", "next": "extract"} - assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6} out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7}, context=Context()) - assert out["next"] == "summarise" and made[1].carries["at"] == 8 + assert out == {"task": "score", "next": "extract"} + assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8} + out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9}, + context=Context()) + assert out["next"] == "summarise" and made[1].carries["at"] == 10 diff --git a/tests/test_search.py b/tests/test_search.py index d253985..fdefec1 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -20,6 +20,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 +from app import prep # noqa: E402 from plugins import readers, search # noqa: E402 from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402 from plugins.search import SearchSource # noqa: E402 @@ -28,6 +29,8 @@ from app.jobs import pull # noqa: E402 from pipeline import Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 +from prepped import upsert # noqa: E402 + FIXTURES = ROOT / "tests" / "fixtures" / "search" @@ -108,9 +111,9 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> it.source = "ats" return it - store.upsert_role(item("a", place="Copenhagen, Denmark")) - store.upsert_role(item("b", place="Remote", country="EU")) - store.upsert_role(item("c", place="Somewhere")) + upsert(store, item("a", place="Copenhagen, Denmark")) + upsert(store, item("b", place="Remote", country="EU")) + upsert(store, item("c", place="Somewhere")) assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} @@ -281,7 +284,9 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo def _pull(store: Store) -> dict[str, Any]: results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] - return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + prep.run(store) + return closed def test_a_complete_search_closes_what_it_stopped_listing( diff --git a/tests/test_storage.py b/tests/test_storage.py index b3fc5c2..2573381 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -19,6 +19,7 @@ from app.models import ( ) from app.storage.store import pack from app.storage import LATEST, Store +from prepped import upsert class Clock: @@ -69,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 21 + assert LATEST == 22 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -186,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated( migrated = Store(path) - assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21] + assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22] acme = migrated.company("acme") assert acme is not None assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example") @@ -236,9 +237,9 @@ def _downgrade_to_v4(path: Path) -> None: def test_v6_moves_the_votes_and_affinities_to_the_default_agent(tmp_path: Path) -> None: path = tmp_path / "v4.db" store = Store(path) - store.upsert_role(posting("liked")) - store.upsert_role(posting("passed")) - store.upsert_role(posting("none")) + upsert(store, posting("liked")) + upsert(store, posting("passed")) + upsert(store, posting("none")) store.add_company("Acme", key="acme") store.add_company("Other", key="other") # The disabled agent has the lower id; the enabled one is the default. @@ -274,7 +275,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it( tmp_path: Path) -> None: path = tmp_path / "v4.db" store = Store(path) - store.upsert_role(posting("liked")) + upsert(store, posting("liked")) store.add_company("Acme", key="acme") _downgrade_to_v4(path) with sqlite3.connect(path) as conn: @@ -292,7 +293,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it( def test_v6_makes_no_agent_when_there_is_nothing_to_move(tmp_path: Path) -> None: path = tmp_path / "v4.db" store = Store(path) - store.upsert_role(posting("plain")) + upsert(store, posting("plain")) store.add_company("Acme", key="acme") _downgrade_to_v4(path) @@ -322,8 +323,8 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( old_date = datetime(2020, 1, 1, tzinfo=UTC) trusted = posting("1", board="greenhouse", published_at=old_date) untrusted = posting("2", published_at=old_date) - store.upsert_role(trusted) - store.upsert_role(untrusted) + upsert(store, trusted) + upsert(store, untrusted) before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid) assert before_trusted is not None and before_untrusted is not None @@ -354,7 +355,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( given = posting("3", place="Bogota, CO", country="US") # the board said so extracted = posting("4", place="Bogota, CO") for item in (wrong, several, given, extracted): - store.upsert_role(item) + upsert(store, item) with sqlite3.connect(path) as conn: _undo_v14(conn) conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") @@ -375,7 +376,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: Path) -> None: path = tmp_path / "v13.db" store = Store(path) - store.upsert_role(posting("a")) + upsert(store, posting("a")) uid = posting("a").uid backend = store.save_agent("Backend") blank = store.save_agent("Blank") @@ -391,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21] + assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -415,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None migrated = Store(path) - assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21] + assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22] acme = migrated.company("acme") assert acme is not None assert (acme.careers_url, acme.notes, acme.hand_reason) == ( @@ -433,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( unknown = posting("unknown") unknown.title = "Backend Engineer" for item in (plain, extracted, kept, unknown): - store.upsert_role(item) + upsert(store, item) with sqlite3.connect(path) as conn: _undo_v17(conn) conn.execute("UPDATE roles SET seniority = ''") @@ -444,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( migrated = Store(path) - assert migrated.migrations_applied == [17, 18, 19, 20, 21] + assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22] with sqlite3.connect(path) as conn: rows = {uid: (fold, level) for uid, fold, level in conn.execute( "SELECT uid, place_fold, seniority FROM roles")} @@ -492,8 +493,8 @@ def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None: def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage( store: Store) -> None: - store.upsert_role(posting("a")) - store.upsert_role(posting("b")) + upsert(store, posting("a")) + upsert(store, posting("b")) a, b = posting("a").uid, posting("b").uid top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0) blank = store.save_agent("Blank", limits=top, @@ -585,11 +586,11 @@ def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None: def test_upsert_twice_keeps_first_seen_and_bumps_last_seen( store: Store, clock: Clock) -> None: - assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is True + assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is True first = store.role(posting().uid) assert first is not None clock.tick() - assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is False + assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is False again = store.role(posting().uid) assert again is not None assert again.first_seen == first.first_seen == first.created_at @@ -601,18 +602,18 @@ def test_upsert_twice_keeps_first_seen_and_bumps_last_seen( def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None: listed = posting() listed.body = "" - store.upsert_role(listed) + upsert(store, listed) store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) assert store.roles_to_embed("m", 10) == [] - store.upsert_role(listed) # listed again, still without text + upsert(store, listed) # listed again, still without text assert store.roles_to_embed("m", 10) == [] - store.upsert_role(posting()) # the role's own page was read + upsert(store, posting()) # the role's own page was read assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) - store.upsert_role(listed) # later pulls send no text - store.upsert_role(posting()) # or the same text again + upsert(store, listed) # later pulls send no text + upsert(store, posting()) # or the same text again assert store.roles_to_embed("m", 10) == [] @@ -624,9 +625,9 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives( listed.body = "" bare = posting() bare.body = "" - store.upsert_role(bare) # the title alone + upsert(store, bare) # the title alone store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))]) - store.upsert_role(listed) # a teaser beats the title + upsert(store, listed) # a teaser beats the title assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] role = store.role(listed.uid) assert role is not None and role.teaser == "Build the payments core." @@ -634,21 +635,21 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives( assert store.roles_page()[0].excerpt == "Build the payments core." store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) - store.upsert_role(bare) # a pull without it keeps it + upsert(store, bare) # a pull without it keeps it assert store.roles_to_embed("m", 10) == [] assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr] - store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it + upsert(store, posting(teaser="Build the payments core.")) # the text beats it assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] role = store.role(listed.uid) assert role is not None and document(role).endswith("We build things.") store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) - store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text + upsert(store, posting(teaser="Another teaser.")) # a new teaser under the text assert store.roles_to_embed("m", 10) == [] def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None: - store.upsert_role(posting("1")) # "Senior Backend Engineer" + upsert(store, posting("1")) # "Senior Backend Engineer" uid = posting("1").uid def levels() -> tuple[str, str, str, str]: @@ -658,12 +659,12 @@ def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> No role.seniority_title) assert levels() == ("senior", "", "", "senior") - store.upsert_role(posting("1", seniority="lead")) + upsert(store, posting("1", seniority="lead")) assert levels() == ("lead", "", "lead", "senior") - store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary + upsert(store, posting("1", seniority="Teamleitung")) # not in the vocabulary assert levels() == ("senior", "", "", "senior") store.set_extracted(uid, {"seniority": "staff"}) - store.upsert_role(posting("1", seniority="lead")) + upsert(store, posting("1", seniority="lead")) assert levels() == ("staff", "staff", "lead", "senior") @@ -675,7 +676,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> stated = posting("stated", seniority="lead") extracted = posting("extracted") for item in (joined, detailed, stated, extracted): - store.upsert_role(item) + upsert(store, item) with sqlite3.connect(path) as conn: _undo_v19(conn) # A role whose own page was read is pulled with an empty place in raw. @@ -686,7 +687,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> migrated = Store(path) - assert migrated.migrations_applied == [19, 20, 21] + assert migrated.migrations_applied == [19, 20, 21, 22] assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"] assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"] assert migrated.role_places(stated.uid) == [] @@ -703,7 +704,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: - store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000", + upsert(store, posting(place="Berlin", remote=True, salary_min="80000", salary_max=95000, currency="EUR", department="Eng")) role = store.role(posting().uid) assert role is not None @@ -713,7 +714,7 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: assert role.description == "We build things." assert role.original_url.endswith("/123") # A later fetch saying less does not erase what was known. - store.upsert_role(posting()) + upsert(store, posting()) role = store.role(posting().uid) assert role is not None assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000) @@ -725,16 +726,16 @@ def test_upsert_replaces_salary_text_including_with_empty(store: Store) -> None: every pull, and a pull that states no pay at all clears it rather than leaving a stale line behind — the opposite of the keep-filled rule `test_upsert_unpacks_raw_and_keeps_filled_columns` checks just above.""" - store.upsert_role(posting(salary_text="EUR 70,000 to 90,000 per year")) + upsert(store, posting(salary_text="EUR 70,000 to 90,000 per year")) role = store.role(posting().uid) assert role is not None and role.salary_text == "EUR 70,000 to 90,000 per year" - store.upsert_role(posting(salary_text="EUR 75,000 to 95,000 per year")) + upsert(store, posting(salary_text="EUR 75,000 to 95,000 per year")) role = store.role(posting().uid) assert role is not None and role.salary_text == "EUR 75,000 to 95,000 per year" # The board stops stating pay: the stored line is cleared, not kept. - store.upsert_role(posting()) + upsert(store, posting()) role = store.role(posting().uid) assert role is not None and role.salary_text == "" @@ -743,10 +744,10 @@ def test_salary_state_is_derived_from_the_columns_and_the_text(store: Store) -> """Not a stored column (docs/decisions.md): 'normalised' needs only `salary_min` or `salary_max`, whatever `salary_text` says beside it; 'stated' is `salary_text` alone; neither is ''.""" - store.upsert_role(posting("1", salary_text="EUR 70,000 to 90,000 per year", + upsert(store, posting("1", salary_text="EUR 70,000 to 90,000 per year", salary_min=70000, salary_max=90000, currency="EUR")) - store.upsert_role(posting("2", salary_text="USD 45 to 60 per hour")) - store.upsert_role(posting("3")) + upsert(store, posting("2", salary_text="USD 45 to 60 per hour")) + upsert(store, posting("3")) by_id = {r.source_id: r for r in store.roles()} assert by_id["1"].salary_state == "normalised" assert by_id["2"].salary_state == "stated" @@ -760,27 +761,27 @@ def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest( that finally reports the true, earlier date must still win — an edit on the board only ever moves published_at later, never earlier, so the earliest posted_at ever computed is the one to keep.""" - store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20, + upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 20, tzinfo=UTC))) first = store.role(posting().uid) assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00" clock.tick() - store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1, + upsert(store, posting(board="greenhouse", published_at=datetime(2026, 1, 1, tzinfo=UTC))) earlier = store.role(posting().uid) assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00" clock.tick() # A later, LATER published_at (an edit) must not push posted_at forward. - store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25, + upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 25, tzinfo=UTC))) still_earlier = store.role(posting().uid) assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00" def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None: - store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC))) + upsert(store, posting(published_at=datetime(2020, 1, 1, tzinfo=UTC))) role = store.role(posting().uid) assert role is not None assert role.posted_at == role.first_seen @@ -791,7 +792,7 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No only set it before extract has run, and once extract has answered — even with a different function than the pull's guess — no later pull may put the guess back. The no-flip property across pull, extract, pull.""" - store.upsert_role(posting(function="engineering")) + upsert(store, posting(function="engineering")) uid = posting().uid assert store.role(uid).function == "engineering" # type: ignore[union-attr] @@ -801,28 +802,28 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No # A later pull with the classifier's old (now wrong) guess must not flip # it back — the "board wins" COALESCE that every other raw field gets # would otherwise put "engineering" straight back on every pull. - store.upsert_role(posting(function="engineering")) + upsert(store, posting(function="engineering")) assert store.role(uid).function == "sales" # type: ignore[union-attr] def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None: - store.upsert_role(posting(function="engineering")) + upsert(store, posting(function="engineering")) uid = posting().uid store.set_extracted(uid, {}) # extract answered this role, but not function assert store.role(uid).extracted_at is not None # type: ignore[union-attr] # extracted_at alone freezes it, whatever a later pull's guess says. - store.upsert_role(posting(function="sales")) + upsert(store, posting(function="sales")) assert store.role(uid).function == "engineering" # type: ignore[union-attr] def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None: - store.upsert_role(posting(function="engineering")) + upsert(store, posting(function="engineering")) role = store.role(posting().uid) assert role is not None and role.function == "engineering" # Before extract, a fresh guess still overwrites the old one, the same # "board wins" behaviour as place, country and the rest. - store.upsert_role(posting(function="sales")) + upsert(store, posting(function="sales")) role = store.role(posting().uid) assert role is not None and role.function == "sales" @@ -834,33 +835,33 @@ def _folded(store: Store, uid: str) -> str: def test_place_fold_is_written_with_place(store: Store) -> None: - store.upsert_role(posting("1", place="København K")) + upsert(store, posting("1", place="København K")) uid = posting("1").uid assert _folded(store, uid) == "kobenhavn k" - store.upsert_role(posting("1")) # says less: both kept + upsert(store, posting("1")) # says less: both kept assert (store.role(uid).place, _folded(store, uid)) == ( # type: ignore[union-attr] "København K", "kobenhavn k") - store.upsert_role(posting("1", place="Kgs. Lyngby")) + upsert(store, posting("1", place="Kgs. Lyngby")) assert _folded(store, uid) == "kgs lyngby" - store.upsert_role(posting("2")) + upsert(store, posting("2")) store.set_extracted(posting("2").uid, {"place": "Malmö, Sweden"}) assert _folded(store, posting("2").uid) == "malmo sweden" def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> None: - store.upsert_role(posting("1")) # "Senior Backend Engineer" + upsert(store, posting("1")) # "Senior Backend Engineer" uid = posting("1").uid assert store.role(uid).seniority == "senior" # type: ignore[union-attr] retitled = posting("1") retitled.title = "Backend Engineer" - store.upsert_role(retitled) # the title no longer says + upsert(store, retitled) # the title no longer says assert store.role(uid).seniority == "" # type: ignore[union-attr] - store.upsert_role(posting("1")) + upsert(store, posting("1")) store.set_extracted(uid, {"seniority": "lead"}) # extract overwrites the title's assert store.role(uid).seniority == "lead" # type: ignore[union-attr] - store.upsert_role(posting("1")) # and a pull does not put it back + upsert(store, posting("1")) # and a pull does not put it back assert store.role(uid).seniority == "lead" # type: ignore[union-attr] store.set_extracted(uid, {}) assert store.role(uid).seniority == "lead" # type: ignore[union-attr] @@ -868,14 +869,14 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both( store: Store) -> None: - store.upsert_role(posting("1", seniority="junior")) + upsert(store, posting("1", seniority="junior")) uid = posting("1").uid role = store.role(uid) assert role is not None assert (role.seniority, role.seniority_stated, role.seniority_title, role.seniority_model) == ("junior", "junior", "senior", "") store.set_extracted(uid, {"seniority": "staff"}) - store.upsert_role(posting("1", seniority="junior")) + upsert(store, posting("1", seniority="junior")) role = store.role(uid) assert role is not None assert (role.seniority, role.seniority_stated, role.seniority_model) == \ @@ -884,18 +885,18 @@ def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both( def test_company_falls_back_to_the_subscription(store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse") - store.upsert_role(posting(), sub) + upsert(store, posting(), sub) role = store.role(posting().uid) assert role is not None assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id) def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin")) + upsert(store, posting("1", company="Acme", place="Berlin")) clock.tick() - store.upsert_role(posting("2", company="Acme", place="Munich")) + upsert(store, posting("2", company="Acme", place="Munich")) clock.tick() - store.upsert_role(posting("3", company="Other", place="Berlin")) + upsert(store, posting("3", company="Other", place="Berlin")) assert [r.source_id for r in store.roles()] == ["3", "2", "1"] assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"] assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"] @@ -905,9 +906,9 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None: def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None: """A role from a trusted board found today but posted months ago must not outrank one the app has genuinely known about for longer.""" - store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen + upsert(store, posting("recent")) # untrusted: posted_at=first_seen clock.tick() - store.upsert_role(posting("old", board="greenhouse", + upsert(store, posting("old", board="greenhouse", published_at=datetime(2020, 1, 1, tzinfo=UTC))) assert [r.source_id for r in store.roles()] == ["recent", "old"] assert [r.source_id for r in store.roles(days=30)] == ["recent"] @@ -933,7 +934,7 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index( can instead pick `idx_roles_closed` and sort every open role, which on the live database (15,558 of 15,815 roles open) measured over 30x slower than the plan this asserts (see docs/decisions.md).""" - store.upsert_role(posting("1")) + upsert(store, posting("1")) sql, _ = store._roles_from( company=None, place=None, open_only=True, q=None, function=None, remote=None, days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any", @@ -955,10 +956,10 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index( def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin", + upsert(store, posting("1", company="Acme", place="Berlin", function="engineering", remote=True)) clock.tick(hours=48) - store.upsert_role(posting("2", company="Acme", place="Munich", + upsert(store, posting("2", company="Acme", place="Munich", function="sales")) assert [r.source_id for r in store.roles(q="senior engineer")] == ["2", "1"] assert [r.source_id for r in store.roles(q="backend nonsense")] == [] @@ -977,12 +978,12 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No """`pay` filters in SQL, the same as the other filters `_roles_from` builds, so it narrows what a `limit` is taken from rather than the page already built (docs/decisions.md, "filtering before LIMIT").""" - store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000, + upsert(store, posting("normalised", salary_min=70000, salary_max=90000, currency="EUR", salary_text="EUR 70,000 to 90,000 per year")) clock.tick() - store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour")) + upsert(store, posting("stated", salary_text="USD 45 to 60 per hour")) clock.tick() - store.upsert_role(posting("none")) + upsert(store, posting("none")) assert {r.source_id for r in store.roles(pay="stated")} == {"normalised", "stated"} assert [r.source_id for r in store.roles(pay="normalised")] == ["normalised"] assert [r.source_id for r in store.roles(pay="stated_only")] == ["stated"] @@ -993,9 +994,9 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No def test_roles_filter_by_company_keys(store: Store) -> None: - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Acme Labs")) - store.upsert_role(posting("3", company="Other")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Acme Labs")) + upsert(store, posting("3", company="Other")) assert {r.source_id for r in store.roles(company_keys=["acme"])} == {"1"} assert {r.source_id for r in store.roles(company_keys=["acme", "other"])} == {"1", "3"} assert store.count_roles(company_keys=["acme-labs"]) == 1 @@ -1008,7 +1009,7 @@ def test_roles_filter_by_place_tokens_on_whole_words(store: Store) -> None: ("offices", "Copenhagen; London; Warsaw", ""), ("aarhus", "Aarhus C", "DK"), ("malmo", "Malmö", ""), ("lund", "Lund", "SE"), ("billund", "Billund", "DK"), ("berlin", "Berlin", "")): - store.upsert_role(posting(sid, place=place, country=country)) + upsert(store, posting(sid, place=place, country=country)) def found(*tokens: str) -> set[str]: return {r.source_id for r in store.roles(places=tokens)} @@ -1030,7 +1031,7 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store ("lead", "Tech Lead"), ("none", "Engineer")): item = posting(sid) item.title = title - store.upsert_role(item) + upsert(store, item) def found(*levels: str, unknown: bool = True) -> set[str]: return {r.source_id for r in store.roles(seniority=levels, seniority_unknown=unknown)} @@ -1046,9 +1047,9 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store def test_roles_pay_unknown_lets_the_roles_with_no_pay_through(store: Store) -> None: - store.upsert_role(posting("normalised", salary_min=70000, currency="EUR")) - store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour")) - store.upsert_role(posting("none")) + upsert(store, posting("normalised", salary_min=70000, currency="EUR")) + upsert(store, posting("stated", salary_text="USD 45 to 60 per hour")) + upsert(store, posting("none")) assert {r.source_id for r in store.roles(pay="normalised", pay_unknown=True)} == \ {"normalised", "none"} assert {r.source_id for r in store.roles(pay="stated", pay_unknown=True)} == \ @@ -1066,7 +1067,7 @@ def test_role_facets_count_each_facet_with_its_own_filter_left_out(store: Store) ("e", "Senior Engineer", {"company": "Other"})): item = posting(sid, **raw) item.title = title - store.upsert_role(item) + upsert(store, item) facets = store.role_facets(seniority=["junior"], seniority_unknown=False, pay="normalised") @@ -1087,10 +1088,10 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store) # One key, three spellings: the company row's name first, then the roles' # by how many carry each. store.add_company("Ørsted Group", key="ørsted") - store.upsert_role(posting("1", company="Ørsted A/S", place="Gentofte")) - store.upsert_role(posting("2", company="Ørsted", place="Gentofte")) - store.upsert_role(posting("3", company="Ørsted", place="Malmö")) - store.upsert_role(posting("4", company="Hidden", place="Malmö")) + upsert(store, posting("1", company="Ørsted A/S", place="Gentofte")) + upsert(store, posting("2", company="Ørsted", place="Gentofte")) + upsert(store, posting("3", company="Ørsted", place="Malmö")) + upsert(store, posting("4", company="Hidden", place="Malmö")) agent = store.save_agent("A") store.set_affinity(agent.id, "hidden", -1) store.set_affinity(agent.id, "ørsted", 2) @@ -1099,17 +1100,17 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store) assert sorted(store.open_companies()) == [ ("hidden", ["Hidden"], 1, 0), ("ørsted", spellings, 3, 0)] assert store.open_companies(agent.id) == [("ørsted", spellings, 3, 2)] - assert sorted(store.open_places()) == [("Gentofte", "", 2), ("Malmö", "SE", 2)] + assert sorted(store.open_places()) == [("Gentofte", "DK", 2), ("Malmö", "SE", 2)] def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None: agent = store.save_agent("A") store.add_company("Acme", key="acme") store.set_affinity(agent.id, "acme", -1) - store.upsert_role(posting("other", company="Other")) + upsert(store, posting("other", company="Other")) clock.tick() for i in range(3): - store.upsert_role(posting(f"a{i}", company="Acme")) + upsert(store, posting(f"a{i}", company="Acme")) clock.tick() # Acme's roles are the three newest, so a plain limit=1 shows only Acme; # hide_muted has to reach past all three to find "other" rather than @@ -1128,7 +1129,7 @@ def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) def test_roles_page_by_offset_and_count_the_same_filter( store: Store, clock: Clock) -> None: for sid in ("1", "2", "3", "4"): - store.upsert_role(posting(sid, company="Acme" if sid != "4" else "Other")) + upsert(store, posting(sid, company="Acme" if sid != "4" else "Other")) clock.tick() assert [r.source_id for r in store.roles(limit=2, offset=1)] == ["3", "2"] assert [r.source_id for r in store.roles(limit=2, offset=3)] == ["1"] @@ -1142,7 +1143,7 @@ def test_roles_page_by_offset_and_count_the_same_filter( def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> None: for sid in ("up", "down", "fine", "none"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") role = store.set_vote(agent.id, uid["up"], 1) @@ -1179,7 +1180,7 @@ def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> N def test_votes_are_per_agent(store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid one = store.save_agent("One") two = store.save_agent("Two") @@ -1209,7 +1210,7 @@ def test_votes_are_per_agent(store: Store) -> None: def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes( store: Store, clock: Clock) -> None: for sid in ("1", "2", "3"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) clock.tick() uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") @@ -1226,7 +1227,7 @@ def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes( def test_read_state_is_set_cleared_and_a_vote_marks_read( store: Store, clock: Clock) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid role = store.role(uid) assert role is not None and role.read_at is None @@ -1252,8 +1253,8 @@ def test_read_state_is_set_cleared_and_a_vote_marks_read( def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) -> None: - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) + upsert(store, posting("1")) + upsert(store, posting("2")) uid = {r.source_id: r.uid for r in store.roles()} store.set_read(uid["1"]) @@ -1263,7 +1264,7 @@ def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid long_tag = "x" * 50 @@ -1283,9 +1284,9 @@ def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None: def test_roles_filters_by_tags_any_and_all(store: Store) -> None: - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) - store.upsert_role(posting("3")) + upsert(store, posting("1")) + upsert(store, posting("2")) + upsert(store, posting("3")) uid = {r.source_id: r.uid for r in store.roles()} store.set_tags(uid["1"], ["python", "backend"]) store.set_tags(uid["2"], ["python", "frontend"]) @@ -1305,9 +1306,9 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) -> store.add_company("Hidden", key="hidden") agent = store.save_agent("A") store.set_affinity(agent.id, "hidden", -1) - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Acme")) - store.upsert_role(posting("3", company="Hidden")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Acme")) + upsert(store, posting("3", company="Hidden")) uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)} store.set_tags(uid["1"], ["python", "backend"]) store.set_tags(uid["2"], ["python"]) @@ -1320,7 +1321,7 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) -> def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Store) -> None: - store.upsert_role(posting("a")) + upsert(store, posting("a")) uid = store.roles()[0].uid agent = store.save_agent("A") store.delete_agent(agent.id) @@ -1330,7 +1331,7 @@ def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Sto def test_a_vote_moves_the_served_score_and_the_order(store: Store) -> None: for sid in ("a", "b", "c"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") other = store.save_agent("B") @@ -1354,13 +1355,13 @@ def test_mark_closed_closes_what_the_board_no_longer_lists(store: Store) -> None sub = store.add_subscription("ats", "greenhouse:acme") other = store.add_subscription("ats", "greenhouse:other") for source_id in ("1", "2", "3"): - store.upsert_role(posting(source_id), sub) - store.upsert_role(posting("9"), other) + upsert(store, posting(source_id), sub) + upsert(store, posting("9"), other) assert store.mark_closed("ats", ["1", "3"], sub.id) == 1 assert {r.source_id for r in store.roles()} == {"1", "3", "9"} assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"} # Listed again: open again. - store.upsert_role(posting("2"), sub) + upsert(store, posting("2"), sub) assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"} @@ -1368,7 +1369,7 @@ def test_an_item_without_a_source_is_refused(store: Store) -> None: item = posting() item.source = "" with pytest.raises(ValueError, match="no source"): - store.upsert_role(item) + upsert(store, item) # --------------------------------------------------------------- companies @@ -1472,7 +1473,7 @@ def test_the_default_agent_is_the_lowest_enabled_id(store: Store) -> None: def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None: for sid, place, country in (("lund", "Lund", "SE"), ("billund", "Billund", "DK"), ("kbh", "København K", ""), ("none", "", "")): - store.upsert_role(posting(sid, place=place, country=country)) + upsert(store, posting(sid, place=place, country=country)) agent = store.save_agent("A") uid = {r.source_id: r.uid for r in store.roles()} @@ -1490,8 +1491,8 @@ def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None def test_open_role_counts_and_company_boards(store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse") store.add_company("Acme", key="acme") - store.upsert_role(posting("1", company="Acme", place="Berlin"), sub) - store.upsert_role(posting("2", company="Acme", place="Munich"), sub) + upsert(store, posting("1", company="Acme", place="Berlin"), sub) + upsert(store, posting("2", company="Acme", place="Munich"), sub) store.mark_closed("ats", ["1"], sub.id) assert store.open_role_counts() == {"acme": 1} assert store.company_boards() == {"acme": "greenhouse"} @@ -1524,7 +1525,7 @@ def test_a_clash_on_replace_leaves_the_old_board_and_its_roles(store: Store) -> store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme") _, beta, _ = store.set_company_board("beta", "lever:beta", host="lever", name="Beta") assert beta is not None - store.upsert_role(posting("1", company="Beta"), beta) + upsert(store, posting("1", company="Beta"), beta) with pytest.raises(ValueError): store.set_company_board("beta", "greenhouse:acme", host="greenhouse", fields={"ats": "greenhouse"}) @@ -1539,8 +1540,8 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles( name="Acme") assert old is not None store.update_subscription(old.id, fetch=False, target_id="gh-1") - store.upsert_role(posting("1", company="Acme"), old) - store.upsert_role(posting("2", company="Acme"), old) + upsert(store, posting("1", company="Acme"), old) + upsert(store, posting("2", company="Acme"), old) company, new, closed = store.set_company_board("acme", "ashby:acme", host="ashby", fields={"ats": "ashby"}) @@ -1550,7 +1551,7 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles( "ashby:acme", "ashby", "", 1, 1) assert company.ats == "ashby" # The same board again closes nothing. - store.upsert_role(posting("3", company="Acme"), new) + upsert(store, posting("3", company="Acme"), new) assert store.set_company_board("acme", "ashby:acme", host="ashby")[2] == 0 assert [r.source_id for r in store.roles()] == ["3"] @@ -1560,7 +1561,7 @@ def test_detaching_a_board_hides_it_closes_its_roles_and_keeps_the_row( _, sub, _ = store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme") assert sub is not None - store.upsert_role(posting("1", company="Acme"), sub) + upsert(store, posting("1", company="Acme"), sub) company, detached, closed = store.set_company_board("acme", "") assert closed == 1 and company.key == "acme" assert detached is not None @@ -1627,7 +1628,7 @@ def test_events_are_append_only(store: Store) -> None: def test_an_application_may_point_at_a_role(store: Store) -> None: - store.upsert_role(posting()) + upsert(store, posting()) saved = store.save_application(Application(role_uid=posting().uid, status="applied")) assert saved.role_uid == posting().uid assert [e.status for e in store.application_events(saved.id)] == ["applied"] @@ -1636,7 +1637,7 @@ def test_an_application_may_point_at_a_role(store: Store) -> None: # -------------------------------------------------- summary and posting text def test_the_posting_text_is_stored_and_stamped(store: Store) -> None: - store.upsert_role(posting()) + upsert(store, posting()) role = store.set_article_text(posting().uid, "The whole posting.") assert role is not None assert role.article_text == "The whole posting." and role.article_at == store.now() @@ -1644,7 +1645,7 @@ def test_the_posting_text_is_stored_and_stamped(store: Store) -> None: def test_a_summary_is_written_once(store: Store) -> None: - store.upsert_role(posting()) + upsert(store, posting()) uid = posting().uid assert store.set_summary(uid, "First.") is True assert store.set_summary(uid, "Second.") is False @@ -1655,7 +1656,7 @@ def test_a_summary_is_written_once(store: Store) -> None: def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer( store: Store) -> None: - store.upsert_role(posting()) + upsert(store, posting()) uid = posting().uid assert store.set_summary(uid, "First.", ["Python", "backend"]) is True role = store.role(uid) @@ -1675,7 +1676,7 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits( agent that hid it. An agent saved before v8 has `limits` of '{}', which reads as the defaults.""" for sid in "abcd": - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) a, b, c, d = (posting(s).uid for s in "abcd") with store.transaction() as conn: conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d)) diff --git a/tests/test_web.py b/tests/test_web.py index fd5e3d6..9b12e21 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -23,6 +23,7 @@ from app.web.api import _score_after_change from app.web.pages import SECTIONS from auth import AuthConfig from desk.models import Item +from prepped import upsert class Clock: @@ -75,9 +76,9 @@ def client(app: FastAPI) -> Iterator[TestClient]: def test_list_roles_filters_by_query_place_and_function( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering")) - store.upsert_role(posting("2", company="Acme", place="Munich", function="sales")) - store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering")) + upsert(store, posting("1", company="Acme", place="Berlin", function="engineering")) + upsert(store, posting("2", company="Acme", place="Munich", function="sales")) + upsert(store, posting("3", company="Other", place="Berlin", function="engineering")) uid = {r.source_id: r.uid for r in store.roles()} res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"}) @@ -100,7 +101,7 @@ def test_list_roles_filters_by_company_keys_places_and_seniority( ("berlin", "Other", "Berlin", "Senior Engineer")): item = posting(sid, company=company, place=place) item.title = title - store.upsert_role(item) + upsert(store, item) uid = {r.source_id: r.uid for r in store.roles()} def shown(**params: Any) -> set[str]: @@ -133,7 +134,7 @@ def test_role_facets_count_each_facet_under_the_other_filters( ("c", "Engineer", {"salary_text": "EUR 50 per hour"})): item = posting(sid, **raw) item.title = title - store.upsert_role(item) + upsert(store, item) res = client.get("/api/roles/facets", params={"seniority": ["senior"], "seniority_unknown": "false", @@ -152,7 +153,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden( for sid, company in (("1", "Ørsted"), ("2", "Ørsted"), ("3", "Data Minds"), ("4", "Big Data Corp"), ("5", "Big Data Corp"), ("6", "Big Data Corp"), ("7", "Metadata Ltd"), ("8", "Hidden Data")): - store.upsert_role(posting(sid, company=company)) + upsert(store, posting(sid, company=company)) agent = store.save_agent("A") store.set_affinity(agent.id, "hidden-data", -1) store.set_affinity(agent.id, "ørsted", 2) @@ -177,7 +178,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden( def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None: for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"), ("4", "København K"), ("5", "Kolding")): - store.upsert_role(posting(sid, place=place)) + upsert(store, posting(sid, place=place)) res = client.get("/api/suggest/places", params={"q": "ska"}) assert res.status_code == 200 @@ -200,10 +201,10 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store) """`salary_text` and `salary_state` (app/models.py) are on every role the API serves, and `pay` (`Store.PAY`) filters the feed by it, in SQL before `limit` like the other filters.""" - store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000, + upsert(store, posting("normalised", salary_min=70000, salary_max=90000, currency="EUR", salary_text="EUR 70,000 to 90,000 per year")) - store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour")) - store.upsert_role(posting("none")) + upsert(store, posting("stated", salary_text="USD 45 to 60 per hour")) + upsert(store, posting("none")) uid = {r.source_id: r.uid for r in store.roles()} items = {r["uid"]: r for r in client.get("/api/roles").json()["items"]} @@ -223,7 +224,7 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store) def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None: for i in range(3): - store.upsert_role(posting(str(i))) + upsert(store, posting(str(i))) assert len(client.get("/api/roles", params={"limit": 1}).json()["items"]) == 1 assert client.get("/api/roles", params={"limit": 0}).status_code == 422 assert client.get("/api/roles", params={"limit": 1001}).status_code == 422 @@ -231,8 +232,8 @@ def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None: def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Store) -> None: store.add_company("Acme", key="acme") - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Other")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Other")) agent = store.save_agent("A") other = store.save_agent("B") uid = {r.source_id: r.uid for r in store.roles()} @@ -265,10 +266,10 @@ def test_muted_is_excluded_before_the_limit_not_after( store.clock = clock store.add_company("Acme", key="acme") store.set_affinity(store.save_agent("A").id, "acme", -1) - store.upsert_role(posting("other", company="Other")) + upsert(store, posting("other", company="Other")) clock.tick() for i in range(3): - store.upsert_role(posting(f"a{i}", company="Acme")) + upsert(store, posting(f"a{i}", company="Acme")) clock.tick() uid = {r.source_id: r.uid for r in store.roles()} @@ -282,7 +283,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total( clock = Clock() store.clock = clock for i in range(5): - store.upsert_role(posting(str(i))) + upsert(store, posting(str(i))) clock.tick() uid = {r.source_id: r.uid for r in store.roles()} @@ -298,7 +299,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total( def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) -> None: for sid in ("liked", "passed", "fine", "open"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") @@ -340,7 +341,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) def test_votes_are_per_agent_over_the_api(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid one = store.save_agent("One") two = store.save_agent("Two") @@ -364,7 +365,7 @@ def test_a_vote_or_an_affinity_needs_an_agent_to_belong_to( client: TestClient, store: Store) -> None: """Nothing to vote as: a 422 with a sentence, not a vote that lands nowhere. Reading is fine, everything is simply unjudged and neutral.""" - store.upsert_role(posting("1")) + upsert(store, posting("1")) store.add_company("Acme", key="acme") uid = posting("1").uid @@ -382,7 +383,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes( clock = Clock() store.clock = clock for sid in ("1", "2", "3"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) clock.tick() uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") @@ -403,7 +404,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes( def test_a_vote_moves_a_role_in_the_ranking_at_once( client: TestClient, store: Store) -> None: for sid in ("a", "b", "c"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} label = {v: k for k, v in uid.items()} agent = store.save_agent("A") @@ -424,7 +425,7 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once( def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid res = client.post(f"/api/roles/{uid}/read") @@ -440,8 +441,8 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) + upsert(store, posting("1")) + upsert(store, posting("2")) ids = {r.source_id: r.uid for r in store.roles()} uid = posting("1").uid client.post(f"/api/roles/{uid}/read") @@ -455,8 +456,8 @@ def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) + upsert(store, posting("1")) + upsert(store, posting("2")) uid = {r.source_id: r.uid for r in store.roles()} store.set_tags(uid["1"], ["python", "backend"]) store.set_tags(uid["2"], ["python", "frontend"]) @@ -476,8 +477,8 @@ def test_filters_lists_tag_counts_and_respects_show_muted( store.add_company("Hidden", key="hidden") agent = store.save_agent("A") # the default agent, whose hidden companies count store.set_affinity(agent.id, "hidden", -1) - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Hidden")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Hidden")) uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)} store.set_tags(uid["1"], ["python"]) store.set_tags(uid["2"], ["python"]) @@ -491,7 +492,7 @@ def test_filters_lists_tag_counts_and_respects_show_muted( def test_one_role_and_its_read_page(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin")) + upsert(store, posting("1", company="Acme", place="Berlin")) uid = posting("1").uid res = client.get(f"/api/roles/{uid}") @@ -524,7 +525,7 @@ def test_companies_lists_board_and_open_role_count( client: TestClient, store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse") store.add_company("Acme", key="acme", ats="greenhouse", country="DK") - store.upsert_role(posting("1", company="Acme"), sub) + upsert(store, posting("1", company="Acme"), sub) store.add_company("Other", key="other", domain="other.example") # no board found store.add_company("Novo", key="novo", ats="taleo", # followed, not readable careers_url="https://novo.example/careers") @@ -561,7 +562,7 @@ def test_judging_a_company_known_only_from_a_search_creates_its_row( for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")): item = posting(f"n{i}", company=name) item.source = "search" - store.upsert_role(item, search) + upsert(store, item, search) key = "novo-nordisk" assert store.company(key) is None @@ -712,7 +713,7 @@ def test_the_watchlist_endpoints_404_for_an_unknown_company(client: TestClient) def test_save_creates_an_application_with_a_saved_event( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin")) + upsert(store, posting("1", company="Acme", place="Berlin")) uid = posting("1").uid res = client.post("/api/applications", json={"role_uid": uid, "status": "saved"}) @@ -744,7 +745,7 @@ def test_save_an_unknown_role_is_refused(client: TestClient) -> None: def test_set_status_validates_and_writes_an_event( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme")) + upsert(store, posting("1", company="Acme")) uid = posting("1").uid saved = client.post("/api/applications", json={"role_uid": uid}).json() @@ -761,8 +762,8 @@ def test_set_status_validates_and_writes_an_event( def test_unsave_drops_a_saved_application_and_refuses_one_that_moved_on( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Acme")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Acme")) first = client.post("/api/applications", json={"role_uid": posting("1").uid}).json() second = client.post("/api/applications", json={"role_uid": posting("2").uid}).json() @@ -1069,7 +1070,7 @@ def test_a_duplicate_or_empty_prompt_name_is_a_422_sentence( def test_the_role_endpoint_serves_the_agents_score_and_its_prompt( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid agent = store.save_agent("A", prompts=[]) store.replace_scores(agent.id, [(uid, 0.8, "", "Forward deployed")]) @@ -1088,7 +1089,7 @@ def test_the_role_endpoint_serves_the_agents_score_and_its_prompt( def test_deleting_an_agent_takes_its_votes_and_affinities_with_it( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) store.add_company("Acme", key="acme") uid = posting("1").uid agent = client.post("/api/agents", json={"name": "A"}).json() @@ -1127,7 +1128,7 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated( client: TestClient, store: Store) -> None: for sid in ("low", "high", "gated"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uids = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") store.replace_scores(agent.id, [(uids["low"], 0.2, ""), (uids["high"], 0.8, ""), @@ -1152,7 +1153,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns( item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid", function="engineering") item.body = long_description - store.upsert_role(item) + upsert(store, item) uid = item.uid row = client.get("/api/roles").json()["items"][0] @@ -1184,7 +1185,7 @@ def test_list_roles_page_stays_small_even_with_long_postings( for n in range(60): item = posting(str(n), company="Acme") item.body = big - store.upsert_role(item) + upsert(store, item) with store.transaction() as conn: conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?", (big, json.dumps({"blob": big}), big)) @@ -1197,7 +1198,7 @@ def test_list_roles_page_stays_small_even_with_long_postings( def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid store.set_summary(uid, "A short, hand-written summary.") row = client.get("/api/roles").json()["items"][0] @@ -1221,9 +1222,9 @@ def test_sources_lists_searches_and_boards_with_their_countries( client: TestClient, store: Store) -> None: board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") - store.upsert_role(posting("1", company="Acme", place="Copenhagen"), board) - store.upsert_role(posting("2", company="Acme", place="Boston, MA"), board) - store.upsert_role(posting("3", company="Acme", place="Aarhus"), board) + upsert(store, posting("1", company="Acme", place="Copenhagen"), board) + upsert(store, posting("2", company="Acme", place="Boston, MA"), board) + upsert(store, posting("3", company="Acme", place="Aarhus"), board) store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend", "area": 1}) @@ -1244,7 +1245,7 @@ def test_switching_a_board_off_closes_its_roles_and_on_queues_a_pull( client: TestClient, store: Store, queue: Any) -> None: board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") - store.upsert_role(posting("1", company="Acme"), board) + upsert(store, posting("1", company="Acme"), board) off = client.post(f"/api/sources/{board.id}/switch", json={"on": False}).json() assert off["closed"] == 1 and off["source"]["on"] is False @@ -1345,7 +1346,7 @@ def test_a_companys_board_is_set_replaced_and_detached( body = res.json() assert (body["found"], body["closed"], body["source"]["target"]) == (0, 0, "greenhouse:acme") sub_id = body["source"]["id"] - store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id)) + upsert(store, posting("1", company="Acme"), store.subscription(sub_id)) [row] = client.get("/api/companies").json() assert (row["board"], row["subscription_id"], row["ats"], row["ats_slug"]) == ( "greenhouse", sub_id, "greenhouse", "acme") @@ -1418,7 +1419,7 @@ def test_a_company_from_a_search_only_is_no_longer_one_once_it_has_a_reason( search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex") item = posting("1", company="Novo") item.source = "search" - store.upsert_role(item, search) + upsert(store, item, search) client.post("/api/companies/novo/affinity", json={"affinity": 1}) [row] = client.get("/api/companies").json() assert (row["from_search"], row["by_hand"]) == (True, False) @@ -1514,14 +1515,14 @@ def test_a_company_is_deleted_unless_it_is_read( ).json()["source"]["id"] res = client.delete("/api/companies/acme") assert res.status_code == 409 and "greenhouse:acme" in res.json()["detail"] - store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id)) + upsert(store, posting("1", company="Acme"), store.subscription(sub_id)) client.delete("/api/companies/acme/board") assert client.delete("/api/companies/acme").status_code == 204 # Open roles hold a company even where no board is on: they would be # orphaned from the list. store.add_company("Beta", key="beta") - store.upsert_role(posting("2", company="Beta")) + upsert(store, posting("2", company="Beta")) res = client.delete("/api/companies/beta") assert res.status_code == 409 and "open roles" in res.json()["detail"] assert store.company("beta") is not None @@ -1562,7 +1563,7 @@ def test_the_posting_text_is_fetched_once( from desk.tools import scrape from desk.tools.article import ArticleContent - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid asked: list[str] = [] @@ -1608,8 +1609,8 @@ def test_summarise_queues_one_role_and_the_job_writes_it( monkeypatch.setitem(JOBS, "summarise_batch", replace(JOBS["summarise_batch"], isolate=False, timeout=0.0)) client = TestClient(app) - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) + upsert(store, posting("1")) + upsert(store, posting("2")) uid, bare = posting("1").uid, posting("2").uid with store.transaction() as conn: conn.execute("UPDATE roles SET description = '' WHERE uid = ?", (bare,)) @@ -1656,7 +1657,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role( board.body = "The whole posting." search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex") for item in (board, search, posting("alone", company="Other")): - store.upsert_role(item) + upsert(store, item) store.rebuild_dupes() page = client.get("/api/roles").json() @@ -1675,7 +1676,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role( def test_suggest_companies_matches_through_an_alias(client: TestClient, store: Store) -> None: store.add_company("Ørsted") - store.upsert_role(posting("1", company="Orsted")) + upsert(store, posting("1", company="Orsted")) store.add_alias("ørsted", "Dong Energy") for q in ("orsted", "ørsted", "dong"): From 56fad284a63b91519b7e346243e6118650e71fc9 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 11:23:34 +0200 Subject: [PATCH 4/4] Prep reads remote from a Danish or Swedish place line, and from a search role's title The search providers stop mapping remote_word themselves (R2), so prep reads the same words: in any role's place line after remote_of, and in the title of a role from a search. Co-Authored-By: Claude Opus 5.5 --- app/prep/stage.py | 5 +++-- app/prep/words.py | 27 ++++++++++++++++++++++++--- app/storage/store.py | 2 +- tests/test_prep.py | 20 ++++++++++++++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/app/prep/stage.py b/app/prep/stage.py index 02aa499..5c9dc8c 100644 --- a/app/prep/stage.py +++ b/app/prep/stage.py @@ -85,8 +85,9 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, "seniority_title": titled, "employment": words.employment(host, row["employment_host"], tables), "remote": words.remote(host, row["remote_host"] or None, (), "", tables), - "remote_places": words.remote(host, None, _places(row["raw_places"]), row["place"], - tables), + "remote_places": words.remote( + host, None, _places(row["raw_places"]), row["place"], tables, + title=row["title"] if row["source"] == "search" else ""), "country": country(gaz, place_ids, row["country_host"]), } diff --git a/app/prep/words.py b/app/prep/words.py index 4d71dc8..da1ee7b 100644 --- a/app/prep/words.py +++ b/app/prep/words.py @@ -44,6 +44,23 @@ def _match(rules: Mapping[str, Any], key: str) -> str | None: return None +#: Remote and hybrid in a line of Danish, Swedish or English that `remote_of` +#: does not read: "Nordhavn og mulighed for hjemmearbejde", "Arbete på distans". +#: Remote first, since "fuldt hjemmearbejde" is remote. +_REMOTE_LINE = re.compile( + r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE) +_HYBRID_LINE = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) + + +def _line_word(text: str) -> str: + """`remote_of`'s reading of a line, else its Danish and Swedish words.""" + if found := remote_of(text): + return found + if _REMOTE_LINE.search(text): + return "remote" + return "hybrid" if _HYBRID_LINE.search(text) else "" + + def _squash(word: str) -> str: return "".join(c for c in word.lower() if c not in " -_") @@ -86,7 +103,7 @@ def _place_text(place: Any) -> str: def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "", - tables: Tables = TABLES) -> str: + tables: Tables = TABLES, title: str = "") -> str: """How the role is worked, one of `REMOTE` or ''. `value` is the host's own (a bool, or its word; `host_word`'s 'true' and @@ -95,7 +112,9 @@ def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "", every role that is not remote, hybrid ones included. A word goes through the host's table, else the shared map. When the value says nothing, the place line or the place texts are read for a word - (`remote_of`: "Hybrid, Berlin"), and last a place the reader marked + (`remote_of`: "Hybrid, Berlin", and the Danish and Swedish words), then + `title` when given, which the stage gives for a search provider's role + only ("Backend Developer - Remote"), and last a place the reader marked remote is remote.""" if value is True or value == "true": return "remote" @@ -103,7 +122,9 @@ def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "", if said and (found := lookup("remote", host, str(value), tables)): return found places = list(places) - if found := remote_of(line or "; ".join(map(_place_text, places))): + if found := _line_word(line or "; ".join(map(_place_text, places))): + return found + if title and (found := _line_word(title)): return found if any(isinstance(p, Mapping) and p.get("remote") is True for p in places): return "remote" diff --git a/app/storage/store.py b/app/storage/store.py index b5b9932..2959cb0 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -1273,7 +1273,7 @@ class Store: due (`prep_due` above 0), every open role, or every role.""" where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope] return self._rows( - f"""SELECT uid, prep_due, {_HOST} AS host, title, company, company_key, place, + f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place, seniority_host, employment_host, remote_host, country_host, seniority_model, seniority_stated, seniority_title, seniority, employment, remote, country, diff --git a/tests/test_prep.py b/tests/test_prep.py index cfa6439..734683b 100644 --- a/tests/test_prep.py +++ b/tests/test_prep.py @@ -93,6 +93,26 @@ def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() - "", TABLES) == "remote" +def test_a_line_in_danish_or_swedish_and_a_search_title_are_read() -> None: + assert words.remote("jobindex", None, (), "Nordhavn og mulighed for hjemmearbejde") \ + == "hybrid" + assert words.remote("jobindex", None, (), "Aarhus, fuldt hjemmearbejde") == "remote" + assert words.remote("platsbanken", None, (), "Malmö, arbete på distans") == "remote" + assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer - Remote") \ + == "remote" + assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer") == "" + + +def test_a_search_roles_title_counts_and_a_boards_does_not(store: Store) -> None: + store.upsert_role(item("1", title="Engineer (remote)", place="Berlin")) + store.upsert_role(item("2", title="Engineer (remote)", place="Berlin", host="jobindex", + provider="jobindex")) + with store.transaction() as conn: + conn.execute("UPDATE roles SET source = 'search' WHERE source_id = 'jobindex:2'") + prep.run(store, tables=TABLES) + assert (row(store, "1")["remote"], row(store, "2", "jobindex")["remote"]) == ("", "remote") + + # ----------------------------------------------------------------- country @pytest.fixture