Share the readers' place, table-lookup and country helpers, unify the employment vocabulary, and list the enrich switch with the stages

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 10:13:24 +02:00
1 parent ca80219bbb
commit 903b69b6d3
34 files changed
+286 -203

No files matched your search

+2 -2
View File
@@ -152,8 +152,8 @@ def build_parser() -> argparse.ArgumentParser:
prompt.add_argument("--remove", action="store_true", help="remove the prompt instead") prompt.add_argument("--remove", action="store_true", help="remove the prompt instead")
stages = sub.add_parser( stages = sub.add_parser(
"stages", help="show whether extract and summarise run, or switch one") "stages", help="show whether extract, summarise and enrich run, or switch one")
stages.add_argument("name", nargs="?", choices=["extract", "summarise"]) stages.add_argument("name", nargs="?", choices=["extract", "summarise", "enrich"])
stages.add_argument("state", nargs="?", choices=["on", "off"]) stages.add_argument("state", nargs="?", choices=["on", "off"])
sub.add_parser("seniority", help="per host, how often the stated level equals the model's") sub.add_parser("seniority", help="per host, how often the stated level equals the model's")
+10 -4
View File
@@ -13,8 +13,8 @@ finders ask it (`extract.extract_task`, `summarise.summarise_task`).
asks no model and is on by default (`enrich.enabled`); its switch is asks no model and is on by default (`enrich.enabled`); its switch is
`stage:enrich`, set the same way through `PUT /api/stages/enrich`. `stage:enrich`, set the same way through `PUT /api/stages/enrich`.
`describe` is what `GET /api/stages` and `app stages` show: for each model `describe` is what `GET /api/stages` and `app stages` show: for each stage
stage whether it is on, where that comes from, the model it uses, and who with a switch, the model stages and `enrich`, whether it is on, where that comes from, the model it uses, and who
would answer. would answer.
""" """
@@ -33,6 +33,8 @@ NAMES = ("extract", "summarise")
#: Every stage with a switch: the model stages and `enrich`. #: Every stage with a switch: the model stages and `enrich`.
SWITCHES = (*NAMES, "enrich") SWITCHES = (*NAMES, "enrich")
#: What `describe_one` says of a stage that asks no model.
NO_ASK = "reads role pages, asks no model"
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, " NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
"see docs/llm-backends.md") "see docs/llm-backends.md")
ANSWERS = { ANSWERS = {
@@ -42,7 +44,7 @@ ANSWERS = {
class Stage(BaseModel): class Stage(BaseModel):
"""One model stage as the page shows it. `source` is "stored" when the """One stage as the page shows it. `source` is "stored" when the
switch set it and "config" when it is still config.yaml's default. switch set it and "config" when it is still config.yaml's default.
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says `backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
it in words.""" it in words."""
@@ -89,6 +91,10 @@ def answer(config: Config) -> tuple[str, str]:
def describe_one(config: Config, store: Store, name: str) -> Stage: def describe_one(config: Config, store: Store, name: str) -> Stage:
if name not in NAMES:
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
model="", backend="", answer=NO_ASK)
backend, said = answer(config) backend, said = answer(config)
return Stage(name=name, on=stage_on(config, store, name), return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored", source="config" if stored(store, name) is None else "stored",
@@ -97,4 +103,4 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
def describe(config: Config, store: Store) -> list[Stage]: def describe(config: Config, store: Store) -> list[Stage]:
return [describe_one(config, store, name) for name in NAMES] return [describe_one(config, store, name) for name in SWITCHES]
+3 -2
View File
@@ -1,5 +1,5 @@
/* The Runs section of Sources: the "Run the pull now" button, and the /* The Runs section of Sources: the "Run the pull now" button, and the
* switches for the two stages that ask a model. * switches for the stages of a pull: the two that ask a model and enrich.
* *
* The switches are `/api/stages`: one row per stage, drawn as the search * The switches are `/api/stages`: one row per stage, drawn as the search
* rows above are (`.srow`, `.switch`), and a switch is a PUT that stores the * rows above are (`.srow`, `.switch`), and a switch is a PUT that stores the
@@ -62,6 +62,7 @@
const DOES = { const DOES = {
extract: "fills in country, seniority, salary and the like from what the posting says", extract: "fills in country, seniority, salary and the like from what the posting says",
summarise: "writes a short summary and tags for the best roles", summarise: "writes a short summary and tags for the best roles",
enrich: "reads the own page of the best roles whose list gave no text",
}; };
const el = (tag, className, text) => { const el = (tag, className, text) => {
@@ -105,7 +106,7 @@
if (!stage.on) row.dataset.muted = "true"; if (!stage.on) row.dataset.muted = "true";
const name = el("span", "name"); const name = el("span", "name");
name.append(el("span", "", stage.name[0].toUpperCase() + stage.name.slice(1)), name.append(el("span", "", stage.name[0].toUpperCase() + stage.name.slice(1)),
el("span", "badge", stage.model)); ...(stage.model ? [el("span", "badge", stage.model)] : []));
row.append(name, stageSwitch(stage, () => renderStages(stages))); row.append(name, stageSwitch(stage, () => renderStages(stages)));
const sub = el("span", "sub"); const sub = el("span", "sub");
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `); sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
+1 -1
View File
@@ -414,7 +414,7 @@ class StageBody(BaseModel):
@router.get("/stages") @router.get("/stages")
def list_stages(store: StoreDep, config: ConfigDep) -> list[jobs.stages.Stage]: def list_stages(store: StoreDep, config: ConfigDep) -> list[jobs.stages.Stage]:
"""The two model stages of a pull: whether each is on, whether the """The stages of a pull with a switch, extract, summarise and enrich: whether each is on, whether the
stored switch or config.yaml says so, its model, and who would answer.""" stored switch or config.yaml says so, its model, and who would answer."""
return jobs.stages.describe(config, store) return jobs.stages.describe(config, store)
+23
View File
@@ -178,6 +178,29 @@ posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role copy is a row with its one place, and `dupes.py` folds them into one role
that keeps each copy's link. that keeps each copy's link.
**Helpers.** `._shared` holds what more than one host needs; a module keeps
only its own word tables and uses these:
- `place(text, remote=None)` makes a `Place`: remote when the text has the
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
appends a place unless its text is empty or already in `found`, in the
host's order. A host that states remote itself passes it to `place`, or
builds the dict and passes that to `add_place`.
- `table_word(table, value, log, what)` maps a host's word (stated seniority
or employment) through the module's table, without case. The table's keys
are lowercase. Empty gives None; a known word mapped to `""` (`experienced`,
`permanent`) names no level and gives None; an unknown word gives None and
a debug line naming the host, so the table can grow.
- `EMPLOYMENTS`, the one employment vocabulary: `full_time`, `part_time`,
`contract`, `temporary`, `internship`, `freelance`. Every value of a
module's `_EMPLOYMENT` table is one of these or `""`; the test in
`tests/test_readers_shared.py` holds every module to it. Seniority values
are `app.models.SENIORITIES`.
- `country_if_single(codes)` is the code when the role's places share exactly
one, else None, for a host that states a country per place.
- `remote_word(text)` reads `remote` or `hybrid` from a search provider's
headline or town line (Danish, Swedish, English).
**Teaser.** A short text the list carries (an excerpt, a summary) is handed **Teaser.** A short text the list carries (an excerpt, a summary) is handed
over as `teaser`, plain text, whether or not the full text comes later. It over as `teaser`, plain text, whether or not the full text comes later. It
is shown and embedded until `description` arrives. A reader that sees one is shown and embedded until `description` arrives. A reader that sees one
+59
View File
@@ -8,6 +8,7 @@ import it without either importing the other. The leading underscore keeps
from __future__ import annotations from __future__ import annotations
import logging
import re import re
from collections.abc import Callable from collections.abc import Callable
from datetime import UTC, datetime from datetime import UTC, datetime
@@ -16,6 +17,8 @@ from typing import Any, Protocol
import httpx import httpx
from ._contract import Place
#: What an undeclared module's parser turns a feed into; a module that #: What an undeclared module's parser turns a feed into; a module that
#: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`. #: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`.
#: One dict per row, with the keys #: One dict per row, with the keys
@@ -192,3 +195,59 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) ->
tail = f" per {period}" if period else "" tail = f" per {period}" if period else ""
code = str(currency or "").strip().upper() code = str(currency or "").strip().upper()
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)" return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
#: The employment words a row's `employment` may hold, for every host. A
#: host's table maps its own words onto these; a word that names none of them
#: (a permanent contract says nothing about hours) maps to "".
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
def place(text: str, remote: bool | None = None) -> Place:
"""A `Place` for `text`: remote when the text says the word `remote`,
otherwise as `remote` says (None when the host says nothing)."""
return {"text": text, "remote": True if _REMOTE_WORD.search(text) else remote}
def add_place(found: list[Place], entry: Place) -> None:
"""Append `entry` to `found` unless its text is empty or already there;
the host's order is kept."""
if entry["text"] and entry["text"] not in (p["text"] for p in found):
found.append(entry)
def table_word(table: dict[str, str], value: Any, log: logging.Logger, what: str) -> str | None:
"""`table`'s word for a host's `value`, looked up without case. None when
the host states none, when the table maps the word to "" (a known word
that names no level, such as `experienced`), or when the table does not
know it; that last case is logged at debug level so the table can grow."""
word = str(value or "").strip()
if not word:
return None
found = table.get(word.lower())
if found is None:
log.debug("%s: unknown %s %r", log.name.rsplit(".", 1)[-1], what, word)
return found or None
def country_if_single(codes: Any) -> str | None:
"""The one code among `codes` when there is exactly one, else None: a
country is stated for a role only when all its places share it."""
return next(iter(codes)) if len(codes) == 1 else None
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def remote_word(text: str) -> str | None:
"""`remote`, `hybrid` or None: the word a search provider's headline or
town line says about where the work is done, in Danish, Swedish and
English. A board states its own remote flag and needs none of this."""
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
-19
View File
@@ -1,19 +0,0 @@
"""What the search providers' headlines and town lines say about where the
work is done, in Danish, Swedish and English. Shared by the search readers;
a board states its own remote flag and needs none of this."""
from __future__ import annotations
import re
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def remote_word(text: str) -> str | None:
"""`remote`, `hybrid` or None: the word the text says."""
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
+1 -2
View File
@@ -24,8 +24,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Key, Place, Reads from ._contract import Ask, Fields, Filters, Key, Place, Reads
from ._words import remote_word from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "adzuna" NAME = "adzuna"
LABEL = "Adzuna" LABEL = "Adzuna"
+4 -13
View File
@@ -9,14 +9,13 @@ from __future__ import annotations
import json import json
import logging import logging
import re
from collections.abc import Callable from collections.abc import Callable
from typing import Any from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -42,7 +41,7 @@ _API = "api.ashbyhq.com"
_WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"} _WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"}
#: `employmentType` -> the app's employment words. #: `employmentType` -> the app's employment words.
_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "intern", _EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship",
"contract": "contract", "temporary": "temporary"} "contract": "contract", "temporary": "temporary"}
@@ -124,20 +123,12 @@ def _places(j: dict[str, Any], workplace: str | None) -> list[Place]:
for s in j.get("secondaryLocations") or []] for s in j.get("secondaryLocations") or []]
out: list[Place] = [] out: list[Place] = []
for text, remote in named: for text, remote in named:
if text and text not in (p["text"] for p in out): add_place(out, place(text, remote))
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else remote})
return out return out
def _employment(value: Any) -> str | None: def _employment(value: Any) -> str | None:
word = str(value or "").strip() return table_word(_EMPLOYMENT, value, log, "employmentType")
if not word:
return None
found = _EMPLOYMENT.get(word.lower())
if found is None:
log.debug("ashby: unknown employmentType %r", word)
return found
def parse(body: bytes, slug: str) -> list[Row]: def parse(body: bytes, slug: str) -> list[Row]:
+4 -12
View File
@@ -17,14 +17,13 @@ from __future__ import annotations
import html import html
import json import json
import logging import logging
import re
from collections.abc import Callable from collections.abc import Callable
from typing import Any from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, label_slug_ok, only, parse_dt from ._shared import Get, label_slug_ok, only, parse_dt, place, table_word
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -39,7 +38,7 @@ READS = Reads(
#: the company's own ("Full-Time", "Part-Time", "Contract", "Intern"). #: the company's own ("Full-Time", "Part-Time", "Contract", "Intern").
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", _EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract", "part time": "part_time", "contract": "contract", "contractor": "contract",
"temporary": "temporary", "intern": "intern", "internship": "intern", "temporary": "temporary", "intern": "internship", "internship": "internship",
"freelance": "freelance"} "freelance": "freelance"}
NAME = "bamboohr" NAME = "bamboohr"
@@ -97,18 +96,11 @@ def _place(j: dict[str, Any]) -> str:
def _places(j: dict[str, Any]) -> list[Place]: def _places(j: dict[str, Any]) -> list[Place]:
"""The one place BambooHR states, if it states any.""" """The one place BambooHR states, if it states any."""
text = _place(j) text = _place(j)
return [{"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}] \ return [place(text)] if text else []
if text else []
def _employment(j: dict[str, Any]) -> str | None: def _employment(j: dict[str, Any]) -> str | None:
label = str(j.get("employmentStatusLabel") or "").strip() return table_word(_EMPLOYMENT, j.get("employmentStatusLabel"), log, "employmentStatusLabel")
if not label:
return None
found = _EMPLOYMENT.get(label.lower())
if found is None:
log.debug("bamboohr: unknown employmentStatusLabel %r", label)
return found
def parse(body: bytes, slug: str) -> list[Row]: def parse(body: bytes, slug: str) -> list[Row]:
+5 -8
View File
@@ -24,7 +24,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row from ._contract import Ask, Fields, Reads, Row
from ._shared import num, only, parse_dt, pay_line from ._shared import num, only, parse_dt, pay_line, table_word
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -65,8 +65,8 @@ _REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
#: The name of a `jobEmploymentTypes` entry -> the app's employment words. #: The name of a `jobEmploymentTypes` entry -> the app's employment words.
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", _EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract", "part time": "part_time", "contract": "contract", "contractor": "contract",
"consultant": "contract", "temporary": "temporary", "intern": "intern", "consultant": "contract", "temporary": "temporary", "intern": "internship",
"internship": "intern", "freelance": "freelance"} "internship": "internship", "freelance": "freelance"}
def slug_ok(slug: str) -> bool: def slug_ok(slug: str) -> bool:
@@ -123,12 +123,9 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]:
def _employment(job: dict[str, Any]) -> str | None: def _employment(job: dict[str, Any]) -> str | None:
"""The first `jobEmploymentTypes` entry that names a known type.""" """The first `jobEmploymentTypes` entry that names a known type."""
for entry in job.get("jobEmploymentTypes") or []: for entry in job.get("jobEmploymentTypes") or []:
name = str(((entry or {}).get("employmentType") or {}).get("name") or "").strip() name = ((entry or {}).get("employmentType") or {}).get("name")
if not name: if found := table_word(_EMPLOYMENT, name, log, "employment type"):
continue
if found := _EMPLOYMENT.get(name.lower()):
return found return found
log.debug("deel: unknown employment type %r", name)
return None return None
+2 -3
View File
@@ -44,7 +44,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads, Row from ._contract import Ask, Fields, Filters, Place, Reads, Row
from ._countries import name_of from ._countries import name_of
from ._shared import Get, Page, is_web_url, only, parse_dt from ._shared import Get, Page, add_place, is_web_url, only, parse_dt, place
READS = Reads( READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated", scope="board", listing="paged", page_size=10, text="detail", complete="stated",
@@ -131,8 +131,7 @@ def _places(locations: Any) -> list[Place]:
if name and name not in names: if name and name not in names:
names.append(name) names.append(name)
text = ", ".join(names) text = ", ".join(names)
if text and text not in (p["text"] for p in out): add_place(out, place(text))
out.append({"text": text, "remote": True if "remote" in text.lower() else None})
return out return out
+6 -10
View File
@@ -27,7 +27,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, only, parse_dt from ._shared import Get, Page, add_place, only, parse_dt, table_word
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -69,9 +69,9 @@ DETAIL_QUERY = (
_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"} _REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
#: `job.employmentType` -> the app's employment words. #: `job.employmentType` -> the app's employment words.
_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACT": "contract", _EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contract": "contract",
"CONTRACTOR": "contract", "INTERN": "intern", "INTERNSHIP": "intern", "contractor": "contract", "intern": "internship", "internship": "internship",
"TEMPORARY": "temporary", "FREELANCE": "freelance"} "temporary": "temporary", "freelance": "freelance"}
def slug_ok(slug: str) -> bool: def slug_ok(slug: str) -> bool:
@@ -131,8 +131,7 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
places: list[Place] = [] places: list[Place] = []
for loc in locations: for loc in locations:
text = str(loc.get("city") or loc.get("name") or "").strip() text = str(loc.get("city") or loc.get("name") or "").strip()
if text and text not in (pl["text"] for pl in places): add_place(places, {"text": text, "remote": loc["isRemote"]
places.append({"text": text, "remote": loc["isRemote"]
if isinstance(loc.get("isRemote"), bool) else None}) if isinstance(loc.get("isRemote"), bool) else None})
job = p.get("job") or {} job = p.get("job") or {}
remote = _REMOTE.get(str(job.get("locationType") or "").upper()) remote = _REMOTE.get(str(job.get("locationType") or "").upper())
@@ -150,11 +149,8 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
"remote": remote, "remote": remote,
"salary_text": "", "salary_text": "",
} }
if word := str(job.get("employmentType") or "").strip().upper(): if employment := table_word(_EMPLOYMENT, job.get("employmentType"), log, "employmentType"):
if employment := _EMPLOYMENT.get(word):
row["employment"] = employment row["employment"] = employment
else:
log.debug("gem: unknown employmentType %r", word)
out.append(row) out.append(row)
return out return out
+4 -7
View File
@@ -15,7 +15,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
NAME = "greenhouse" NAME = "greenhouse"
LABEL = "Greenhouse" LABEL = "Greenhouse"
@@ -132,19 +132,16 @@ def _places(j: dict[str, Any]) -> list[Place]:
out: list[Place] = [] out: list[Place] = []
for office in j.get("offices") or []: for office in j.get("offices") or []:
text = str((office or {}).get("location") or (office or {}).get("name") or "").strip() text = str((office or {}).get("location") or (office or {}).get("name") or "").strip()
if text and text not in (p["text"] for p in out): add_place(out, place(text))
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
if not out or not name or _PLACEHOLDER.fullmatch(name): if not out or not name or _PLACEHOLDER.fullmatch(name):
return out or ([{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) return out or ([place(name)] if name else [])
else None}] if name else [])
# A name that says something the offices do not (a "Remote India" beside # A name that says something the offices do not (a "Remote India" beside
# an office in Bangalore) is one more place; one they already contain, # an office in Bangalore) is one more place; one they already contain,
# or that contains them, is the same place said twice. # or that contains them, is the same place said twice.
low = name.lower() low = name.lower()
if any(p["text"].lower() in low or low in p["text"].lower() for p in out): if any(p["text"].lower() in low or low in p["text"].lower() for p in out):
return out return out
return [{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) else None}, *out] return [place(name), *out]
def parse(body: bytes, slug: str) -> list[Row]: def parse(body: bytes, slug: str) -> list[Row]:
+3 -7
View File
@@ -23,7 +23,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -44,7 +44,7 @@ _EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time",
"full-time": "full_time", "regular part time": "part_time", "full-time": "full_time", "regular part time": "part_time",
"part time": "part_time", "part-time": "part_time", "contract": "contract", "part time": "part_time", "part-time": "part_time", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "fixed term": "temporary", "contractor": "contract", "temporary": "temporary", "fixed term": "temporary",
"intern": "intern", "internship": "intern", "freelance": "freelance"} "intern": "internship", "internship": "internship", "freelance": "freelance"}
NAME = "hibob" NAME = "hibob"
LABEL = "HiBob" LABEL = "HiBob"
@@ -132,11 +132,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"salary_max": salary_max, "salary_max": salary_max,
"currency": code, "currency": code,
} }
kind = str(ad.get("employmentType") or "").strip() if employment := table_word(_EMPLOYMENT, ad.get("employmentType"), log, "employmentType"):
if kind:
if employment := _EMPLOYMENT.get(kind.lower()):
row["employment"] = employment row["employment"] = employment
else:
log.debug("hibob: unknown employmentType %r", kind)
out.append(row) out.append(row)
return out return out
+6 -12
View File
@@ -38,7 +38,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, only, parse_dt from ._shared import Get, Page, add_place, only, parse_dt, place, table_word
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -50,9 +50,9 @@ READS = Reads(
) )
#: `employment_type` -> the app's employment words. #: `employment_type` -> the app's employment words.
_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACTOR": "contract", _EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contractor": "contract",
"CONTRACT": "contract", "TEMPORARY": "temporary", "INTERN": "intern", "contract": "contract", "temporary": "temporary", "intern": "internship",
"INTERNSHIP": "intern", "FREELANCE": "freelance"} "internship": "internship", "freelance": "freelance"}
NAME = "jibe" NAME = "jibe"
LABEL = "Jibe" LABEL = "Jibe"
@@ -120,9 +120,7 @@ def _places(j: dict[str, Any]) -> list[Place]:
out: list[Place] = [] out: list[Place] = []
for part in line.split(";"): for part in line.split(";"):
text = part.strip() text = part.strip()
if text and text not in (p["text"] for p in out): add_place(out, place(text))
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
return out return out
@@ -162,11 +160,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
code = str(j.get("country_code") or "").strip().upper() code = str(j.get("country_code") or "").strip().upper()
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code): if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code):
row["country"] = code row["country"] = code
kind = str(j.get("employment_type") or "").strip() if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
if kind:
if employment := _EMPLOYMENT.get(kind.upper()):
row["employment"] = employment row["employment"] = employment
else:
log.debug("jibe: unknown employment_type %r", kind)
out.append(row) out.append(row)
return out return out
+1 -2
View File
@@ -28,8 +28,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html from desk.sources import strip_html
from ._words import remote_word from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
NAME = "jobindex" NAME = "jobindex"
+9 -25
View File
@@ -25,8 +25,8 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, parse_dt from ._shared import Get, parse_dt, place, table_word
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -70,7 +70,7 @@ _SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern",
#: The embed's `employment_type` -> the app's employment words. #: The embed's `employment_type` -> the app's employment words.
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", _EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "consultant": "contract", "part time": "part_time", "contract": "contract", "consultant": "contract",
"temporary": "temporary", "internship": "intern", "intern": "intern", "temporary": "temporary", "internship": "internship", "intern": "internship",
"freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"} "freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"}
_ASSIGNMENT = "JBL.embed_v2['jobs'] = " _ASSIGNMENT = "JBL.embed_v2['jobs'] = "
@@ -179,22 +179,6 @@ def _published(value: Any) -> Any:
return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}") return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}")
def _mapped(table: dict[str, str], value: Any, what: str) -> str | None:
"""`table`'s word for the host's `value`; None when the host states none
or the table does not know it (logged, so the table can grow)."""
word = str(value or "").strip()
if not word:
return None
found = table.get(word.lower())
if found is None:
log.debug("jobylon: unknown %s %r", what, word)
return found or None
def _place(text: str) -> Place:
return {"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}
def parse(body: bytes, slug: str) -> list[Row]: def parse(body: bytes, slug: str) -> list[Row]:
"""The embed page -> one row per role. `posted` is the day `published_date` """The embed page -> one row per role. `posted` is the day `published_date`
names, which `detail` replaces with the exact time. `location` joins the names, which `detail` replaces with the exact time. `location` joins the
@@ -210,7 +194,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
if not title or not job_id or not _ROLE_PATH.fullmatch(path): if not title or not job_id or not _ROLE_PATH.fullmatch(path):
continue continue
function = str(j.get("function") or "").strip() function = str(j.get("function") or "").strip()
places = [_place(name) for name in dict.fromkeys( places = [place(name) for name in dict.fromkeys(
str(name).strip() for name in j.get("locations") or [] if str(name).strip())] str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
row: Row = { row: Row = {
"source_id": job_id, "source_id": job_id,
@@ -229,9 +213,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split()) summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split())
if summary and summary != "None": if summary and summary != "None":
row["teaser"] = summary row["teaser"] = summary
if seniority := _mapped(_SENIORITY, j.get("experience"), "experience"): if seniority := table_word(_SENIORITY, j.get("experience"), log, "experience"):
row["seniority"] = seniority row["seniority"] = seniority
if employment := _mapped(_EMPLOYMENT, j.get("employment_type"), "employment_type"): if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
row["employment"] = employment row["employment"] = employment
out.append(row) out.append(row)
return out return out
@@ -252,8 +236,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
else: else:
raise ValueError("no JobPosting on the role's page") raise ValueError("no JobPosting on the role's page")
places = [] places = []
for place in data.get("jobLocation") or []: for loc in data.get("jobLocation") or []:
address = (place or {}).get("address") or {} address = (loc or {}).get("address") or {}
name = str(address.get("streetAddress") or address.get("addressLocality") or "").strip() name = str(address.get("streetAddress") or address.get("addressLocality") or "").strip()
if name and name not in places: if name and name not in places:
places.append(name) places.append(name)
@@ -263,5 +247,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
} }
if places: if places:
found["location"] = "; ".join(places) found["location"] = "; ".join(places)
found["places"] = [_place(name) for name in places] found["places"] = [place(name) for name in places]
return found return found
+4 -10
View File
@@ -24,7 +24,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -40,7 +40,7 @@ READS = Reads(
#: text on Lever ("Permanent" is seen too and says nothing about hours). #: text on Lever ("Permanent" is seen too and says nothing about hours).
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time", _EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time",
"part-time": "part_time", "part time": "part_time", "parttime": "part_time", "part-time": "part_time", "part time": "part_time", "parttime": "part_time",
"intern": "intern", "internship": "intern", "contract": "contract", "intern": "internship", "internship": "internship", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "freelance": "freelance"} "contractor": "contract", "temporary": "temporary", "freelance": "freelance"}
NAME = "lever" NAME = "lever"
@@ -115,9 +115,7 @@ def _places(categories: dict[str, Any]) -> list[Place]:
out: list[Place] = [] out: list[Place] = []
for name in names: for name in names:
text = str(name or "").strip() text = str(name or "").strip()
if text and text not in (p["text"] for p in out): add_place(out, place(text))
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
return out return out
@@ -158,11 +156,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
country = str(j.get("country") or "").strip().upper() country = str(j.get("country") or "").strip().upper()
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country): if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country):
row["country"] = country row["country"] = country
commitment = str(categories.get("commitment") or "").strip() if employment := table_word(_EMPLOYMENT, categories.get("commitment"), log, "commitment"):
if commitment:
if employment := _EMPLOYMENT.get(commitment.lower()):
row["employment"] = employment row["employment"] = employment
else:
log.debug("lever: unknown commitment %r", commitment)
rows.append(row) rows.append(row)
return rows return rows
+3 -4
View File
@@ -35,7 +35,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES, code_of from ._countries import NAMES, code_of
from ._shared import Get, Page, Row, only, parse_dt from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt
NAME = "oracle" NAME = "oracle"
LABEL = "Oracle Recruiting Cloud" LABEL = "Oracle Recruiting Cloud"
@@ -197,8 +197,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
_text(role.get("PrimaryLocationCountry"))), _text(role.get("PrimaryLocationCountry"))),
*((_text(s.get("Name")), _text(s.get("CountryCode"))) *((_text(s.get("Name")), _text(s.get("CountryCode")))
for s in secondary)): for s in secondary)):
if name and name not in (p["text"] for p in places): add_place(places, {"text": name, "remote": {"remote": True, "hybrid": False,
places.append({"text": name, "remote": {"remote": True, "hybrid": False,
"onsite": False}.get(remote or "")}) "onsite": False}.get(remote or "")})
if code: if code:
codes.add(code.upper()) codes.add(code.upper())
@@ -213,7 +212,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
or _text(role.get("Department")) or _text(role.get("Organization"))), or _text(role.get("Department")) or _text(role.get("Organization"))),
"description": "", "description": "",
"teaser": _plain(role.get("ShortDescriptionStr")), "teaser": _plain(role.get("ShortDescriptionStr")),
"country": next(iter(codes)) if len(codes) == 1 else None, "country": country_if_single(codes),
"remote": remote, "remote": remote,
"salary_text": "", # the resource states no pay "salary_text": "", # the resource states no pay
}) })
+3 -9
View File
@@ -14,7 +14,7 @@ from collections.abc import Callable
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, table_word, yearly_amount
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -86,12 +86,7 @@ def _salary_text(salary_info: ET.Element) -> str:
def _seniority(word: str) -> str | None: def _seniority(word: str) -> str | None:
key = word.strip().lower() return table_word(_SENIORITY, word, log, "seniority")
if key not in _SENIORITY:
if key:
log.debug("personio: unknown seniority %r", word)
return None
return _SENIORITY[key] or None
def _employment(position: ET.Element) -> str | None: def _employment(position: ET.Element) -> str | None:
@@ -113,8 +108,7 @@ def _places(position: ET.Element) -> list[Place]:
*(o.text for o in position.findall("additionalOffices/office"))] *(o.text for o in position.findall("additionalOffices/office"))]
for name in names: for name in names:
text = (name or "").strip() text = (name or "").strip()
if text and text not in (p["text"] for p in found): add_place(found, place(text))
found.append({"text": text, "remote": True if "remote" in text.lower() else None})
return found return found
+4 -5
View File
@@ -31,7 +31,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import name_of from ._countries import name_of
from ._shared import Get, Page, Row, only, parse_dt from ._shared import Get, Page, Row, add_place, only, parse_dt
NAME = "phenom" NAME = "phenom"
LABEL = "Phenom" LABEL = "Phenom"
@@ -163,10 +163,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
if not title or not key: if not title or not key:
continue continue
places: list[Place] = [] places: list[Place] = []
for place in (job.get("location"), *(job.get("multi_location") or [])): for text in (job.get("location"), *(job.get("multi_location") or [])):
if isinstance(place, str) and place.strip() \ if isinstance(text, str):
and place.strip() not in (p["text"] for p in places): add_place(places, {"text": text.strip(), "remote": None})
places.append({"text": place.strip(), "remote": None})
tail = _slug_of(title) tail = _slug_of(title)
out.append({ out.append({
"source_id": key, "source_id": key,
+1 -2
View File
@@ -22,8 +22,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._words import remote_word
NAME = "platsbanken" NAME = "platsbanken"
LABEL = "Platsbanken" LABEL = "Platsbanken"
+5 -10
View File
@@ -15,7 +15,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount from ._shared import Row, add_place, country_if_single, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -83,12 +83,7 @@ def _salary_text(salary: dict[str, Any]) -> str:
def _seniority(code: str) -> str | None: def _seniority(code: str) -> str | None:
key = code.strip().lower() return table_word(_SENIORITY, code, log, "experience_code")
if key not in _SENIORITY:
if key:
log.debug("recruitee: unknown experience_code %r", code)
return None
return _SENIORITY[key] or None
def _employment(code: str) -> str | None: def _employment(code: str) -> str | None:
@@ -112,8 +107,8 @@ def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
def add(text: str) -> None: def add(text: str) -> None:
text = text.strip() text = text.strip()
if text and text.lower() not in _NO_PLACE and text not in (p["text"] for p in found): if text.lower() not in _NO_PLACE:
found.append({"text": text, "remote": None}) add_place(found, {"text": text, "remote": None})
for loc in offer.get("locations") or []: for loc in offer.get("locations") or []:
if not isinstance(loc, dict): if not isinstance(loc, dict):
@@ -165,7 +160,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"salary_min": salary_min, "salary_min": salary_min,
"salary_max": salary_max, "salary_max": salary_max,
"currency": currency, "currency": currency,
"country": next(iter(codes)) if len(codes) == 1 else None, "country": country_if_single(codes),
"seniority": _seniority(str(j.get("experience_code") or "")), "seniority": _seniority(str(j.get("experience_code") or "")),
"employment": _employment(str(j.get("employment_type_code") or "")), "employment": _employment(str(j.get("employment_type_code") or "")),
}) })
+3 -3
View File
@@ -20,7 +20,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Row, parse_dt, yearly_amount from ._shared import Get, Row, add_place, parse_dt, yearly_amount
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -100,8 +100,8 @@ def parse(body: bytes, slug: str) -> list[Row]:
continue continue
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip() place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
places.setdefault(job_id, []) places.setdefault(job_id, [])
if place and place not in (p["text"] for p in places[job_id]): if place:
places[job_id].append(_place(place)) add_place(places[job_id], _place(place))
roles.setdefault(job_id, { roles.setdefault(job_id, {
"source_id": job_id, "source_id": job_id,
"title": title, "title": title,
+2 -5
View File
@@ -19,7 +19,7 @@ from collections.abc import Callable
from typing import Any from typing import Any
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok, table_word
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -97,10 +97,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None: def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None:
"""The `id` of the object `job[key]` looked up in `table`; none for a word """The `id` of the object `job[key]` looked up in `table`; none for a word
the table maps to none, and a word it does not know is logged.""" the table maps to none, and a word it does not know is logged."""
word = str((job.get(key) or {}).get("id") or "").strip().lower() return table_word(table, (job.get(key) or {}).get("id"), log, key)
if word and word not in table:
log.debug("smartrecruiters: unknown %s %r", key, word)
return table.get(word) or None
def rows(data: dict[str, Any], slug: str) -> list[Row]: def rows(data: dict[str, Any], slug: str) -> list[Row]:
+3 -4
View File
@@ -59,7 +59,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES from ._countries import NAMES
from ._shared import Get, Page, Row, is_web_url, only, parse_dt from ._shared import Get, Page, Row, add_place, country_if_single, is_web_url, only, parse_dt
NAME = "successfactors" NAME = "successfactors"
LABEL = "SuccessFactors" LABEL = "SuccessFactors"
@@ -392,13 +392,12 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
addresses = _addresses(text) or [meta] addresses = _addresses(text) or [meta]
places: list[Place] = [] places: list[Place] = []
for address in addresses: for address in addresses:
if (place := _location(address)) and place not in (p["text"] for p in places): add_place(places, {"text": _location(address), "remote": None})
places.append({"text": place, "remote": None})
if places: if places:
out["places"] = places out["places"] = places
out["location"] = "; ".join(p["text"] for p in places) out["location"] = "; ".join(p["text"] for p in places)
codes = {_country_code(address) for address in addresses} codes = {_country_code(address) for address in addresses}
if len(codes) == 1 and (code := next(iter(codes))): if code := country_if_single(codes):
out["country"] = code out["country"] = code
return out return out
+3 -4
View File
@@ -17,7 +17,7 @@ from typing import Any
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
NAME = "teamtailor" NAME = "teamtailor"
LABEL = "Teamtailor" LABEL = "Teamtailor"
@@ -129,9 +129,8 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
for loc in item.findall(f"{_TT}locations/{_TT}location"): for loc in item.findall(f"{_TT}locations/{_TT}location"):
city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip() city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip()
country = (loc.findtext(f"{_TT}country") or "").strip() country = (loc.findtext(f"{_TT}country") or "").strip()
place = city if country == city else ", ".join(p for p in (city, country) if p) text = city if country == city else ", ".join(p for p in (city, country) if p)
if place and place not in (p["text"] for p in places): add_place(places, {"text": text, "remote": remote})
places.append({"text": place, "remote": remote})
out.append({ out.append({
"source_id": (item.findtext("guid") or url).strip(), "source_id": (item.findtext("guid") or url).strip(),
"title": title, "title": title,
+4 -8
View File
@@ -13,7 +13,7 @@ from collections.abc import Callable
from typing import Any from typing import Any
from ._contract import Ask, Fields, Place, Reads from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt, path_slug_ok, table_word
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -82,8 +82,7 @@ def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
if part and part not in parts: if part and part not in parts:
parts.append(part) parts.append(part)
text = ", ".join(parts) text = ", ".join(parts)
if text and text not in (p["text"] for p in found): add_place(found, {"text": text, "remote": remote})
found.append({"text": text, "remote": remote})
if code := str(loc.get("countryCode") or "").strip().upper(): if code := str(loc.get("countryCode") or "").strip().upper():
codes.add(code) codes.add(code)
return found, codes return found, codes
@@ -107,9 +106,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
continue continue
remote = bool(j.get("remote")) or j.get("workplace") == "remote" remote = bool(j.get("remote")) or j.get("workplace") == "remote"
places, codes = _places(j, remote) places, codes = _places(j, remote)
kind = str(j.get("type") or "").strip().lower()
if kind and kind not in _EMPLOYMENT:
log.debug("workable: unknown type %r", kind)
dept = j.get("department") or "" dept = j.get("department") or ""
if isinstance(dept, list): if isinstance(dept, list):
dept = ", ".join(d for d in dept if d) dept = ", ".join(d for d in dept if d)
@@ -124,7 +120,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"description": "", "description": "",
"remote": remote, "remote": remote,
"salary_text": "", # Workable's list endpoint states no pay at all "salary_text": "", # Workable's list endpoint states no pay at all
"country": next(iter(codes)) if len(codes) == 1 else None, "country": country_if_single(codes),
"employment": _EMPLOYMENT.get(kind), "employment": table_word(_EMPLOYMENT, j.get("type"), log, "type"),
}) })
return out return out
+4 -5
View File
@@ -39,7 +39,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import code_of from ._countries import code_of
from ._shared import Get, Page, Row, only, parse_dt from ._shared import Get, Page, Row, add_place, only, parse_dt, table_word
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -320,7 +320,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
out["location"] = "; ".join(p["text"] for p in places) out["location"] = "; ".join(p["text"] for p in places)
if len(places) == 1 and (code := _country_code(posting)): if len(places) == 1 and (code := _country_code(posting)):
out["country"] = code out["country"] = code
if employment := _EMPLOYMENT.get(str(posting.get("timeType") or "").strip().lower()): if employment := table_word(_EMPLOYMENT, posting.get("timeType"), log, "timeType"):
out["employment"] = employment out["employment"] = employment
if posted := parse_dt(posting.get("startDate")): if posted := parse_dt(posting.get("startDate")):
out["posted"] = posted out["posted"] = posted
@@ -344,9 +344,8 @@ def _places(posting: dict[str, Any], remote: str | None) -> list[Place]:
(LEGO's "Billund") gets it from the role's `country`.""" (LEGO's "Billund") gets it from the role's `country`."""
found: list[Place] = [] found: list[Place] = []
for name in [posting.get("location"), *(posting.get("additionalLocations") or [])]: for name in [posting.get("location"), *(posting.get("additionalLocations") or [])]:
place = _place(name, remote) if place := _place(name, remote):
if place and place["text"] not in (p["text"] for p in found): add_place(found, place)
found.append(place)
country = str((posting.get("country") or {}).get("descriptor") or "").strip() country = str((posting.get("country") or {}).get("descriptor") or "").strip()
if len(found) == 1 and country and country.lower() not in found[0]["text"].lower(): if len(found) == 1 and country and country.lower() not in found[0]["text"].lower():
found[0]["text"] += f", {country}" found[0]["text"] += f", {country}"
+1 -1
View File
@@ -141,7 +141,7 @@ def test_the_experience_is_mapped_and_a_word_the_table_lacks_is_logged(
assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values()) assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values())
rows = {r["source_id"]: r for r in jobylon.parse( rows = {r["source_id"]: r for r in jobylon.parse(
(FIXTURES / "jobylon-places.html").read_bytes(), "3143")} (FIXTURES / "jobylon-places.html").read_bytes(), "3143")}
assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "intern" assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "internship"
def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None: def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None:
+1 -1
View File
@@ -56,7 +56,7 @@ def test_it_reads_title_company_town_date_and_teaser() -> None:
def test_it_reads_hybrid_and_remote_from_the_town_line() -> None: def test_it_reads_hybrid_and_remote_from_the_town_line() -> None:
from plugins.readers._words import remote_word from plugins.readers._shared import remote_word
assert remote_word("Birkerød, Hybrid position") == "hybrid" assert remote_word("Birkerød, Hybrid position") == "hybrid"
assert remote_word("Nordhavn og mulighed for hjemmearbejde") == "hybrid" assert remote_word("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
+85
View File
@@ -0,0 +1,85 @@
"""The helpers the host modules share, and the one employment vocabulary."""
from __future__ import annotations
import logging
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import _shared # noqa: E402
from plugins.readers._shared import ( # noqa: E402
EMPLOYMENTS,
add_place,
country_if_single,
place,
remote_word,
table_word,
)
#: The module-level names a reader keeps its employment words in.
_TABLES = ("_EMPLOYMENT", "_CONTRACT", "_SCHEDULE")
def test_every_employment_table_maps_into_the_shared_vocabulary() -> None:
found = 0
for name, module in readers.READERS.items():
for table_name in _TABLES:
table = getattr(module, table_name, None)
if table is None:
continue
found += 1
values = table.values() if isinstance(table, dict) else [v for _k, v in table]
# "" is a known word that names no employment ("permanent").
assert set(values) <= {*EMPLOYMENTS, ""}, (name, table_name)
assert found >= 15
def test_the_vocabulary_is_the_documented_one() -> None:
assert EMPLOYMENTS == ("full_time", "part_time", "contract", "temporary", "internship",
"freelance")
def test_place_is_remote_on_the_word_and_otherwise_as_stated() -> None:
assert place("Remote (DE)") == {"text": "Remote (DE)", "remote": True}
assert place("Berlin, remote") == {"text": "Berlin, remote", "remote": True}
assert place("Berlin") == {"text": "Berlin", "remote": None}
assert place("Berlin", False) == {"text": "Berlin", "remote": False}
assert place("Remote", False)["remote"] is True
assert place("Remotely", None)["remote"] is None
def test_add_place_skips_empty_and_repeated_texts_and_keeps_order() -> None:
found: list = []
for text in ("Berlin", "", "Oslo", "Berlin"):
add_place(found, place(text))
assert [p["text"] for p in found] == ["Berlin", "Oslo"]
def test_table_word_maps_ignores_case_and_logs_only_unknown_words(caplog) -> None:
table = {"full-time": "full_time", "permanent": ""}
log = logging.getLogger("plugins.readers.acme")
with caplog.at_level(logging.DEBUG, logger=log.name):
assert table_word(table, " Full-Time ", log, "type") == "full_time"
assert table_word(table, "Permanent", log, "type") is None
assert table_word(table, None, log, "type") is None
assert not caplog.records
assert table_word(table, "Odd", log, "type") is None
assert [r.getMessage() for r in caplog.records] == ["acme: unknown type 'Odd'"]
def test_country_if_single_needs_exactly_one_code() -> None:
assert country_if_single({"DE"}) == "DE"
assert country_if_single({"DE", "SE"}) is None
assert country_if_single(set()) is None
def test_remote_word_reads_danish_swedish_and_english() -> None:
assert remote_word("Remote, Danmark") == "remote"
assert remote_word("Birkerød, Hybrid position") == "hybrid"
assert remote_word("Hedehusene") is None
assert _shared.remote_word is remote_word
+17 -3
View File
@@ -143,7 +143,7 @@ def client(app: FastAPI) -> Iterator[TestClient]:
yield c yield c
def test_get_stages_lists_both_with_their_source_model_and_who_answers( def test_get_stages_lists_all_three_with_their_source_model_and_who_answers(
client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
answered_by(monkeypatch, "api") answered_by(monkeypatch, "api")
res = client.get("/api/stages") res = client.get("/api/stages")
@@ -156,6 +156,8 @@ def test_get_stages_lists_both_with_their_source_model_and_who_answers(
{"name": "summarise", "on": False, "source": "config", {"name": "summarise", "on": False, "source": "config",
"model": "claude-haiku-4-5-20251001", "backend": "api", "model": "claude-haiku-4-5-20251001", "backend": "api",
"answer": "answered by the Claude API"}, "answer": "answered by the Claude API"},
{"name": "enrich", "on": True, "source": "config", "model": "", "backend": "",
"answer": "reads role pages, asks no model"},
] ]
@@ -164,7 +166,7 @@ def test_who_answers_is_said_for_the_api_claude_code_and_nobody(
answers = {} answers = {}
for backend in ("api", "claude_code", ""): for backend in ("api", "claude_code", ""):
answered_by(monkeypatch, backend) answered_by(monkeypatch, backend)
[first, second] = client.get("/api/stages").json() [first, second, _enrich] = client.get("/api/stages").json()
assert first["backend"] == second["backend"] == backend assert first["backend"] == second["backend"] == backend
answers[backend] = first["answer"] answers[backend] = first["answer"]
@@ -191,6 +193,18 @@ def test_put_stores_the_setting_returns_the_same_shape_and_starts_nothing(
assert res.json()["on"] is False and store.state["stage:summarise"] == "off" assert res.json()["on"] is False and store.state["stage:summarise"] == "off"
def test_the_enrich_switch_is_listed_and_stored_like_the_others(
client: TestClient, store: Store) -> None:
res = client.put("/api/stages/enrich", json={"on": False})
assert res.status_code == 200
assert res.json()["name"] == "enrich" and res.json()["on"] is False
assert store.state["stage:enrich"] == "off"
listed = {stage["name"]: stage for stage in client.get("/api/stages").json()}
assert list(listed) == ["extract", "summarise", "enrich"]
assert listed["enrich"]["on"] is False and listed["enrich"]["source"] == "stored"
def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> None: def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> None:
assert client.put("/api/stages/embed", json={"on": True}).status_code == 404 assert client.put("/api/stages/embed", json={"on": True}).status_code == 404
assert client.put("/api/stages/extract", json={"on": "sometimes"}).status_code == 422 assert client.put("/api/stages/extract", json={"on": "sometimes"}).status_code == 422
@@ -200,7 +214,7 @@ def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> No
def test_agent_backend_off_is_named_rather_than_blamed_on_missing_keys( def test_agent_backend_off_is_named_rather_than_blamed_on_missing_keys(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, store: Store) -> None: tmp_path: Path, monkeypatch: pytest.MonkeyPatch, store: Store) -> None:
off = Config(database=str(tmp_path / "roledesk.db"), agent={"backend": "off"}) off = Config(database=str(tmp_path / "roledesk.db"), agent={"backend": "off"})
[first, _] = stages.describe(off, store) [first, *_] = stages.describe(off, store)
assert first.backend == "" and "agent.backend" in first.answer assert first.backend == "" and "agent.backend" in first.answer